1 |
function [nt,nf] = mnc_assembly(fpat,vars, fout,fsize) |
2 |
|
3 |
% Function [nt,nf] = mnc_assembly(fpat,vars, fout,fsize) |
4 |
% |
5 |
% INPUTS |
6 |
% fpat string containing the file pattern |
7 |
% vars structure array of variable names |
8 |
% |
9 |
% fout output file pattern (DEF: "all.%05d.nc") |
10 |
% fsize max output file size (DEF: 2.0e+9 = +/-2GB) |
11 |
% |
12 |
% OUTPUTS |
13 |
% nt number of usable tiles found |
14 |
% nf number of output files written |
15 |
% |
16 |
% This function "assembles" MNC output. It finds all the per-tile |
17 |
% NetCDF files that match the input pattern, does some basic "sanity" |
18 |
% tests to determine whether the files have compatible sizes, and |
19 |
% then assembles all of the requested data (all of the variables) |
20 |
% into one or more "global" NetCDF files. The global files have |
21 |
% the following dimension conventions: |
22 |
% |
23 |
% "exch 1": all values are within a global horizontal grid |
24 |
% and indicies are (X,Y,Z,T) |
25 |
% |
26 |
% "exch 2": all values are within one of up to six "faces" |
27 |
% of a global cube with indicies (Xf,Yf,F,Z,T) |
28 |
% |
29 |
% where "X,Y.Z,T" are global space/time indicies, "Xf,Yf" are local |
30 |
% per-face spatial indicies, and "F" is a face index. |
31 |
% |
32 |
% An example of how to use this script is: |
33 |
% |
34 |
% vars = struct([]); |
35 |
% vars(1).name = 'iter'; |
36 |
% vars(2).name = 'U'; |
37 |
% vars(3).name = 'Unk'; |
38 |
% vars(4).name = 'V'; |
39 |
% vars(5).name = 'Temp'; |
40 |
% vars(6).name = 'S'; |
41 |
% fpat = 'exp0_20041126_0001/state.0000.%06d.nc'; |
42 |
% [nt,nf] = mnc_assembly(fpat,vars); |
43 |
% |
44 |
% and the resutlt is written as "all.00000.nc" |
45 |
|
46 |
% $Header: $ |
47 |
% $Name: $ |
48 |
|
49 |
%===== Argument checking and defaults ===== |
50 |
|
51 |
if nargin < 2 |
52 |
disp('Error: there must be at least 2 arguments!'); |
53 |
return |
54 |
end |
55 |
|
56 |
if nargin < 3 |
57 |
fout = 'all.%05d.nc'; |
58 |
end |
59 |
if nargin < 4 |
60 |
fsize = 2.0e+9; |
61 |
end |
62 |
|
63 |
|
64 |
%===== Find and open all the matching files ===== |
65 |
|
66 |
nt = 0; |
67 |
nf = 0; |
68 |
all_ncf = struct([]); |
69 |
|
70 |
% Find all of the files |
71 |
exch2_msg = 0; |
72 |
tmax = 200; |
73 |
frdone = 0; |
74 |
it = 0; |
75 |
while frdone == 0 |
76 |
|
77 |
it = it + 1; |
78 |
fnm = sprintf(fpat,it); |
79 |
% disp(fnm); |
80 |
|
81 |
% Check that the file exists |
82 |
fid = fopen(fnm, 'r'); |
83 |
if fid < 0 |
84 |
if it >= tmax |
85 |
frdone = 1; |
86 |
end |
87 |
continue; |
88 |
end |
89 |
|
90 |
% Open the NetCDF file |
91 |
fnc = netcdf(fnm, 'nowrite'); |
92 |
if length(fnc) == 0 |
93 |
continue; |
94 |
end |
95 |
|
96 |
% Check for exch1/exch2 grid |
97 |
exch = 1; |
98 |
exch2_myFace = fnc.exch2_myFace(:); |
99 |
if length(exch2_myFace) ~= 0 |
100 |
exch = 2; |
101 |
if exch2_msg == 0 |
102 |
exch2_msg = 1; |
103 |
disp(' Grid type appears to be: "exch2"'); |
104 |
end |
105 |
end |
106 |
|
107 |
n = length(all_ncf) + 1; |
108 |
all_ncf(n).name = fnm; |
109 |
all_ncf(n).nc = {fnc}; |
110 |
all_ncf(n).exch = exch; |
111 |
all_ncf(n).tile_number = fnc.tile_number(1); |
112 |
all_ncf(n).bi = fnc.bi(1); |
113 |
all_ncf(n).bj = fnc.bj(1); |
114 |
all_ncf(n).sNx = fnc.sNx(1); |
115 |
all_ncf(n).sNy = fnc.sNy(1); |
116 |
all_ncf(n).Nx = fnc.Nx(1); |
117 |
all_ncf(n).Ny = fnc.Ny(1); |
118 |
all_ncf(n).Z = fnc.Z(1); |
119 |
|
120 |
if exch == 2 |
121 |
all_ncf(n).exch2_myFace = exch2_myFace; |
122 |
all_ncf(n).exch2_tbasex = fnc.exch2_tbasex(1); |
123 |
all_ncf(n).exch2_tbasey = fnc.exch2_tbasex(1); |
124 |
end |
125 |
|
126 |
clear fnc |
127 |
end |
128 |
|
129 |
|
130 |
%===== Do some basic sanity checks ===== |
131 |
|
132 |
% check for number of files/tiles found |
133 |
if length(all_ncf) == 0 |
134 |
disp('Error: no tiles found--no need to do any assembly!'); |
135 |
return |
136 |
elseif length(all_ncf) == 1 |
137 |
disp('Error: one tile found--no need to do any assembly!'); |
138 |
return |
139 |
else |
140 |
disp(sprintf(' Found %d files matching the pattern: "%s"', ... |
141 |
length(all_ncf), fpat )); |
142 |
end |
143 |
|
144 |
% check for consistent "exch" version |
145 |
if prod(double([all_ncf.exch] == all_ncf(1).exch)) ~= 1 |
146 |
disp('Error: not all the "exch" types of the files match.'); |
147 |
return; |
148 |
end |
149 |
|
150 |
% check for consistent sNx,sNy |
151 |
if (prod(double([all_ncf.sNx] == all_ncf(1).sNx)) ~= 1) ... |
152 |
| (prod(double([all_ncf.sNy] == all_ncf(1).sNy)) ~= 1) |
153 |
disp('Error: the "sNx,sNy" values for all the tiles are not'); |
154 |
disp(' uniform. Future versions of this function will be'); |
155 |
disp(' able to handle non-uniform grid sizes but this'); |
156 |
disp(' feature is not yet implemented.'); |
157 |
return; |
158 |
end |
159 |
|
160 |
% check for redundant tiles and "time series" output |
161 |
if length(all_ncf) ~= length(unique([all_ncf.tile_number])) |
162 |
disp('Error: redundant tiles were found. Please check that'); |
163 |
disp(' the file pattern does not specify output spanning'); |
164 |
disp(' multiple model runs or even multiple time series'); |
165 |
disp(' within a single model run. For multi-time-series'); |
166 |
disp(' data sets, EACH "LEVEL" IN THE OUTPUT SERIES MUST'); |
167 |
disp(' BE ASSEMBLED SEPARATERLY.'); |
168 |
return |
169 |
end |
170 |
|
171 |
|
172 |
%===== Get the dims/vars associations ===== |
173 |
|
174 |
mydims = struct('names', {}, 'lens', {}); |
175 |
myvars = struct([]); |
176 |
clear tncf; |
177 |
for ivar = 1:length(vars) |
178 |
mydim_names = {}; |
179 |
mydim_sizes = {}; |
180 |
myatt.names = {}; |
181 |
myatt.types = {}; |
182 |
myatt.data = {}; |
183 |
myname = vars(ivar).name; |
184 |
disp([' Looking for variable: ' myname]); |
185 |
|
186 |
itile = 1; |
187 |
tncf = all_ncf(itile).nc{1}; |
188 |
ncv = tncf{myname}; |
189 |
len = length(ncv); |
190 |
if length(ncv) == 0 |
191 |
warns = [' Warning: variable "%s" is not defined in "%s"\n' ... |
192 |
' so it will be ignored.']; |
193 |
disp(sprintf(warns,myname,all_ncf(itile).name)); |
194 |
continue |
195 |
end |
196 |
mytype = datatype(ncv); |
197 |
tmpdims = dim(ncv); |
198 |
for inm = 1:length(tmpdims) |
199 |
mydim_names{inm} = name(tmpdims{inm}); |
200 |
mydim_sizes{inm} = tmpdims{inm}(:); |
201 |
end |
202 |
for iat = 1:length(att(ncv)) |
203 |
aaa = att(ncv); |
204 |
myatt.names(iat) = { name(aaa{iat}) }; |
205 |
myatt.types(iat) = { datatype(aaa{iat}) }; |
206 |
aab = aaa{iat}; |
207 |
myatt.data(iat) = { aab(:) }; |
208 |
end |
209 |
|
210 |
% confirm: vars have same dim names across all files |
211 |
ierr = 0; |
212 |
for itile = 2:length(all_ncf) |
213 |
tncf = all_ncf(itile).nc{1}; |
214 |
ncv = tncf{myname}; |
215 |
len = length(ncv); |
216 |
if length(ncv) == 0 |
217 |
warns = [' Warning: variable "%s" is not defined in "%s"\n' ... |
218 |
' so it will be ignored.']; |
219 |
disp(sprintf(warns,myname,all_ncf(itile).name)); |
220 |
continue |
221 |
end |
222 |
tmpdims = dim(ncv); |
223 |
for inm = 1:length(tmpdims) |
224 |
if mydim_names{inm} ~= name(tmpdims{inm}) |
225 |
warns = ... |
226 |
[' Warning: variable "%s" is not CONSISTENTLY defined.\n' ... |
227 |
' It has different dimensions in different files so\n' ... |
228 |
' so it will be ignored.']; |
229 |
disp(sprintf(warns,myname)); |
230 |
ierr = 1; |
231 |
break |
232 |
end |
233 |
mydim_sizes{inm} = max([ tmpdims{inm}(:) mydim_sizes{inm} ]); |
234 |
end |
235 |
|
236 |
end |
237 |
|
238 |
if ierr == 0 |
239 |
% check: does the variable have a "horizontal" component |
240 |
has_horiz = 0; |
241 |
horiz_names = { 'X' 'Y' 'Xp1' 'Yp1' }; |
242 |
for id = 1:length(mydim_names) |
243 |
if length([intersect(horiz_names,mydim_names{id})]) > 0 |
244 |
has_horiz = 1; |
245 |
end |
246 |
end |
247 |
% disp([ ' ' myname ' ' sprintf('%d',has_horiz) ]); |
248 |
|
249 |
imy = length(myvars) + 1; |
250 |
myvars(imy).name = myname; |
251 |
myvars(imy).type = mytype; |
252 |
myvars(imy).dim_names = mydim_names; |
253 |
myvars(imy).dim_sizes = mydim_sizes; |
254 |
myvars(imy).atts = myatt; |
255 |
myvars(imy).has_horiz = has_horiz; |
256 |
|
257 |
% this is necessary to make it work with Matlab 6.5 |
258 |
if isempty([mydims.names]) |
259 |
addl = mydim_names; |
260 |
else |
261 |
addl = setdiff(mydim_names,[mydims.names]); |
262 |
end |
263 |
for iaddl = 1:length(addl) |
264 |
np1 = length(mydims) + 1; |
265 |
mydims(np1).names = addl(iaddl); |
266 |
mydims(np1).lens = mydim_sizes(find(strcmp(addl(iaddl),mydim_names))); |
267 |
end |
268 |
|
269 |
end |
270 |
end |
271 |
|
272 |
% For exch == 2, we need to add a "face" dimension |
273 |
if all_ncf(1).exch == 2 |
274 |
np1 = length(mydims) + 1; |
275 |
mydims(np1).names = { 'iface' }; |
276 |
mydims(np1).lens = { length(unique([all_ncf.exch2_myFace])) }; |
277 |
end |
278 |
|
279 |
% myvars.name |
280 |
% myvars.dim_names |
281 |
% myvars.dim_sizes |
282 |
% myvars(2).dim_names |
283 |
% myvars(2).dim_names(4) |
284 |
|
285 |
% mydims |
286 |
% length(mydims) |
287 |
% [ mydims.names ] |
288 |
% [ mydims.lens ] |
289 |
|
290 |
|
291 |
%===== Assemble! ===== |
292 |
|
293 |
|
294 |
if all_ncf(1).exch == 1 |
295 |
|
296 |
% exch "1": |
297 |
|
298 |
% $$$ bi_max = max([all_ncf.bi]); |
299 |
% $$$ bj_max = max([all_ncf.bj]); |
300 |
% $$$ Xmax = bi_max * all_ncf(1).sNx; |
301 |
% $$$ Ymax = bj_max * all_ncf(1).sNy; |
302 |
Xmax = all_ncf(1).Nx; |
303 |
Ymax = all_ncf(1).Ny; |
304 |
% at this point I have to make some assumptions about the domain |
305 |
% decomposition |
306 |
bi_max = Xmax/all_ncf(1).sNx; |
307 |
bj_max = Ymax/all_ncf(1).sNy; |
308 |
itile = 0; |
309 |
for bj=1:bj_max |
310 |
for bi=1:bi_max |
311 |
itile = itile+1; |
312 |
all_ncf(itile).bi=bi; |
313 |
all_ncf(itile).bj=bj; |
314 |
end |
315 |
end |
316 |
|
317 |
horzdim = struct('names',{},'lens',{}); |
318 |
horzdim(1).names = { 'X' }; horzdim(1).lens = { Xmax }; |
319 |
horzdim(2).names = {'Xp1'}; horzdim(2).lens = { Xmax + 1 }; |
320 |
horzdim(3).names = { 'Y' }; horzdim(3).lens = { Ymax }; |
321 |
horzdim(4).names = {'Yp1'}; horzdim(4).lens = { Ymax + 1 }; |
322 |
horzdim(5).names = { 'T' }; horzdim(5).lens = { 0 }; |
323 |
|
324 |
iseq = 0; |
325 |
foutnm = sprintf(fout, iseq); |
326 |
fonc = netcdf(foutnm,'clobber'); % Should append-or-create! |
327 |
|
328 |
for idim = 1:length(mydims) |
329 |
dname = mydims(idim).names{1}; |
330 |
ind = find(strcmp(dname,[horzdim.names])); |
331 |
if length(ind) ~= 0 |
332 |
dlen = horzdim(ind).lens{1}; |
333 |
else |
334 |
dlen = mydims(idim).lens{1}; |
335 |
end |
336 |
comm = sprintf('fonc(''%s'') = %d;',dname,dlen); |
337 |
eval(comm); |
338 |
end |
339 |
|
340 |
for ivar = 1:length(myvars) |
341 |
comm = sprintf('fonc{''%s''} = nc%s( ',myvars(ivar).name,myvars(ivar).type); |
342 |
id = 1; |
343 |
comm = [ comm sprintf('''%s''',myvars(ivar).dim_names{id}) ]; |
344 |
for id = 2:length(myvars(ivar).dim_names) |
345 |
comm = [ comm sprintf(',''%s''',myvars(ivar).dim_names{id}) ]; |
346 |
end |
347 |
comm = [ comm ' );' ]; |
348 |
eval(comm); |
349 |
for iat = 1:length(myvars(ivar).atts.names) |
350 |
comm = sprintf( ... |
351 |
'fonc{''%s''}.%s = nc%s( myvars(ivar).atts.data{iat} );', ... |
352 |
myvars(ivar).name, ... |
353 |
myvars(ivar).atts.names{iat}, ... |
354 |
myvars(ivar).atts.types{iat} ); |
355 |
eval(comm); |
356 |
end |
357 |
end |
358 |
|
359 |
% for itime = 1:Tmax |
360 |
|
361 |
% Here is where we need to check the output file size and start |
362 |
% another file in the sequence, if necessary. |
363 |
|
364 |
for ivar = 1:length(myvars) |
365 |
disp(sprintf(' Copying variable: %s',myvars(ivar).name)) |
366 |
for itile = 1:length(all_ncf) |
367 |
|
368 |
if (myvars(ivar).has_horiz == 1) | (itile == 1) |
369 |
|
370 |
clear nct; |
371 |
nct = all_ncf(itile).nc{1}; |
372 |
ox_off = (all_ncf(itile).bi - 1)*all_ncf(itile).sNx; |
373 |
oy_off = (all_ncf(itile).bj - 1)*all_ncf(itile).sNy; |
374 |
diml_in = ''; |
375 |
diml_out = ''; |
376 |
for jj = 1:length(myvars(ivar).dim_names) |
377 |
doff = 1; |
378 |
if jj > 1 |
379 |
diml_in = sprintf('%s,',diml_in); |
380 |
diml_out = sprintf('%s,',diml_out); |
381 |
end |
382 |
dlen = myvars(ivar).dim_sizes{jj}; |
383 |
diml_in = sprintf('%s%s',diml_in, ':'); |
384 |
fchar = myvars(ivar).dim_names{jj}(1); |
385 |
% disp([' fchar = ' fchar ' ' myvars(ivar).dim_names{jj}]); |
386 |
if strcmp(myvars(ivar).dim_names{jj}(1),'X') == 1 |
387 |
doff = ox_off + doff; |
388 |
dlen = ox_off + dlen; |
389 |
end |
390 |
if strcmp(myvars(ivar).dim_names{jj}(1),'Y') == 1 |
391 |
doff = oy_off + doff; |
392 |
dlen = oy_off + dlen; |
393 |
end |
394 |
diml_out = sprintf('%s%d%s%d',diml_out,doff,':',dlen); |
395 |
end |
396 |
|
397 |
comm = sprintf( ... |
398 |
'fonc{''%s''}(%s) = nct{''%s''}(%s);', ... |
399 |
myvars(ivar).name, diml_out, myvars(ivar).name, diml_in ); |
400 |
% disp([ ' comm: ' comm ]); |
401 |
eval(comm); |
402 |
|
403 |
end |
404 |
|
405 |
end |
406 |
end |
407 |
% end |
408 |
|
409 |
fonc = close(fonc); |
410 |
|
411 |
elseif all_ncf(1).exch == 2 |
412 |
|
413 |
% exch "2": |
414 |
Xmax = 0; |
415 |
Ymax = 0; |
416 |
for ii = 1:length(all_ncf) |
417 |
Xmax = max(Xmax, (all_ncf(ii).exch2_tbasex + all_ncf(ii).sNx)); |
418 |
Ymax = max(Ymax, (all_ncf(ii).exch2_tbasey + all_ncf(ii).sNy)); |
419 |
end |
420 |
|
421 |
horzdim = struct('names',{},'lens',{}); |
422 |
horzdim(1).names = { 'X' }; horzdim(1).lens = { Xmax }; |
423 |
horzdim(2).names = {'Xp1'}; horzdim(2).lens = { Xmax + 1 }; |
424 |
horzdim(3).names = { 'Y' }; horzdim(3).lens = { Ymax }; |
425 |
horzdim(4).names = {'Yp1'}; horzdim(4).lens = { Ymax + 1 }; |
426 |
horzdim(5).names = { 'T' }; horzdim(5).lens = { 0 }; |
427 |
|
428 |
iseq = 0; |
429 |
foutnm = sprintf(fout, iseq); |
430 |
fonc = netcdf(foutnm,'clobber'); % Should append-or-create! |
431 |
|
432 |
for idim = 1:length(mydims) |
433 |
dname = mydims(idim).names{1}; |
434 |
ind = find(strcmp(dname,[horzdim.names])); |
435 |
if length(ind) ~= 0 |
436 |
dlen = horzdim(ind).lens{1}; |
437 |
else |
438 |
dlen = mydims(idim).lens{1}; |
439 |
end |
440 |
comm = sprintf('fonc(''%s'') = %d;',dname,dlen); |
441 |
eval(comm); |
442 |
end |
443 |
|
444 |
for ivar = 1:length(myvars) |
445 |
comm = sprintf('fonc{''%s''} = nc%s( ',myvars(ivar).name,myvars(ivar).type); |
446 |
id = 1; |
447 |
comm = [ comm sprintf('''%s''',myvars(ivar).dim_names{id}) ]; |
448 |
for id = 2:length(myvars(ivar).dim_names) |
449 |
dname = myvars(ivar).dim_names{id}; |
450 |
if (dname(1) == 'Y') & (myvars(ivar).has_horiz == 1) |
451 |
comm = [ comm sprintf(',''%s''','iface') ]; |
452 |
end |
453 |
comm = [ comm sprintf(',''%s''',dname) ]; |
454 |
end |
455 |
comm = [ comm ' );' ]; |
456 |
eval(comm); |
457 |
for iat = 1:length(myvars(ivar).atts.names) |
458 |
comm = sprintf( ... |
459 |
'fonc{''%s''}.%s = nc%s( myvars(ivar).atts.data{iat} );', ... |
460 |
myvars(ivar).name, ... |
461 |
myvars(ivar).atts.names{iat}, ... |
462 |
myvars(ivar).atts.types{iat} ); |
463 |
eval(comm); |
464 |
end |
465 |
end |
466 |
|
467 |
% Here is where we need to check the output file size and start |
468 |
% another file in the sequence, if necessary. |
469 |
|
470 |
for ivar = 1:length(myvars) |
471 |
disp(sprintf(' Copying variable: %s',myvars(ivar).name)) |
472 |
for itile = 1:length(all_ncf) |
473 |
|
474 |
if (myvars(ivar).has_horiz == 1) | (itile == 1) |
475 |
|
476 |
clear nct; |
477 |
nct = all_ncf(itile).nc{1}; |
478 |
ox_off = all_ncf(itile).exch2_tbasex; |
479 |
oy_off = all_ncf(itile).exch2_tbasey; |
480 |
diml_tin = ''; |
481 |
diml_res = ''; |
482 |
diml_in = ''; |
483 |
diml_out = ''; |
484 |
if length(myvars(ivar).dim_names) < 2 |
485 |
comm = sprintf( ... |
486 |
'fonc{''%s''}(%s%d) = nct{''%s''}(:);', ... |
487 |
myvars(ivar).name, '1:', myvars(ivar).dim_sizes{1}, ... |
488 |
myvars(ivar).name ); |
489 |
% disp([ ' ' comm ]); |
490 |
eval(comm); |
491 |
else |
492 |
for jj = 1:length(myvars(ivar).dim_names) |
493 |
doff = 1; |
494 |
if jj > 1 |
495 |
diml_tin = sprintf('%s,',diml_tin); |
496 |
diml_res = sprintf('%s,',diml_res); |
497 |
diml_in = sprintf('%s,',diml_in); |
498 |
diml_out = sprintf('%s,',diml_out); |
499 |
end |
500 |
dnam = myvars(ivar).dim_names{jj}; |
501 |
dlen = myvars(ivar).dim_sizes{jj}; |
502 |
dlenr = dlen; |
503 |
fchar = myvars(ivar).dim_names{jj}(1); |
504 |
% disp([' fchar = ' fchar ' ' myvars(ivar).dim_names{jj}]); |
505 |
if strcmp(dnam(1),'X') == 1 |
506 |
doff = ox_off + doff; |
507 |
dlen = ox_off + dlen; |
508 |
end |
509 |
if strcmp(dnam(1),'Y') == 1 |
510 |
diml_res = sprintf('%s%s',diml_res, '[],'); |
511 |
diml_in = sprintf('%s%s',diml_in, ':,'); |
512 |
diml_out = sprintf('%s%d%s',diml_out,all_ncf(itile).exch2_myFace,','); |
513 |
doff = oy_off + doff; |
514 |
dlen = oy_off + dlen; |
515 |
end |
516 |
diml_tin = sprintf('%s%s',diml_tin, ':'); |
517 |
diml_res = sprintf('%s%d',diml_res, dlenr); |
518 |
diml_in = sprintf('%s%s',diml_in, ':'); |
519 |
diml_out = sprintf('%s%d%s%d',diml_out,doff,':',dlen); |
520 |
end |
521 |
|
522 |
comm = sprintf( ... |
523 |
'tmp = reshape(nct{''%s''}(%s), %s); fonc{''%s''}(%s) = tmp(%s);', ... |
524 |
myvars(ivar).name, diml_tin, diml_res, myvars(ivar).name, ... |
525 |
diml_out, diml_in ); |
526 |
% disp([ ' ' comm ]); |
527 |
eval(comm); |
528 |
end |
529 |
|
530 |
end |
531 |
|
532 |
end |
533 |
end |
534 |
% end |
535 |
|
536 |
fonc = close(fonc); |
537 |
|
538 |
end |
539 |
|