/[MITgcm]/MITgcm/utils/matlab/mnc_assembly.m
ViewVC logotype

Contents of /MITgcm/utils/matlab/mnc_assembly.m

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.6 - (show annotations) (download)
Sat Feb 17 23:49:43 2007 UTC (14 years, 4 months ago) by jmc
Branch: MAIN
CVS Tags: checkpoint64y, checkpoint64x, checkpoint64z, checkpoint64q, checkpoint64p, checkpoint64s, checkpoint64r, checkpoint64u, checkpoint64t, checkpoint64w, checkpoint64v, checkpoint64i, checkpoint64h, checkpoint64k, checkpoint64j, checkpoint64m, checkpoint64l, checkpoint64o, checkpoint64n, checkpoint64a, checkpoint64c, checkpoint64b, checkpoint64e, checkpoint64d, checkpoint64g, checkpoint64f, checkpoint58w_post, checkpoint63p, checkpoint63q, checkpoint63r, checkpoint63s, checkpoint63l, checkpoint63m, checkpoint63n, checkpoint63o, checkpoint63h, checkpoint63i, checkpoint63j, checkpoint63k, checkpoint63d, checkpoint63e, checkpoint63f, checkpoint63g, checkpoint63a, checkpoint63b, checkpoint63c, checkpoint64, checkpoint65, checkpoint60, checkpoint61, checkpoint62, checkpoint63, checkpoint66g, checkpoint66f, checkpoint66e, checkpoint66d, checkpoint66c, checkpoint66b, checkpoint66a, checkpoint66o, checkpoint66n, checkpoint66m, checkpoint66l, checkpoint66k, checkpoint66j, checkpoint66i, checkpoint66h, checkpoint58x_post, checkpoint65z, checkpoint65x, checkpoint65y, checkpoint65r, checkpoint65s, checkpoint65p, checkpoint65q, checkpoint65v, checkpoint65w, checkpoint65t, checkpoint65u, checkpoint65j, checkpoint65k, checkpoint65h, checkpoint65i, checkpoint65n, checkpoint65o, checkpoint65l, checkpoint65m, checkpoint65b, checkpoint65c, checkpoint65a, checkpoint65f, checkpoint65g, checkpoint65d, checkpoint65e, checkpoint59q, checkpoint59p, checkpoint59r, checkpoint59e, checkpoint59d, checkpoint59g, checkpoint59f, checkpoint59a, checkpoint59c, checkpoint59b, checkpoint59m, checkpoint59l, checkpoint59o, checkpoint59n, checkpoint59i, checkpoint59h, checkpoint59k, checkpoint59j, checkpoint59, checkpoint62c, checkpoint62b, checkpoint62a, checkpoint62g, checkpoint62f, checkpoint62e, checkpoint62d, checkpoint62k, checkpoint62j, checkpoint62i, checkpoint62h, checkpoint62o, checkpoint62n, checkpoint62m, checkpoint62l, checkpoint62s, checkpoint62r, checkpoint62q, checkpoint62p, checkpoint62w, checkpoint62v, checkpoint62u, checkpoint62t, checkpoint62z, checkpoint62y, checkpoint62x, checkpoint58y_post, checkpoint61f, checkpoint61g, checkpoint61d, checkpoint61e, checkpoint61b, checkpoint61c, checkpoint61a, checkpoint61n, checkpoint61o, checkpoint61l, checkpoint61m, checkpoint61j, checkpoint61k, checkpoint61h, checkpoint61i, checkpoint61v, checkpoint61w, checkpoint61t, checkpoint61u, checkpoint61r, checkpoint61s, checkpoint61p, checkpoint61q, checkpoint61z, checkpoint61x, checkpoint61y, HEAD
Changes since 1.5: +2 -0 lines
add $Header:  $ and $Name:  & (for CVS)

1 function [nt,nf] = mnc_assembly(fpat,vars, fout,fsize)
2
3 % Function [nt,nf] = mnc_assembly(fpat,vars, fout,fsize)
4 %
5 % INPUTS
6 % fpat string containing the file pattern
7 % vars structure array of variable names
8 %
9 % fout output file pattern (DEF: "all.%05d.nc")
10 % fsize max output file size (DEF: 2.0e+9 = +/-2GB)
11 %
12 % OUTPUTS
13 % nt number of usable tiles found
14 % nf number of output files written
15 %
16 % This function "assembles" MNC output. It finds all the per-tile
17 % NetCDF files that match the input pattern, does some basic "sanity"
18 % tests to determine whether the files have compatible sizes, and
19 % then assembles all of the requested data (all of the variables)
20 % into one or more "global" NetCDF files. The global files have
21 % the following dimension conventions:
22 %
23 % "exch 1": all values are within a global horizontal grid
24 % and indicies are (X,Y,Z,T)
25 %
26 % "exch 2": all values are within one of up to six "faces"
27 % of a global cube with indicies (Xf,Yf,F,Z,T)
28 %
29 % where "X,Y.Z,T" are global space/time indicies, "Xf,Yf" are local
30 % per-face spatial indicies, and "F" is a face index.
31 %
32 % An example of how to use this script is:
33 %
34 % vars = struct([]);
35 % vars(1).name = 'iter';
36 % vars(2).name = 'U';
37 % vars(3).name = 'Unk';
38 % vars(4).name = 'V';
39 % vars(5).name = 'Temp';
40 % vars(6).name = 'S';
41 % fpat = 'exp0_20041126_0001/state.0000.%06d.nc';
42 % [nt,nf] = mnc_assembly(fpat,vars);
43 %
44 % and the resutlt is written as "all.00000.nc"
45
46 % $Header: $
47 % $Name: $
48
49 %===== Argument checking and defaults =====
50
51 if nargin < 2
52 disp('Error: there must be at least 2 arguments!');
53 return
54 end
55
56 if nargin < 3
57 fout = 'all.%05d.nc';
58 end
59 if nargin < 4
60 fsize = 2.0e+9;
61 end
62
63
64 %===== Find and open all the matching files =====
65
66 nt = 0;
67 nf = 0;
68 all_ncf = struct([]);
69
70 % Find all of the files
71 exch2_msg = 0;
72 tmax = 200;
73 frdone = 0;
74 it = 0;
75 while frdone == 0
76
77 it = it + 1;
78 fnm = sprintf(fpat,it);
79 % disp(fnm);
80
81 % Check that the file exists
82 fid = fopen(fnm, 'r');
83 if fid < 0
84 if it >= tmax
85 frdone = 1;
86 end
87 continue;
88 end
89
90 % Open the NetCDF file
91 fnc = netcdf(fnm, 'nowrite');
92 if length(fnc) == 0
93 continue;
94 end
95
96 % Check for exch1/exch2 grid
97 exch = 1;
98 exch2_myFace = fnc.exch2_myFace(:);
99 if length(exch2_myFace) ~= 0
100 exch = 2;
101 if exch2_msg == 0
102 exch2_msg = 1;
103 disp(' Grid type appears to be: "exch2"');
104 end
105 end
106
107 n = length(all_ncf) + 1;
108 all_ncf(n).name = fnm;
109 all_ncf(n).nc = {fnc};
110 all_ncf(n).exch = exch;
111 all_ncf(n).tile_number = fnc.tile_number(1);
112 all_ncf(n).bi = fnc.bi(1);
113 all_ncf(n).bj = fnc.bj(1);
114 all_ncf(n).sNx = fnc.sNx(1);
115 all_ncf(n).sNy = fnc.sNy(1);
116 all_ncf(n).Nx = fnc.Nx(1);
117 all_ncf(n).Ny = fnc.Ny(1);
118 all_ncf(n).Z = fnc.Z(1);
119
120 if exch == 2
121 all_ncf(n).exch2_myFace = exch2_myFace;
122 all_ncf(n).exch2_tbasex = fnc.exch2_tbasex(1);
123 all_ncf(n).exch2_tbasey = fnc.exch2_tbasex(1);
124 end
125
126 clear fnc
127 end
128
129
130 %===== Do some basic sanity checks =====
131
132 % check for number of files/tiles found
133 if length(all_ncf) == 0
134 disp('Error: no tiles found--no need to do any assembly!');
135 return
136 elseif length(all_ncf) == 1
137 disp('Error: one tile found--no need to do any assembly!');
138 return
139 else
140 disp(sprintf(' Found %d files matching the pattern: "%s"', ...
141 length(all_ncf), fpat ));
142 end
143
144 % check for consistent "exch" version
145 if prod(double([all_ncf.exch] == all_ncf(1).exch)) ~= 1
146 disp('Error: not all the "exch" types of the files match.');
147 return;
148 end
149
150 % check for consistent sNx,sNy
151 if (prod(double([all_ncf.sNx] == all_ncf(1).sNx)) ~= 1) ...
152 | (prod(double([all_ncf.sNy] == all_ncf(1).sNy)) ~= 1)
153 disp('Error: the "sNx,sNy" values for all the tiles are not');
154 disp(' uniform. Future versions of this function will be');
155 disp(' able to handle non-uniform grid sizes but this');
156 disp(' feature is not yet implemented.');
157 return;
158 end
159
160 % check for redundant tiles and "time series" output
161 if length(all_ncf) ~= length(unique([all_ncf.tile_number]))
162 disp('Error: redundant tiles were found. Please check that');
163 disp(' the file pattern does not specify output spanning');
164 disp(' multiple model runs or even multiple time series');
165 disp(' within a single model run. For multi-time-series');
166 disp(' data sets, EACH "LEVEL" IN THE OUTPUT SERIES MUST');
167 disp(' BE ASSEMBLED SEPARATERLY.');
168 return
169 end
170
171
172 %===== Get the dims/vars associations =====
173
174 mydims = struct('names', {}, 'lens', {});
175 myvars = struct([]);
176 clear tncf;
177 for ivar = 1:length(vars)
178 mydim_names = {};
179 mydim_sizes = {};
180 myatt.names = {};
181 myatt.types = {};
182 myatt.data = {};
183 myname = vars(ivar).name;
184 disp([' Looking for variable: ' myname]);
185
186 itile = 1;
187 tncf = all_ncf(itile).nc{1};
188 ncv = tncf{myname};
189 len = length(ncv);
190 if length(ncv) == 0
191 warns = [' Warning: variable "%s" is not defined in "%s"\n' ...
192 ' so it will be ignored.'];
193 disp(sprintf(warns,myname,all_ncf(itile).name));
194 continue
195 end
196 mytype = datatype(ncv);
197 tmpdims = dim(ncv);
198 for inm = 1:length(tmpdims)
199 mydim_names{inm} = name(tmpdims{inm});
200 mydim_sizes{inm} = tmpdims{inm}(:);
201 end
202 for iat = 1:length(att(ncv))
203 aaa = att(ncv);
204 myatt.names(iat) = { name(aaa{iat}) };
205 myatt.types(iat) = { datatype(aaa{iat}) };
206 aab = aaa{iat};
207 myatt.data(iat) = { aab(:) };
208 end
209
210 % confirm: vars have same dim names across all files
211 ierr = 0;
212 for itile = 2:length(all_ncf)
213 tncf = all_ncf(itile).nc{1};
214 ncv = tncf{myname};
215 len = length(ncv);
216 if length(ncv) == 0
217 warns = [' Warning: variable "%s" is not defined in "%s"\n' ...
218 ' so it will be ignored.'];
219 disp(sprintf(warns,myname,all_ncf(itile).name));
220 continue
221 end
222 tmpdims = dim(ncv);
223 for inm = 1:length(tmpdims)
224 if mydim_names{inm} ~= name(tmpdims{inm})
225 warns = ...
226 [' Warning: variable "%s" is not CONSISTENTLY defined.\n' ...
227 ' It has different dimensions in different files so\n' ...
228 ' so it will be ignored.'];
229 disp(sprintf(warns,myname));
230 ierr = 1;
231 break
232 end
233 mydim_sizes{inm} = max([ tmpdims{inm}(:) mydim_sizes{inm} ]);
234 end
235
236 end
237
238 if ierr == 0
239 % check: does the variable have a "horizontal" component
240 has_horiz = 0;
241 horiz_names = { 'X' 'Y' 'Xp1' 'Yp1' };
242 for id = 1:length(mydim_names)
243 if length([intersect(horiz_names,mydim_names{id})]) > 0
244 has_horiz = 1;
245 end
246 end
247 % disp([ ' ' myname ' ' sprintf('%d',has_horiz) ]);
248
249 imy = length(myvars) + 1;
250 myvars(imy).name = myname;
251 myvars(imy).type = mytype;
252 myvars(imy).dim_names = mydim_names;
253 myvars(imy).dim_sizes = mydim_sizes;
254 myvars(imy).atts = myatt;
255 myvars(imy).has_horiz = has_horiz;
256
257 % this is necessary to make it work with Matlab 6.5
258 if isempty([mydims.names])
259 addl = mydim_names;
260 else
261 addl = setdiff(mydim_names,[mydims.names]);
262 end
263 for iaddl = 1:length(addl)
264 np1 = length(mydims) + 1;
265 mydims(np1).names = addl(iaddl);
266 mydims(np1).lens = mydim_sizes(find(strcmp(addl(iaddl),mydim_names)));
267 end
268
269 end
270 end
271
272 % For exch == 2, we need to add a "face" dimension
273 if all_ncf(1).exch == 2
274 np1 = length(mydims) + 1;
275 mydims(np1).names = { 'iface' };
276 mydims(np1).lens = { length(unique([all_ncf.exch2_myFace])) };
277 end
278
279 % myvars.name
280 % myvars.dim_names
281 % myvars.dim_sizes
282 % myvars(2).dim_names
283 % myvars(2).dim_names(4)
284
285 % mydims
286 % length(mydims)
287 % [ mydims.names ]
288 % [ mydims.lens ]
289
290
291 %===== Assemble! =====
292
293
294 if all_ncf(1).exch == 1
295
296 % exch "1":
297
298 % $$$ bi_max = max([all_ncf.bi]);
299 % $$$ bj_max = max([all_ncf.bj]);
300 % $$$ Xmax = bi_max * all_ncf(1).sNx;
301 % $$$ Ymax = bj_max * all_ncf(1).sNy;
302 Xmax = all_ncf(1).Nx;
303 Ymax = all_ncf(1).Ny;
304 % at this point I have to make some assumptions about the domain
305 % decomposition
306 bi_max = Xmax/all_ncf(1).sNx;
307 bj_max = Ymax/all_ncf(1).sNy;
308 itile = 0;
309 for bj=1:bj_max
310 for bi=1:bi_max
311 itile = itile+1;
312 all_ncf(itile).bi=bi;
313 all_ncf(itile).bj=bj;
314 end
315 end
316
317 horzdim = struct('names',{},'lens',{});
318 horzdim(1).names = { 'X' }; horzdim(1).lens = { Xmax };
319 horzdim(2).names = {'Xp1'}; horzdim(2).lens = { Xmax + 1 };
320 horzdim(3).names = { 'Y' }; horzdim(3).lens = { Ymax };
321 horzdim(4).names = {'Yp1'}; horzdim(4).lens = { Ymax + 1 };
322 horzdim(5).names = { 'T' }; horzdim(5).lens = { 0 };
323
324 iseq = 0;
325 foutnm = sprintf(fout, iseq);
326 fonc = netcdf(foutnm,'clobber'); % Should append-or-create!
327
328 for idim = 1:length(mydims)
329 dname = mydims(idim).names{1};
330 ind = find(strcmp(dname,[horzdim.names]));
331 if length(ind) ~= 0
332 dlen = horzdim(ind).lens{1};
333 else
334 dlen = mydims(idim).lens{1};
335 end
336 comm = sprintf('fonc(''%s'') = %d;',dname,dlen);
337 eval(comm);
338 end
339
340 for ivar = 1:length(myvars)
341 comm = sprintf('fonc{''%s''} = nc%s( ',myvars(ivar).name,myvars(ivar).type);
342 id = 1;
343 comm = [ comm sprintf('''%s''',myvars(ivar).dim_names{id}) ];
344 for id = 2:length(myvars(ivar).dim_names)
345 comm = [ comm sprintf(',''%s''',myvars(ivar).dim_names{id}) ];
346 end
347 comm = [ comm ' );' ];
348 eval(comm);
349 for iat = 1:length(myvars(ivar).atts.names)
350 comm = sprintf( ...
351 'fonc{''%s''}.%s = nc%s( myvars(ivar).atts.data{iat} );', ...
352 myvars(ivar).name, ...
353 myvars(ivar).atts.names{iat}, ...
354 myvars(ivar).atts.types{iat} );
355 eval(comm);
356 end
357 end
358
359 % for itime = 1:Tmax
360
361 % Here is where we need to check the output file size and start
362 % another file in the sequence, if necessary.
363
364 for ivar = 1:length(myvars)
365 disp(sprintf(' Copying variable: %s',myvars(ivar).name))
366 for itile = 1:length(all_ncf)
367
368 if (myvars(ivar).has_horiz == 1) | (itile == 1)
369
370 clear nct;
371 nct = all_ncf(itile).nc{1};
372 ox_off = (all_ncf(itile).bi - 1)*all_ncf(itile).sNx;
373 oy_off = (all_ncf(itile).bj - 1)*all_ncf(itile).sNy;
374 diml_in = '';
375 diml_out = '';
376 for jj = 1:length(myvars(ivar).dim_names)
377 doff = 1;
378 if jj > 1
379 diml_in = sprintf('%s,',diml_in);
380 diml_out = sprintf('%s,',diml_out);
381 end
382 dlen = myvars(ivar).dim_sizes{jj};
383 diml_in = sprintf('%s%s',diml_in, ':');
384 fchar = myvars(ivar).dim_names{jj}(1);
385 % disp([' fchar = ' fchar ' ' myvars(ivar).dim_names{jj}]);
386 if strcmp(myvars(ivar).dim_names{jj}(1),'X') == 1
387 doff = ox_off + doff;
388 dlen = ox_off + dlen;
389 end
390 if strcmp(myvars(ivar).dim_names{jj}(1),'Y') == 1
391 doff = oy_off + doff;
392 dlen = oy_off + dlen;
393 end
394 diml_out = sprintf('%s%d%s%d',diml_out,doff,':',dlen);
395 end
396
397 comm = sprintf( ...
398 'fonc{''%s''}(%s) = nct{''%s''}(%s);', ...
399 myvars(ivar).name, diml_out, myvars(ivar).name, diml_in );
400 % disp([ ' comm: ' comm ]);
401 eval(comm);
402
403 end
404
405 end
406 end
407 % end
408
409 fonc = close(fonc);
410
411 elseif all_ncf(1).exch == 2
412
413 % exch "2":
414 Xmax = 0;
415 Ymax = 0;
416 for ii = 1:length(all_ncf)
417 Xmax = max(Xmax, (all_ncf(ii).exch2_tbasex + all_ncf(ii).sNx));
418 Ymax = max(Ymax, (all_ncf(ii).exch2_tbasey + all_ncf(ii).sNy));
419 end
420
421 horzdim = struct('names',{},'lens',{});
422 horzdim(1).names = { 'X' }; horzdim(1).lens = { Xmax };
423 horzdim(2).names = {'Xp1'}; horzdim(2).lens = { Xmax + 1 };
424 horzdim(3).names = { 'Y' }; horzdim(3).lens = { Ymax };
425 horzdim(4).names = {'Yp1'}; horzdim(4).lens = { Ymax + 1 };
426 horzdim(5).names = { 'T' }; horzdim(5).lens = { 0 };
427
428 iseq = 0;
429 foutnm = sprintf(fout, iseq);
430 fonc = netcdf(foutnm,'clobber'); % Should append-or-create!
431
432 for idim = 1:length(mydims)
433 dname = mydims(idim).names{1};
434 ind = find(strcmp(dname,[horzdim.names]));
435 if length(ind) ~= 0
436 dlen = horzdim(ind).lens{1};
437 else
438 dlen = mydims(idim).lens{1};
439 end
440 comm = sprintf('fonc(''%s'') = %d;',dname,dlen);
441 eval(comm);
442 end
443
444 for ivar = 1:length(myvars)
445 comm = sprintf('fonc{''%s''} = nc%s( ',myvars(ivar).name,myvars(ivar).type);
446 id = 1;
447 comm = [ comm sprintf('''%s''',myvars(ivar).dim_names{id}) ];
448 for id = 2:length(myvars(ivar).dim_names)
449 dname = myvars(ivar).dim_names{id};
450 if (dname(1) == 'Y') & (myvars(ivar).has_horiz == 1)
451 comm = [ comm sprintf(',''%s''','iface') ];
452 end
453 comm = [ comm sprintf(',''%s''',dname) ];
454 end
455 comm = [ comm ' );' ];
456 eval(comm);
457 for iat = 1:length(myvars(ivar).atts.names)
458 comm = sprintf( ...
459 'fonc{''%s''}.%s = nc%s( myvars(ivar).atts.data{iat} );', ...
460 myvars(ivar).name, ...
461 myvars(ivar).atts.names{iat}, ...
462 myvars(ivar).atts.types{iat} );
463 eval(comm);
464 end
465 end
466
467 % Here is where we need to check the output file size and start
468 % another file in the sequence, if necessary.
469
470 for ivar = 1:length(myvars)
471 disp(sprintf(' Copying variable: %s',myvars(ivar).name))
472 for itile = 1:length(all_ncf)
473
474 if (myvars(ivar).has_horiz == 1) | (itile == 1)
475
476 clear nct;
477 nct = all_ncf(itile).nc{1};
478 ox_off = all_ncf(itile).exch2_tbasex;
479 oy_off = all_ncf(itile).exch2_tbasey;
480 diml_tin = '';
481 diml_res = '';
482 diml_in = '';
483 diml_out = '';
484 if length(myvars(ivar).dim_names) < 2
485 comm = sprintf( ...
486 'fonc{''%s''}(%s%d) = nct{''%s''}(:);', ...
487 myvars(ivar).name, '1:', myvars(ivar).dim_sizes{1}, ...
488 myvars(ivar).name );
489 % disp([ ' ' comm ]);
490 eval(comm);
491 else
492 for jj = 1:length(myvars(ivar).dim_names)
493 doff = 1;
494 if jj > 1
495 diml_tin = sprintf('%s,',diml_tin);
496 diml_res = sprintf('%s,',diml_res);
497 diml_in = sprintf('%s,',diml_in);
498 diml_out = sprintf('%s,',diml_out);
499 end
500 dnam = myvars(ivar).dim_names{jj};
501 dlen = myvars(ivar).dim_sizes{jj};
502 dlenr = dlen;
503 fchar = myvars(ivar).dim_names{jj}(1);
504 % disp([' fchar = ' fchar ' ' myvars(ivar).dim_names{jj}]);
505 if strcmp(dnam(1),'X') == 1
506 doff = ox_off + doff;
507 dlen = ox_off + dlen;
508 end
509 if strcmp(dnam(1),'Y') == 1
510 diml_res = sprintf('%s%s',diml_res, '[],');
511 diml_in = sprintf('%s%s',diml_in, ':,');
512 diml_out = sprintf('%s%d%s',diml_out,all_ncf(itile).exch2_myFace,',');
513 doff = oy_off + doff;
514 dlen = oy_off + dlen;
515 end
516 diml_tin = sprintf('%s%s',diml_tin, ':');
517 diml_res = sprintf('%s%d',diml_res, dlenr);
518 diml_in = sprintf('%s%s',diml_in, ':');
519 diml_out = sprintf('%s%d%s%d',diml_out,doff,':',dlen);
520 end
521
522 comm = sprintf( ...
523 'tmp = reshape(nct{''%s''}(%s), %s); fonc{''%s''}(%s) = tmp(%s);', ...
524 myvars(ivar).name, diml_tin, diml_res, myvars(ivar).name, ...
525 diml_out, diml_in );
526 % disp([ ' ' comm ]);
527 eval(comm);
528 end
529
530 end
531
532 end
533 end
534 % end
535
536 fonc = close(fonc);
537
538 end
539

  ViewVC Help
Powered by ViewVC 1.1.22