/[MITgcm]/MITgcm/utils/matlab/mnc_assembly.m
ViewVC logotype

Contents of /MITgcm/utils/matlab/mnc_assembly.m

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.5 - (show annotations) (download)
Tue Aug 2 19:43:21 2005 UTC (18 years, 9 months ago) by edhill
Branch: MAIN
CVS Tags: checkpoint58l_post, checkpoint57t_post, checkpoint58e_post, checkpoint57v_post, checkpoint58u_post, checkpoint57s_post, checkpoint58r_post, checkpoint57y_post, checkpoint58n_post, checkpoint58t_post, checkpoint58h_post, checkpoint57y_pre, checkpoint58q_post, checkpoint58j_post, checkpoint57r_post, checkpoint58, checkpoint58f_post, checkpoint57x_post, checkpoint58d_post, checkpoint58c_post, checkpoint57w_post, checkpint57u_post, checkpoint58a_post, checkpoint58i_post, checkpoint57q_post, checkpoint58g_post, checkpoint58o_post, checkpoint57z_post, checkpoint58k_post, checkpoint58v_post, checkpoint58s_post, checkpoint58p_post, checkpoint58b_post, checkpoint58m_post
Changes since 1.4: +14 -0 lines
 o add more to help

1 function [nt,nf] = mnc_assembly(fpat,vars, fout,fsize)
2
3 % Function [nt,nf] = mnc_assembly(fpat,vars, fout,fsize)
4 %
5 % INPUTS
6 % fpat string containing the file pattern
7 % vars structure array of variable names
8 %
9 % fout output file pattern (DEF: "all.%05d.nc")
10 % fsize max output file size (DEF: 2.0e+9 = +/-2GB)
11 %
12 % OUTPUTS
13 % nt number of usable tiles found
14 % nf number of output files written
15 %
16 % This function "assembles" MNC output. It finds all the per-tile
17 % NetCDF files that match the input pattern, does some basic "sanity"
18 % tests to determine whether the files have compatible sizes, and
19 % then assembles all of the requested data (all of the variables)
20 % into one or more "global" NetCDF files. The global files have
21 % the following dimension conventions:
22 %
23 % "exch 1": all values are within a global horizontal grid
24 % and indicies are (X,Y,Z,T)
25 %
26 % "exch 2": all values are within one of up to six "faces"
27 % of a global cube with indicies (Xf,Yf,F,Z,T)
28 %
29 % where "X,Y.Z,T" are global space/time indicies, "Xf,Yf" are local
30 % per-face spatial indicies, and "F" is a face index.
31 %
32 % An example of how to use this script is:
33 %
34 % vars = struct([]);
35 % vars(1).name = 'iter';
36 % vars(2).name = 'U';
37 % vars(3).name = 'Unk';
38 % vars(4).name = 'V';
39 % vars(5).name = 'Temp';
40 % vars(6).name = 'S';
41 % fpat = 'exp0_20041126_0001/state.0000.%06d.nc';
42 % [nt,nf] = mnc_assembly(fpat,vars);
43 %
44 % and the resutlt is written as "all.00000.nc"
45
46
47 %===== Argument checking and defaults =====
48
49 if nargin < 2
50 disp('Error: there must be at least 2 arguments!');
51 return
52 end
53
54 if nargin < 3
55 fout = 'all.%05d.nc';
56 end
57 if nargin < 4
58 fsize = 2.0e+9;
59 end
60
61
62 %===== Find and open all the matching files =====
63
64 nt = 0;
65 nf = 0;
66 all_ncf = struct([]);
67
68 % Find all of the files
69 exch2_msg = 0;
70 tmax = 200;
71 frdone = 0;
72 it = 0;
73 while frdone == 0
74
75 it = it + 1;
76 fnm = sprintf(fpat,it);
77 % disp(fnm);
78
79 % Check that the file exists
80 fid = fopen(fnm, 'r');
81 if fid < 0
82 if it >= tmax
83 frdone = 1;
84 end
85 continue;
86 end
87
88 % Open the NetCDF file
89 fnc = netcdf(fnm, 'nowrite');
90 if length(fnc) == 0
91 continue;
92 end
93
94 % Check for exch1/exch2 grid
95 exch = 1;
96 exch2_myFace = fnc.exch2_myFace(:);
97 if length(exch2_myFace) ~= 0
98 exch = 2;
99 if exch2_msg == 0
100 exch2_msg = 1;
101 disp(' Grid type appears to be: "exch2"');
102 end
103 end
104
105 n = length(all_ncf) + 1;
106 all_ncf(n).name = fnm;
107 all_ncf(n).nc = {fnc};
108 all_ncf(n).exch = exch;
109 all_ncf(n).tile_number = fnc.tile_number(1);
110 all_ncf(n).bi = fnc.bi(1);
111 all_ncf(n).bj = fnc.bj(1);
112 all_ncf(n).sNx = fnc.sNx(1);
113 all_ncf(n).sNy = fnc.sNy(1);
114 all_ncf(n).Nx = fnc.Nx(1);
115 all_ncf(n).Ny = fnc.Ny(1);
116 all_ncf(n).Z = fnc.Z(1);
117
118 if exch == 2
119 all_ncf(n).exch2_myFace = exch2_myFace;
120 all_ncf(n).exch2_tbasex = fnc.exch2_tbasex(1);
121 all_ncf(n).exch2_tbasey = fnc.exch2_tbasex(1);
122 end
123
124 clear fnc
125 end
126
127
128 %===== Do some basic sanity checks =====
129
130 % check for number of files/tiles found
131 if length(all_ncf) == 0
132 disp('Error: no tiles found--no need to do any assembly!');
133 return
134 elseif length(all_ncf) == 1
135 disp('Error: one tile found--no need to do any assembly!');
136 return
137 else
138 disp(sprintf(' Found %d files matching the pattern: "%s"', ...
139 length(all_ncf), fpat ));
140 end
141
142 % check for consistent "exch" version
143 if prod(double([all_ncf.exch] == all_ncf(1).exch)) ~= 1
144 disp('Error: not all the "exch" types of the files match.');
145 return;
146 end
147
148 % check for consistent sNx,sNy
149 if (prod(double([all_ncf.sNx] == all_ncf(1).sNx)) ~= 1) ...
150 | (prod(double([all_ncf.sNy] == all_ncf(1).sNy)) ~= 1)
151 disp('Error: the "sNx,sNy" values for all the tiles are not');
152 disp(' uniform. Future versions of this function will be');
153 disp(' able to handle non-uniform grid sizes but this');
154 disp(' feature is not yet implemented.');
155 return;
156 end
157
158 % check for redundant tiles and "time series" output
159 if length(all_ncf) ~= length(unique([all_ncf.tile_number]))
160 disp('Error: redundant tiles were found. Please check that');
161 disp(' the file pattern does not specify output spanning');
162 disp(' multiple model runs or even multiple time series');
163 disp(' within a single model run. For multi-time-series');
164 disp(' data sets, EACH "LEVEL" IN THE OUTPUT SERIES MUST');
165 disp(' BE ASSEMBLED SEPARATERLY.');
166 return
167 end
168
169
170 %===== Get the dims/vars associations =====
171
172 mydims = struct('names', {}, 'lens', {});
173 myvars = struct([]);
174 clear tncf;
175 for ivar = 1:length(vars)
176 mydim_names = {};
177 mydim_sizes = {};
178 myatt.names = {};
179 myatt.types = {};
180 myatt.data = {};
181 myname = vars(ivar).name;
182 disp([' Looking for variable: ' myname]);
183
184 itile = 1;
185 tncf = all_ncf(itile).nc{1};
186 ncv = tncf{myname};
187 len = length(ncv);
188 if length(ncv) == 0
189 warns = [' Warning: variable "%s" is not defined in "%s"\n' ...
190 ' so it will be ignored.'];
191 disp(sprintf(warns,myname,all_ncf(itile).name));
192 continue
193 end
194 mytype = datatype(ncv);
195 tmpdims = dim(ncv);
196 for inm = 1:length(tmpdims)
197 mydim_names{inm} = name(tmpdims{inm});
198 mydim_sizes{inm} = tmpdims{inm}(:);
199 end
200 for iat = 1:length(att(ncv))
201 aaa = att(ncv);
202 myatt.names(iat) = { name(aaa{iat}) };
203 myatt.types(iat) = { datatype(aaa{iat}) };
204 aab = aaa{iat};
205 myatt.data(iat) = { aab(:) };
206 end
207
208 % confirm: vars have same dim names across all files
209 ierr = 0;
210 for itile = 2:length(all_ncf)
211 tncf = all_ncf(itile).nc{1};
212 ncv = tncf{myname};
213 len = length(ncv);
214 if length(ncv) == 0
215 warns = [' Warning: variable "%s" is not defined in "%s"\n' ...
216 ' so it will be ignored.'];
217 disp(sprintf(warns,myname,all_ncf(itile).name));
218 continue
219 end
220 tmpdims = dim(ncv);
221 for inm = 1:length(tmpdims)
222 if mydim_names{inm} ~= name(tmpdims{inm})
223 warns = ...
224 [' Warning: variable "%s" is not CONSISTENTLY defined.\n' ...
225 ' It has different dimensions in different files so\n' ...
226 ' so it will be ignored.'];
227 disp(sprintf(warns,myname));
228 ierr = 1;
229 break
230 end
231 mydim_sizes{inm} = max([ tmpdims{inm}(:) mydim_sizes{inm} ]);
232 end
233
234 end
235
236 if ierr == 0
237 % check: does the variable have a "horizontal" component
238 has_horiz = 0;
239 horiz_names = { 'X' 'Y' 'Xp1' 'Yp1' };
240 for id = 1:length(mydim_names)
241 if length([intersect(horiz_names,mydim_names{id})]) > 0
242 has_horiz = 1;
243 end
244 end
245 % disp([ ' ' myname ' ' sprintf('%d',has_horiz) ]);
246
247 imy = length(myvars) + 1;
248 myvars(imy).name = myname;
249 myvars(imy).type = mytype;
250 myvars(imy).dim_names = mydim_names;
251 myvars(imy).dim_sizes = mydim_sizes;
252 myvars(imy).atts = myatt;
253 myvars(imy).has_horiz = has_horiz;
254
255 % this is necessary to make it work with Matlab 6.5
256 if isempty([mydims.names])
257 addl = mydim_names;
258 else
259 addl = setdiff(mydim_names,[mydims.names]);
260 end
261 for iaddl = 1:length(addl)
262 np1 = length(mydims) + 1;
263 mydims(np1).names = addl(iaddl);
264 mydims(np1).lens = mydim_sizes(find(strcmp(addl(iaddl),mydim_names)));
265 end
266
267 end
268 end
269
270 % For exch == 2, we need to add a "face" dimension
271 if all_ncf(1).exch == 2
272 np1 = length(mydims) + 1;
273 mydims(np1).names = { 'iface' };
274 mydims(np1).lens = { length(unique([all_ncf.exch2_myFace])) };
275 end
276
277 % myvars.name
278 % myvars.dim_names
279 % myvars.dim_sizes
280 % myvars(2).dim_names
281 % myvars(2).dim_names(4)
282
283 % mydims
284 % length(mydims)
285 % [ mydims.names ]
286 % [ mydims.lens ]
287
288
289 %===== Assemble! =====
290
291
292 if all_ncf(1).exch == 1
293
294 % exch "1":
295
296 % $$$ bi_max = max([all_ncf.bi]);
297 % $$$ bj_max = max([all_ncf.bj]);
298 % $$$ Xmax = bi_max * all_ncf(1).sNx;
299 % $$$ Ymax = bj_max * all_ncf(1).sNy;
300 Xmax = all_ncf(1).Nx;
301 Ymax = all_ncf(1).Ny;
302 % at this point I have to make some assumptions about the domain
303 % decomposition
304 bi_max = Xmax/all_ncf(1).sNx;
305 bj_max = Ymax/all_ncf(1).sNy;
306 itile = 0;
307 for bj=1:bj_max
308 for bi=1:bi_max
309 itile = itile+1;
310 all_ncf(itile).bi=bi;
311 all_ncf(itile).bj=bj;
312 end
313 end
314
315 horzdim = struct('names',{},'lens',{});
316 horzdim(1).names = { 'X' }; horzdim(1).lens = { Xmax };
317 horzdim(2).names = {'Xp1'}; horzdim(2).lens = { Xmax + 1 };
318 horzdim(3).names = { 'Y' }; horzdim(3).lens = { Ymax };
319 horzdim(4).names = {'Yp1'}; horzdim(4).lens = { Ymax + 1 };
320 horzdim(5).names = { 'T' }; horzdim(5).lens = { 0 };
321
322 iseq = 0;
323 foutnm = sprintf(fout, iseq);
324 fonc = netcdf(foutnm,'clobber'); % Should append-or-create!
325
326 for idim = 1:length(mydims)
327 dname = mydims(idim).names{1};
328 ind = find(strcmp(dname,[horzdim.names]));
329 if length(ind) ~= 0
330 dlen = horzdim(ind).lens{1};
331 else
332 dlen = mydims(idim).lens{1};
333 end
334 comm = sprintf('fonc(''%s'') = %d;',dname,dlen);
335 eval(comm);
336 end
337
338 for ivar = 1:length(myvars)
339 comm = sprintf('fonc{''%s''} = nc%s( ',myvars(ivar).name,myvars(ivar).type);
340 id = 1;
341 comm = [ comm sprintf('''%s''',myvars(ivar).dim_names{id}) ];
342 for id = 2:length(myvars(ivar).dim_names)
343 comm = [ comm sprintf(',''%s''',myvars(ivar).dim_names{id}) ];
344 end
345 comm = [ comm ' );' ];
346 eval(comm);
347 for iat = 1:length(myvars(ivar).atts.names)
348 comm = sprintf( ...
349 'fonc{''%s''}.%s = nc%s( myvars(ivar).atts.data{iat} );', ...
350 myvars(ivar).name, ...
351 myvars(ivar).atts.names{iat}, ...
352 myvars(ivar).atts.types{iat} );
353 eval(comm);
354 end
355 end
356
357 % for itime = 1:Tmax
358
359 % Here is where we need to check the output file size and start
360 % another file in the sequence, if necessary.
361
362 for ivar = 1:length(myvars)
363 disp(sprintf(' Copying variable: %s',myvars(ivar).name))
364 for itile = 1:length(all_ncf)
365
366 if (myvars(ivar).has_horiz == 1) | (itile == 1)
367
368 clear nct;
369 nct = all_ncf(itile).nc{1};
370 ox_off = (all_ncf(itile).bi - 1)*all_ncf(itile).sNx;
371 oy_off = (all_ncf(itile).bj - 1)*all_ncf(itile).sNy;
372 diml_in = '';
373 diml_out = '';
374 for jj = 1:length(myvars(ivar).dim_names)
375 doff = 1;
376 if jj > 1
377 diml_in = sprintf('%s,',diml_in);
378 diml_out = sprintf('%s,',diml_out);
379 end
380 dlen = myvars(ivar).dim_sizes{jj};
381 diml_in = sprintf('%s%s',diml_in, ':');
382 fchar = myvars(ivar).dim_names{jj}(1);
383 % disp([' fchar = ' fchar ' ' myvars(ivar).dim_names{jj}]);
384 if strcmp(myvars(ivar).dim_names{jj}(1),'X') == 1
385 doff = ox_off + doff;
386 dlen = ox_off + dlen;
387 end
388 if strcmp(myvars(ivar).dim_names{jj}(1),'Y') == 1
389 doff = oy_off + doff;
390 dlen = oy_off + dlen;
391 end
392 diml_out = sprintf('%s%d%s%d',diml_out,doff,':',dlen);
393 end
394
395 comm = sprintf( ...
396 'fonc{''%s''}(%s) = nct{''%s''}(%s);', ...
397 myvars(ivar).name, diml_out, myvars(ivar).name, diml_in );
398 % disp([ ' comm: ' comm ]);
399 eval(comm);
400
401 end
402
403 end
404 end
405 % end
406
407 fonc = close(fonc);
408
409 elseif all_ncf(1).exch == 2
410
411 % exch "2":
412 Xmax = 0;
413 Ymax = 0;
414 for ii = 1:length(all_ncf)
415 Xmax = max(Xmax, (all_ncf(ii).exch2_tbasex + all_ncf(ii).sNx));
416 Ymax = max(Ymax, (all_ncf(ii).exch2_tbasey + all_ncf(ii).sNy));
417 end
418
419 horzdim = struct('names',{},'lens',{});
420 horzdim(1).names = { 'X' }; horzdim(1).lens = { Xmax };
421 horzdim(2).names = {'Xp1'}; horzdim(2).lens = { Xmax + 1 };
422 horzdim(3).names = { 'Y' }; horzdim(3).lens = { Ymax };
423 horzdim(4).names = {'Yp1'}; horzdim(4).lens = { Ymax + 1 };
424 horzdim(5).names = { 'T' }; horzdim(5).lens = { 0 };
425
426 iseq = 0;
427 foutnm = sprintf(fout, iseq);
428 fonc = netcdf(foutnm,'clobber'); % Should append-or-create!
429
430 for idim = 1:length(mydims)
431 dname = mydims(idim).names{1};
432 ind = find(strcmp(dname,[horzdim.names]));
433 if length(ind) ~= 0
434 dlen = horzdim(ind).lens{1};
435 else
436 dlen = mydims(idim).lens{1};
437 end
438 comm = sprintf('fonc(''%s'') = %d;',dname,dlen);
439 eval(comm);
440 end
441
442 for ivar = 1:length(myvars)
443 comm = sprintf('fonc{''%s''} = nc%s( ',myvars(ivar).name,myvars(ivar).type);
444 id = 1;
445 comm = [ comm sprintf('''%s''',myvars(ivar).dim_names{id}) ];
446 for id = 2:length(myvars(ivar).dim_names)
447 dname = myvars(ivar).dim_names{id};
448 if (dname(1) == 'Y') & (myvars(ivar).has_horiz == 1)
449 comm = [ comm sprintf(',''%s''','iface') ];
450 end
451 comm = [ comm sprintf(',''%s''',dname) ];
452 end
453 comm = [ comm ' );' ];
454 eval(comm);
455 for iat = 1:length(myvars(ivar).atts.names)
456 comm = sprintf( ...
457 'fonc{''%s''}.%s = nc%s( myvars(ivar).atts.data{iat} );', ...
458 myvars(ivar).name, ...
459 myvars(ivar).atts.names{iat}, ...
460 myvars(ivar).atts.types{iat} );
461 eval(comm);
462 end
463 end
464
465 % Here is where we need to check the output file size and start
466 % another file in the sequence, if necessary.
467
468 for ivar = 1:length(myvars)
469 disp(sprintf(' Copying variable: %s',myvars(ivar).name))
470 for itile = 1:length(all_ncf)
471
472 if (myvars(ivar).has_horiz == 1) | (itile == 1)
473
474 clear nct;
475 nct = all_ncf(itile).nc{1};
476 ox_off = all_ncf(itile).exch2_tbasex;
477 oy_off = all_ncf(itile).exch2_tbasey;
478 diml_tin = '';
479 diml_res = '';
480 diml_in = '';
481 diml_out = '';
482 if length(myvars(ivar).dim_names) < 2
483 comm = sprintf( ...
484 'fonc{''%s''}(%s%d) = nct{''%s''}(:);', ...
485 myvars(ivar).name, '1:', myvars(ivar).dim_sizes{1}, ...
486 myvars(ivar).name );
487 % disp([ ' ' comm ]);
488 eval(comm);
489 else
490 for jj = 1:length(myvars(ivar).dim_names)
491 doff = 1;
492 if jj > 1
493 diml_tin = sprintf('%s,',diml_tin);
494 diml_res = sprintf('%s,',diml_res);
495 diml_in = sprintf('%s,',diml_in);
496 diml_out = sprintf('%s,',diml_out);
497 end
498 dnam = myvars(ivar).dim_names{jj};
499 dlen = myvars(ivar).dim_sizes{jj};
500 dlenr = dlen;
501 fchar = myvars(ivar).dim_names{jj}(1);
502 % disp([' fchar = ' fchar ' ' myvars(ivar).dim_names{jj}]);
503 if strcmp(dnam(1),'X') == 1
504 doff = ox_off + doff;
505 dlen = ox_off + dlen;
506 end
507 if strcmp(dnam(1),'Y') == 1
508 diml_res = sprintf('%s%s',diml_res, '[],');
509 diml_in = sprintf('%s%s',diml_in, ':,');
510 diml_out = sprintf('%s%d%s',diml_out,all_ncf(itile).exch2_myFace,',');
511 doff = oy_off + doff;
512 dlen = oy_off + dlen;
513 end
514 diml_tin = sprintf('%s%s',diml_tin, ':');
515 diml_res = sprintf('%s%d',diml_res, dlenr);
516 diml_in = sprintf('%s%s',diml_in, ':');
517 diml_out = sprintf('%s%d%s%d',diml_out,doff,':',dlen);
518 end
519
520 comm = sprintf( ...
521 'tmp = reshape(nct{''%s''}(%s), %s); fonc{''%s''}(%s) = tmp(%s);', ...
522 myvars(ivar).name, diml_tin, diml_res, myvars(ivar).name, ...
523 diml_out, diml_in );
524 % disp([ ' ' comm ]);
525 eval(comm);
526 end
527
528 end
529
530 end
531 end
532 % end
533
534 fonc = close(fonc);
535
536 end
537

  ViewVC Help
Powered by ViewVC 1.1.22