/[MITgcm]/MITgcm/utils/matlab/mnc_assembly.m
ViewVC logotype

Contents of /MITgcm/utils/matlab/mnc_assembly.m

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.4 - (show annotations) (download)
Tue Feb 8 08:45:58 2005 UTC (19 years, 5 months ago) by mlosch
Branch: MAIN
CVS Tags: checkpoint57o_post, checkpoint57m_post, checkpoint57k_post, checkpoint57d_post, checkpoint57g_post, checkpoint57i_post, checkpoint57e_post, checkpoint57g_pre, checkpoint57f_pre, eckpoint57e_pre, checkpoint57h_done, checkpoint57n_post, checkpoint57p_post, checkpoint57f_post, checkpoint57j_post, checkpoint57h_pre, checkpoint57l_post, checkpoint57h_post
Changes since 1.3: +6 -1 lines
o include fix for Matlab 6.1
o mnc_assembly should now work with Matlab 6.1, 6.5, and 7
o for exch type 1, it will also handle both multiple processors and
  tiles
o to be done: do the above for exch type 2 (cubed sphere)

1 function [nt,nf] = mnc_assembly(fpat,vars, fout,fsize)
2
3 % Function [nt,nf] = mnc_assembly(fpat,vars, fout,fsize)
4 %
5 % INPUTS
6 % fpat string containing the file pattern
7 % vars structure array of variable names
8 %
9 % fout output file pattern (DEF: "all.%05d.nc")
10 % fsize max output file size (DEF: 2.0e+9 = +/-2GB)
11 %
12 % OUTPUTS
13 % nt number of usable tiles found
14 % nf number of output files written
15 %
16 % This function "assembles" MNC output. It finds all the per-tile
17 % NetCDF files that match the input pattern, does some basic "sanity"
18 % tests to determine whether the files have compatible sizes, and
19 % then assembles all of the requested data (all of the variables)
20 % into one or more "global" NetCDF files. The global files have
21 % the following dimension conventions:
22 %
23 % "exch 1": all values are within a global horizontal grid
24 % and indicies are (X,Y,Z,T)
25 %
26 % "exch 2": all values are within one of up to six "faces"
27 % of a global cube with indicies (Xf,Yf,F,Z,T)
28 %
29 % where "X,Y.Z,T" are global space/time indicies, "Xf,Yf" are local
30 % per-face spatial indicies, and "F" is a face index.
31
32
33 %===== Argument checking and defaults =====
34
35 if nargin < 2
36 disp('Error: there must be at least 2 arguments!');
37 return
38 end
39
40 if nargin < 3
41 fout = 'all.%05d.nc';
42 end
43 if nargin < 4
44 fsize = 2.0e+9;
45 end
46
47
48 %===== Find and open all the matching files =====
49
50 nt = 0;
51 nf = 0;
52 all_ncf = struct([]);
53
54 % Find all of the files
55 exch2_msg = 0;
56 tmax = 200;
57 frdone = 0;
58 it = 0;
59 while frdone == 0
60
61 it = it + 1;
62 fnm = sprintf(fpat,it);
63 % disp(fnm);
64
65 % Check that the file exists
66 fid = fopen(fnm, 'r');
67 if fid < 0
68 if it >= tmax
69 frdone = 1;
70 end
71 continue;
72 end
73
74 % Open the NetCDF file
75 fnc = netcdf(fnm, 'nowrite');
76 if length(fnc) == 0
77 continue;
78 end
79
80 % Check for exch1/exch2 grid
81 exch = 1;
82 exch2_myFace = fnc.exch2_myFace(:);
83 if length(exch2_myFace) ~= 0
84 exch = 2;
85 if exch2_msg == 0
86 exch2_msg = 1;
87 disp(' Grid type appears to be: "exch2"');
88 end
89 end
90
91 n = length(all_ncf) + 1;
92 all_ncf(n).name = fnm;
93 all_ncf(n).nc = {fnc};
94 all_ncf(n).exch = exch;
95 all_ncf(n).tile_number = fnc.tile_number(1);
96 all_ncf(n).bi = fnc.bi(1);
97 all_ncf(n).bj = fnc.bj(1);
98 all_ncf(n).sNx = fnc.sNx(1);
99 all_ncf(n).sNy = fnc.sNy(1);
100 all_ncf(n).Nx = fnc.Nx(1);
101 all_ncf(n).Ny = fnc.Ny(1);
102 all_ncf(n).Z = fnc.Z(1);
103
104 if exch == 2
105 all_ncf(n).exch2_myFace = exch2_myFace;
106 all_ncf(n).exch2_tbasex = fnc.exch2_tbasex(1);
107 all_ncf(n).exch2_tbasey = fnc.exch2_tbasex(1);
108 end
109
110 clear fnc
111 end
112
113
114 %===== Do some basic sanity checks =====
115
116 % check for number of files/tiles found
117 if length(all_ncf) == 0
118 disp('Error: no tiles found--no need to do any assembly!');
119 return
120 elseif length(all_ncf) == 1
121 disp('Error: one tile found--no need to do any assembly!');
122 return
123 else
124 disp(sprintf(' Found %d files matching the pattern: "%s"', ...
125 length(all_ncf), fpat ));
126 end
127
128 % check for consistent "exch" version
129 if prod(double([all_ncf.exch] == all_ncf(1).exch)) ~= 1
130 disp('Error: not all the "exch" types of the files match.');
131 return;
132 end
133
134 % check for consistent sNx,sNy
135 if (prod(double([all_ncf.sNx] == all_ncf(1).sNx)) ~= 1) ...
136 | (prod(double([all_ncf.sNy] == all_ncf(1).sNy)) ~= 1)
137 disp('Error: the "sNx,sNy" values for all the tiles are not');
138 disp(' uniform. Future versions of this function will be');
139 disp(' able to handle non-uniform grid sizes but this');
140 disp(' feature is not yet implemented.');
141 return;
142 end
143
144 % check for redundant tiles and "time series" output
145 if length(all_ncf) ~= length(unique([all_ncf.tile_number]))
146 disp('Error: redundant tiles were found. Please check that');
147 disp(' the file pattern does not specify output spanning');
148 disp(' multiple model runs or even multiple time series');
149 disp(' within a single model run. For multi-time-series');
150 disp(' data sets, EACH "LEVEL" IN THE OUTPUT SERIES MUST');
151 disp(' BE ASSEMBLED SEPARATERLY.');
152 return
153 end
154
155
156 %===== Get the dims/vars associations =====
157
158 mydims = struct('names', {}, 'lens', {});
159 myvars = struct([]);
160 clear tncf;
161 for ivar = 1:length(vars)
162 mydim_names = {};
163 mydim_sizes = {};
164 myatt.names = {};
165 myatt.types = {};
166 myatt.data = {};
167 myname = vars(ivar).name;
168 disp([' Looking for variable: ' myname]);
169
170 itile = 1;
171 tncf = all_ncf(itile).nc{1};
172 ncv = tncf{myname};
173 len = length(ncv);
174 if length(ncv) == 0
175 warns = [' Warning: variable "%s" is not defined in "%s"\n' ...
176 ' so it will be ignored.'];
177 disp(sprintf(warns,myname,all_ncf(itile).name));
178 continue
179 end
180 mytype = datatype(ncv);
181 tmpdims = dim(ncv);
182 for inm = 1:length(tmpdims)
183 mydim_names{inm} = name(tmpdims{inm});
184 mydim_sizes{inm} = tmpdims{inm}(:);
185 end
186 for iat = 1:length(att(ncv))
187 aaa = att(ncv);
188 myatt.names(iat) = { name(aaa{iat}) };
189 myatt.types(iat) = { datatype(aaa{iat}) };
190 aab = aaa{iat};
191 myatt.data(iat) = { aab(:) };
192 end
193
194 % confirm: vars have same dim names across all files
195 ierr = 0;
196 for itile = 2:length(all_ncf)
197 tncf = all_ncf(itile).nc{1};
198 ncv = tncf{myname};
199 len = length(ncv);
200 if length(ncv) == 0
201 warns = [' Warning: variable "%s" is not defined in "%s"\n' ...
202 ' so it will be ignored.'];
203 disp(sprintf(warns,myname,all_ncf(itile).name));
204 continue
205 end
206 tmpdims = dim(ncv);
207 for inm = 1:length(tmpdims)
208 if mydim_names{inm} ~= name(tmpdims{inm})
209 warns = ...
210 [' Warning: variable "%s" is not CONSISTENTLY defined.\n' ...
211 ' It has different dimensions in different files so\n' ...
212 ' so it will be ignored.'];
213 disp(sprintf(warns,myname));
214 ierr = 1;
215 break
216 end
217 mydim_sizes{inm} = max([ tmpdims{inm}(:) mydim_sizes{inm} ]);
218 end
219
220 end
221
222 if ierr == 0
223 % check: does the variable have a "horizontal" component
224 has_horiz = 0;
225 horiz_names = { 'X' 'Y' 'Xp1' 'Yp1' };
226 for id = 1:length(mydim_names)
227 if length([intersect(horiz_names,mydim_names{id})]) > 0
228 has_horiz = 1;
229 end
230 end
231 % disp([ ' ' myname ' ' sprintf('%d',has_horiz) ]);
232
233 imy = length(myvars) + 1;
234 myvars(imy).name = myname;
235 myvars(imy).type = mytype;
236 myvars(imy).dim_names = mydim_names;
237 myvars(imy).dim_sizes = mydim_sizes;
238 myvars(imy).atts = myatt;
239 myvars(imy).has_horiz = has_horiz;
240
241 % this is necessary to make it work with Matlab 6.5
242 if isempty([mydims.names])
243 addl = mydim_names;
244 else
245 addl = setdiff(mydim_names,[mydims.names]);
246 end
247 for iaddl = 1:length(addl)
248 np1 = length(mydims) + 1;
249 mydims(np1).names = addl(iaddl);
250 mydims(np1).lens = mydim_sizes(find(strcmp(addl(iaddl),mydim_names)));
251 end
252
253 end
254 end
255
256 % For exch == 2, we need to add a "face" dimension
257 if all_ncf(1).exch == 2
258 np1 = length(mydims) + 1;
259 mydims(np1).names = { 'iface' };
260 mydims(np1).lens = { length(unique([all_ncf.exch2_myFace])) };
261 end
262
263 % myvars.name
264 % myvars.dim_names
265 % myvars.dim_sizes
266 % myvars(2).dim_names
267 % myvars(2).dim_names(4)
268
269 % mydims
270 % length(mydims)
271 % [ mydims.names ]
272 % [ mydims.lens ]
273
274
275 %===== Assemble! =====
276
277
278 if all_ncf(1).exch == 1
279
280 % exch "1":
281
282 % $$$ bi_max = max([all_ncf.bi]);
283 % $$$ bj_max = max([all_ncf.bj]);
284 % $$$ Xmax = bi_max * all_ncf(1).sNx;
285 % $$$ Ymax = bj_max * all_ncf(1).sNy;
286 Xmax = all_ncf(1).Nx;
287 Ymax = all_ncf(1).Ny;
288 % at this point I have to make some assumptions about the domain
289 % decomposition
290 bi_max = Xmax/all_ncf(1).sNx;
291 bj_max = Ymax/all_ncf(1).sNy;
292 itile = 0;
293 for bj=1:bj_max
294 for bi=1:bi_max
295 itile = itile+1;
296 all_ncf(itile).bi=bi;
297 all_ncf(itile).bj=bj;
298 end
299 end
300
301 horzdim = struct('names',{},'lens',{});
302 horzdim(1).names = { 'X' }; horzdim(1).lens = { Xmax };
303 horzdim(2).names = {'Xp1'}; horzdim(2).lens = { Xmax + 1 };
304 horzdim(3).names = { 'Y' }; horzdim(3).lens = { Ymax };
305 horzdim(4).names = {'Yp1'}; horzdim(4).lens = { Ymax + 1 };
306 horzdim(5).names = { 'T' }; horzdim(5).lens = { 0 };
307
308 iseq = 0;
309 foutnm = sprintf(fout, iseq);
310 fonc = netcdf(foutnm,'clobber'); % Should append-or-create!
311
312 for idim = 1:length(mydims)
313 dname = mydims(idim).names{1};
314 ind = find(strcmp(dname,[horzdim.names]));
315 if length(ind) ~= 0
316 dlen = horzdim(ind).lens{1};
317 else
318 dlen = mydims(idim).lens{1};
319 end
320 comm = sprintf('fonc(''%s'') = %d;',dname,dlen);
321 eval(comm);
322 end
323
324 for ivar = 1:length(myvars)
325 comm = sprintf('fonc{''%s''} = nc%s( ',myvars(ivar).name,myvars(ivar).type);
326 id = 1;
327 comm = [ comm sprintf('''%s''',myvars(ivar).dim_names{id}) ];
328 for id = 2:length(myvars(ivar).dim_names)
329 comm = [ comm sprintf(',''%s''',myvars(ivar).dim_names{id}) ];
330 end
331 comm = [ comm ' );' ];
332 eval(comm);
333 for iat = 1:length(myvars(ivar).atts.names)
334 comm = sprintf( ...
335 'fonc{''%s''}.%s = nc%s( myvars(ivar).atts.data{iat} );', ...
336 myvars(ivar).name, ...
337 myvars(ivar).atts.names{iat}, ...
338 myvars(ivar).atts.types{iat} );
339 eval(comm);
340 end
341 end
342
343 % for itime = 1:Tmax
344
345 % Here is where we need to check the output file size and start
346 % another file in the sequence, if necessary.
347
348 for ivar = 1:length(myvars)
349 disp(sprintf(' Copying variable: %s',myvars(ivar).name))
350 for itile = 1:length(all_ncf)
351
352 if (myvars(ivar).has_horiz == 1) | (itile == 1)
353
354 clear nct;
355 nct = all_ncf(itile).nc{1};
356 ox_off = (all_ncf(itile).bi - 1)*all_ncf(itile).sNx;
357 oy_off = (all_ncf(itile).bj - 1)*all_ncf(itile).sNy;
358 diml_in = '';
359 diml_out = '';
360 for jj = 1:length(myvars(ivar).dim_names)
361 doff = 1;
362 if jj > 1
363 diml_in = sprintf('%s,',diml_in);
364 diml_out = sprintf('%s,',diml_out);
365 end
366 dlen = myvars(ivar).dim_sizes{jj};
367 diml_in = sprintf('%s%s',diml_in, ':');
368 fchar = myvars(ivar).dim_names{jj}(1);
369 % disp([' fchar = ' fchar ' ' myvars(ivar).dim_names{jj}]);
370 if strcmp(myvars(ivar).dim_names{jj}(1),'X') == 1
371 doff = ox_off + doff;
372 dlen = ox_off + dlen;
373 end
374 if strcmp(myvars(ivar).dim_names{jj}(1),'Y') == 1
375 doff = oy_off + doff;
376 dlen = oy_off + dlen;
377 end
378 diml_out = sprintf('%s%d%s%d',diml_out,doff,':',dlen);
379 end
380
381 comm = sprintf( ...
382 'fonc{''%s''}(%s) = nct{''%s''}(%s);', ...
383 myvars(ivar).name, diml_out, myvars(ivar).name, diml_in );
384 % disp([ ' comm: ' comm ]);
385 eval(comm);
386
387 end
388
389 end
390 end
391 % end
392
393 fonc = close(fonc);
394
395 elseif all_ncf(1).exch == 2
396
397 % exch "2":
398 Xmax = 0;
399 Ymax = 0;
400 for ii = 1:length(all_ncf)
401 Xmax = max(Xmax, (all_ncf(ii).exch2_tbasex + all_ncf(ii).sNx));
402 Ymax = max(Ymax, (all_ncf(ii).exch2_tbasey + all_ncf(ii).sNy));
403 end
404
405 horzdim = struct('names',{},'lens',{});
406 horzdim(1).names = { 'X' }; horzdim(1).lens = { Xmax };
407 horzdim(2).names = {'Xp1'}; horzdim(2).lens = { Xmax + 1 };
408 horzdim(3).names = { 'Y' }; horzdim(3).lens = { Ymax };
409 horzdim(4).names = {'Yp1'}; horzdim(4).lens = { Ymax + 1 };
410 horzdim(5).names = { 'T' }; horzdim(5).lens = { 0 };
411
412 iseq = 0;
413 foutnm = sprintf(fout, iseq);
414 fonc = netcdf(foutnm,'clobber'); % Should append-or-create!
415
416 for idim = 1:length(mydims)
417 dname = mydims(idim).names{1};
418 ind = find(strcmp(dname,[horzdim.names]));
419 if length(ind) ~= 0
420 dlen = horzdim(ind).lens{1};
421 else
422 dlen = mydims(idim).lens{1};
423 end
424 comm = sprintf('fonc(''%s'') = %d;',dname,dlen);
425 eval(comm);
426 end
427
428 for ivar = 1:length(myvars)
429 comm = sprintf('fonc{''%s''} = nc%s( ',myvars(ivar).name,myvars(ivar).type);
430 id = 1;
431 comm = [ comm sprintf('''%s''',myvars(ivar).dim_names{id}) ];
432 for id = 2:length(myvars(ivar).dim_names)
433 dname = myvars(ivar).dim_names{id};
434 if (dname(1) == 'Y') & (myvars(ivar).has_horiz == 1)
435 comm = [ comm sprintf(',''%s''','iface') ];
436 end
437 comm = [ comm sprintf(',''%s''',dname) ];
438 end
439 comm = [ comm ' );' ];
440 eval(comm);
441 for iat = 1:length(myvars(ivar).atts.names)
442 comm = sprintf( ...
443 'fonc{''%s''}.%s = nc%s( myvars(ivar).atts.data{iat} );', ...
444 myvars(ivar).name, ...
445 myvars(ivar).atts.names{iat}, ...
446 myvars(ivar).atts.types{iat} );
447 eval(comm);
448 end
449 end
450
451 % Here is where we need to check the output file size and start
452 % another file in the sequence, if necessary.
453
454 for ivar = 1:length(myvars)
455 disp(sprintf(' Copying variable: %s',myvars(ivar).name))
456 for itile = 1:length(all_ncf)
457
458 if (myvars(ivar).has_horiz == 1) | (itile == 1)
459
460 clear nct;
461 nct = all_ncf(itile).nc{1};
462 ox_off = all_ncf(itile).exch2_tbasex;
463 oy_off = all_ncf(itile).exch2_tbasey;
464 diml_tin = '';
465 diml_res = '';
466 diml_in = '';
467 diml_out = '';
468 if length(myvars(ivar).dim_names) < 2
469 comm = sprintf( ...
470 'fonc{''%s''}(%s%d) = nct{''%s''}(:);', ...
471 myvars(ivar).name, '1:', myvars(ivar).dim_sizes{1}, ...
472 myvars(ivar).name );
473 % disp([ ' ' comm ]);
474 eval(comm);
475 else
476 for jj = 1:length(myvars(ivar).dim_names)
477 doff = 1;
478 if jj > 1
479 diml_tin = sprintf('%s,',diml_tin);
480 diml_res = sprintf('%s,',diml_res);
481 diml_in = sprintf('%s,',diml_in);
482 diml_out = sprintf('%s,',diml_out);
483 end
484 dnam = myvars(ivar).dim_names{jj};
485 dlen = myvars(ivar).dim_sizes{jj};
486 dlenr = dlen;
487 fchar = myvars(ivar).dim_names{jj}(1);
488 % disp([' fchar = ' fchar ' ' myvars(ivar).dim_names{jj}]);
489 if strcmp(dnam(1),'X') == 1
490 doff = ox_off + doff;
491 dlen = ox_off + dlen;
492 end
493 if strcmp(dnam(1),'Y') == 1
494 diml_res = sprintf('%s%s',diml_res, '[],');
495 diml_in = sprintf('%s%s',diml_in, ':,');
496 diml_out = sprintf('%s%d%s',diml_out,all_ncf(itile).exch2_myFace,',');
497 doff = oy_off + doff;
498 dlen = oy_off + dlen;
499 end
500 diml_tin = sprintf('%s%s',diml_tin, ':');
501 diml_res = sprintf('%s%d',diml_res, dlenr);
502 diml_in = sprintf('%s%s',diml_in, ':');
503 diml_out = sprintf('%s%d%s%d',diml_out,doff,':',dlen);
504 end
505
506 comm = sprintf( ...
507 'tmp = reshape(nct{''%s''}(%s), %s); fonc{''%s''}(%s) = tmp(%s);', ...
508 myvars(ivar).name, diml_tin, diml_res, myvars(ivar).name, ...
509 diml_out, diml_in );
510 % disp([ ' ' comm ]);
511 eval(comm);
512 end
513
514 end
515
516 end
517 end
518 % end
519
520 fonc = close(fonc);
521
522 end
523

  ViewVC Help
Powered by ViewVC 1.1.22