/[MITgcm]/MITgcm/utils/matlab/mnc_assembly.m
ViewVC logotype

Annotation of /MITgcm/utils/matlab/mnc_assembly.m

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.4 - (hide annotations) (download)
Tue Feb 8 08:45:58 2005 UTC (19 years, 3 months ago) by mlosch
Branch: MAIN
CVS Tags: checkpoint57o_post, checkpoint57m_post, checkpoint57k_post, checkpoint57d_post, checkpoint57g_post, checkpoint57i_post, checkpoint57e_post, checkpoint57g_pre, checkpoint57f_pre, eckpoint57e_pre, checkpoint57h_done, checkpoint57n_post, checkpoint57p_post, checkpoint57f_post, checkpoint57j_post, checkpoint57h_pre, checkpoint57l_post, checkpoint57h_post
Changes since 1.3: +6 -1 lines
o include fix for Matlab 6.1
o mnc_assembly should now work with Matlab 6.1, 6.5, and 7
o for exch type 1, it will also handle both multiple processors and
  tiles
o to be done: do the above for exch type 2 (cubed sphere)

1 edhill 1.1 function [nt,nf] = mnc_assembly(fpat,vars, fout,fsize)
2    
3     % Function [nt,nf] = mnc_assembly(fpat,vars, fout,fsize)
4     %
5     % INPUTS
6     % fpat string containing the file pattern
7 mlosch 1.3 % vars structure array of variable names
8 edhill 1.1 %
9     % fout output file pattern (DEF: "all.%05d.nc")
10     % fsize max output file size (DEF: 2.0e+9 = +/-2GB)
11     %
12     % OUTPUTS
13     % nt number of usable tiles found
14     % nf number of output files written
15     %
16     % This function "assembles" MNC output. It finds all the per-tile
17     % NetCDF files that match the input pattern, does some basic "sanity"
18     % tests to determine whether the files have compatible sizes, and
19     % then assembles all of the requested data (all of the variables)
20     % into one or more "global" NetCDF files. The global files have
21     % the following dimension conventions:
22     %
23     % "exch 1": all values are within a global horizontal grid
24     % and indicies are (X,Y,Z,T)
25     %
26     % "exch 2": all values are within one of up to six "faces"
27     % of a global cube with indicies (Xf,Yf,F,Z,T)
28     %
29     % where "X,Y.Z,T" are global space/time indicies, "Xf,Yf" are local
30     % per-face spatial indicies, and "F" is a face index.
31    
32    
33     %===== Argument checking and defaults =====
34    
35     if nargin < 2
36     disp('Error: there must be at least 2 arguments!');
37     return
38     end
39    
40     if nargin < 3
41     fout = 'all.%05d.nc';
42     end
43     if nargin < 4
44     fsize = 2.0e+9;
45     end
46    
47    
48     %===== Find and open all the matching files =====
49    
50     nt = 0;
51     nf = 0;
52     all_ncf = struct([]);
53    
54     % Find all of the files
55     exch2_msg = 0;
56     tmax = 200;
57     frdone = 0;
58     it = 0;
59     while frdone == 0
60    
61     it = it + 1;
62     fnm = sprintf(fpat,it);
63     % disp(fnm);
64    
65     % Check that the file exists
66     fid = fopen(fnm, 'r');
67     if fid < 0
68     if it >= tmax
69     frdone = 1;
70     end
71     continue;
72     end
73    
74     % Open the NetCDF file
75     fnc = netcdf(fnm, 'nowrite');
76     if length(fnc) == 0
77     continue;
78     end
79    
80     % Check for exch1/exch2 grid
81     exch = 1;
82     exch2_myFace = fnc.exch2_myFace(:);
83     if length(exch2_myFace) ~= 0
84     exch = 2;
85     if exch2_msg == 0
86     exch2_msg = 1;
87     disp(' Grid type appears to be: "exch2"');
88     end
89     end
90    
91     n = length(all_ncf) + 1;
92     all_ncf(n).name = fnm;
93     all_ncf(n).nc = {fnc};
94     all_ncf(n).exch = exch;
95     all_ncf(n).tile_number = fnc.tile_number(1);
96     all_ncf(n).bi = fnc.bi(1);
97     all_ncf(n).bj = fnc.bj(1);
98     all_ncf(n).sNx = fnc.sNx(1);
99     all_ncf(n).sNy = fnc.sNy(1);
100 mlosch 1.2 all_ncf(n).Nx = fnc.Nx(1);
101     all_ncf(n).Ny = fnc.Ny(1);
102 edhill 1.1 all_ncf(n).Z = fnc.Z(1);
103    
104     if exch == 2
105     all_ncf(n).exch2_myFace = exch2_myFace;
106     all_ncf(n).exch2_tbasex = fnc.exch2_tbasex(1);
107     all_ncf(n).exch2_tbasey = fnc.exch2_tbasex(1);
108     end
109    
110     clear fnc
111     end
112    
113    
114     %===== Do some basic sanity checks =====
115    
116     % check for number of files/tiles found
117     if length(all_ncf) == 0
118     disp('Error: no tiles found--no need to do any assembly!');
119     return
120     elseif length(all_ncf) == 1
121     disp('Error: one tile found--no need to do any assembly!');
122     return
123     else
124     disp(sprintf(' Found %d files matching the pattern: "%s"', ...
125     length(all_ncf), fpat ));
126     end
127    
128     % check for consistent "exch" version
129     if prod(double([all_ncf.exch] == all_ncf(1).exch)) ~= 1
130     disp('Error: not all the "exch" types of the files match.');
131     return;
132     end
133    
134     % check for consistent sNx,sNy
135     if (prod(double([all_ncf.sNx] == all_ncf(1).sNx)) ~= 1) ...
136 mlosch 1.2 | (prod(double([all_ncf.sNy] == all_ncf(1).sNy)) ~= 1)
137 edhill 1.1 disp('Error: the "sNx,sNy" values for all the tiles are not');
138     disp(' uniform. Future versions of this function will be');
139     disp(' able to handle non-uniform grid sizes but this');
140     disp(' feature is not yet implemented.');
141     return;
142     end
143    
144     % check for redundant tiles and "time series" output
145     if length(all_ncf) ~= length(unique([all_ncf.tile_number]))
146     disp('Error: redundant tiles were found. Please check that');
147     disp(' the file pattern does not specify output spanning');
148     disp(' multiple model runs or even multiple time series');
149     disp(' within a single model run. For multi-time-series');
150     disp(' data sets, EACH "LEVEL" IN THE OUTPUT SERIES MUST');
151     disp(' BE ASSEMBLED SEPARATERLY.');
152     return
153     end
154    
155    
156     %===== Get the dims/vars associations =====
157    
158     mydims = struct('names', {}, 'lens', {});
159     myvars = struct([]);
160     clear tncf;
161     for ivar = 1:length(vars)
162     mydim_names = {};
163     mydim_sizes = {};
164     myatt.names = {};
165     myatt.types = {};
166     myatt.data = {};
167     myname = vars(ivar).name;
168     disp([' Looking for variable: ' myname]);
169    
170     itile = 1;
171     tncf = all_ncf(itile).nc{1};
172     ncv = tncf{myname};
173     len = length(ncv);
174     if length(ncv) == 0
175     warns = [' Warning: variable "%s" is not defined in "%s"\n' ...
176     ' so it will be ignored.'];
177     disp(sprintf(warns,myname,all_ncf(itile).name));
178     continue
179     end
180     mytype = datatype(ncv);
181     tmpdims = dim(ncv);
182     for inm = 1:length(tmpdims)
183     mydim_names{inm} = name(tmpdims{inm});
184     mydim_sizes{inm} = tmpdims{inm}(:);
185     end
186     for iat = 1:length(att(ncv))
187     aaa = att(ncv);
188     myatt.names(iat) = { name(aaa{iat}) };
189     myatt.types(iat) = { datatype(aaa{iat}) };
190     aab = aaa{iat};
191     myatt.data(iat) = { aab(:) };
192     end
193    
194     % confirm: vars have same dim names across all files
195     ierr = 0;
196     for itile = 2:length(all_ncf)
197     tncf = all_ncf(itile).nc{1};
198     ncv = tncf{myname};
199     len = length(ncv);
200     if length(ncv) == 0
201     warns = [' Warning: variable "%s" is not defined in "%s"\n' ...
202     ' so it will be ignored.'];
203     disp(sprintf(warns,myname,all_ncf(itile).name));
204     continue
205     end
206     tmpdims = dim(ncv);
207     for inm = 1:length(tmpdims)
208     if mydim_names{inm} ~= name(tmpdims{inm})
209     warns = ...
210     [' Warning: variable "%s" is not CONSISTENTLY defined.\n' ...
211     ' It has different dimensions in different files so\n' ...
212     ' so it will be ignored.'];
213     disp(sprintf(warns,myname));
214     ierr = 1;
215     break
216     end
217     mydim_sizes{inm} = max([ tmpdims{inm}(:) mydim_sizes{inm} ]);
218     end
219    
220     end
221    
222     if ierr == 0
223     % check: does the variable have a "horizontal" component
224     has_horiz = 0;
225     horiz_names = { 'X' 'Y' 'Xp1' 'Yp1' };
226     for id = 1:length(mydim_names)
227     if length([intersect(horiz_names,mydim_names{id})]) > 0
228     has_horiz = 1;
229     end
230     end
231     % disp([ ' ' myname ' ' sprintf('%d',has_horiz) ]);
232    
233     imy = length(myvars) + 1;
234     myvars(imy).name = myname;
235     myvars(imy).type = mytype;
236     myvars(imy).dim_names = mydim_names;
237     myvars(imy).dim_sizes = mydim_sizes;
238     myvars(imy).atts = myatt;
239     myvars(imy).has_horiz = has_horiz;
240    
241 mlosch 1.4 % this is necessary to make it work with Matlab 6.5
242     if isempty([mydims.names])
243     addl = mydim_names;
244     else
245     addl = setdiff(mydim_names,[mydims.names]);
246     end
247 edhill 1.1 for iaddl = 1:length(addl)
248     np1 = length(mydims) + 1;
249     mydims(np1).names = addl(iaddl);
250     mydims(np1).lens = mydim_sizes(find(strcmp(addl(iaddl),mydim_names)));
251     end
252    
253     end
254     end
255    
256     % For exch == 2, we need to add a "face" dimension
257     if all_ncf(1).exch == 2
258     np1 = length(mydims) + 1;
259     mydims(np1).names = { 'iface' };
260     mydims(np1).lens = { length(unique([all_ncf.exch2_myFace])) };
261     end
262    
263     % myvars.name
264     % myvars.dim_names
265     % myvars.dim_sizes
266     % myvars(2).dim_names
267     % myvars(2).dim_names(4)
268    
269     % mydims
270     % length(mydims)
271     % [ mydims.names ]
272     % [ mydims.lens ]
273    
274    
275     %===== Assemble! =====
276    
277    
278     if all_ncf(1).exch == 1
279    
280     % exch "1":
281    
282 mlosch 1.2 % $$$ bi_max = max([all_ncf.bi]);
283     % $$$ bj_max = max([all_ncf.bj]);
284     % $$$ Xmax = bi_max * all_ncf(1).sNx;
285     % $$$ Ymax = bj_max * all_ncf(1).sNy;
286     Xmax = all_ncf(1).Nx;
287     Ymax = all_ncf(1).Ny;
288     % at this point I have to make some assumptions about the domain
289     % decomposition
290     bi_max = Xmax/all_ncf(1).sNx;
291     bj_max = Ymax/all_ncf(1).sNy;
292     itile = 0;
293     for bj=1:bj_max
294     for bi=1:bi_max
295     itile = itile+1;
296     all_ncf(itile).bi=bi;
297     all_ncf(itile).bj=bj;
298     end
299     end
300    
301 edhill 1.1 horzdim = struct('names',{},'lens',{});
302     horzdim(1).names = { 'X' }; horzdim(1).lens = { Xmax };
303     horzdim(2).names = {'Xp1'}; horzdim(2).lens = { Xmax + 1 };
304     horzdim(3).names = { 'Y' }; horzdim(3).lens = { Ymax };
305     horzdim(4).names = {'Yp1'}; horzdim(4).lens = { Ymax + 1 };
306     horzdim(5).names = { 'T' }; horzdim(5).lens = { 0 };
307    
308     iseq = 0;
309     foutnm = sprintf(fout, iseq);
310     fonc = netcdf(foutnm,'clobber'); % Should append-or-create!
311    
312     for idim = 1:length(mydims)
313     dname = mydims(idim).names{1};
314     ind = find(strcmp(dname,[horzdim.names]));
315     if length(ind) ~= 0
316     dlen = horzdim(ind).lens{1};
317     else
318     dlen = mydims(idim).lens{1};
319     end
320     comm = sprintf('fonc(''%s'') = %d;',dname,dlen);
321     eval(comm);
322     end
323    
324     for ivar = 1:length(myvars)
325     comm = sprintf('fonc{''%s''} = nc%s( ',myvars(ivar).name,myvars(ivar).type);
326     id = 1;
327     comm = [ comm sprintf('''%s''',myvars(ivar).dim_names{id}) ];
328     for id = 2:length(myvars(ivar).dim_names)
329     comm = [ comm sprintf(',''%s''',myvars(ivar).dim_names{id}) ];
330     end
331     comm = [ comm ' );' ];
332     eval(comm);
333     for iat = 1:length(myvars(ivar).atts.names)
334     comm = sprintf( ...
335     'fonc{''%s''}.%s = nc%s( myvars(ivar).atts.data{iat} );', ...
336     myvars(ivar).name, ...
337     myvars(ivar).atts.names{iat}, ...
338     myvars(ivar).atts.types{iat} );
339     eval(comm);
340     end
341     end
342    
343     % for itime = 1:Tmax
344    
345     % Here is where we need to check the output file size and start
346     % another file in the sequence, if necessary.
347    
348     for ivar = 1:length(myvars)
349     disp(sprintf(' Copying variable: %s',myvars(ivar).name))
350     for itile = 1:length(all_ncf)
351    
352 mlosch 1.2 if (myvars(ivar).has_horiz == 1) | (itile == 1)
353 edhill 1.1
354     clear nct;
355     nct = all_ncf(itile).nc{1};
356     ox_off = (all_ncf(itile).bi - 1)*all_ncf(itile).sNx;
357     oy_off = (all_ncf(itile).bj - 1)*all_ncf(itile).sNy;
358     diml_in = '';
359     diml_out = '';
360     for jj = 1:length(myvars(ivar).dim_names)
361     doff = 1;
362     if jj > 1
363     diml_in = sprintf('%s,',diml_in);
364     diml_out = sprintf('%s,',diml_out);
365     end
366     dlen = myvars(ivar).dim_sizes{jj};
367     diml_in = sprintf('%s%s',diml_in, ':');
368     fchar = myvars(ivar).dim_names{jj}(1);
369     % disp([' fchar = ' fchar ' ' myvars(ivar).dim_names{jj}]);
370     if strcmp(myvars(ivar).dim_names{jj}(1),'X') == 1
371     doff = ox_off + doff;
372     dlen = ox_off + dlen;
373     end
374     if strcmp(myvars(ivar).dim_names{jj}(1),'Y') == 1
375     doff = oy_off + doff;
376     dlen = oy_off + dlen;
377     end
378     diml_out = sprintf('%s%d%s%d',diml_out,doff,':',dlen);
379     end
380    
381     comm = sprintf( ...
382     'fonc{''%s''}(%s) = nct{''%s''}(%s);', ...
383     myvars(ivar).name, diml_out, myvars(ivar).name, diml_in );
384     % disp([ ' comm: ' comm ]);
385     eval(comm);
386    
387     end
388    
389     end
390     end
391     % end
392    
393     fonc = close(fonc);
394    
395     elseif all_ncf(1).exch == 2
396    
397     % exch "2":
398     Xmax = 0;
399     Ymax = 0;
400     for ii = 1:length(all_ncf)
401     Xmax = max(Xmax, (all_ncf(ii).exch2_tbasex + all_ncf(ii).sNx));
402     Ymax = max(Ymax, (all_ncf(ii).exch2_tbasey + all_ncf(ii).sNy));
403     end
404    
405     horzdim = struct('names',{},'lens',{});
406     horzdim(1).names = { 'X' }; horzdim(1).lens = { Xmax };
407     horzdim(2).names = {'Xp1'}; horzdim(2).lens = { Xmax + 1 };
408     horzdim(3).names = { 'Y' }; horzdim(3).lens = { Ymax };
409     horzdim(4).names = {'Yp1'}; horzdim(4).lens = { Ymax + 1 };
410     horzdim(5).names = { 'T' }; horzdim(5).lens = { 0 };
411    
412     iseq = 0;
413     foutnm = sprintf(fout, iseq);
414     fonc = netcdf(foutnm,'clobber'); % Should append-or-create!
415    
416     for idim = 1:length(mydims)
417     dname = mydims(idim).names{1};
418     ind = find(strcmp(dname,[horzdim.names]));
419     if length(ind) ~= 0
420     dlen = horzdim(ind).lens{1};
421     else
422     dlen = mydims(idim).lens{1};
423     end
424     comm = sprintf('fonc(''%s'') = %d;',dname,dlen);
425     eval(comm);
426     end
427    
428     for ivar = 1:length(myvars)
429     comm = sprintf('fonc{''%s''} = nc%s( ',myvars(ivar).name,myvars(ivar).type);
430     id = 1;
431     comm = [ comm sprintf('''%s''',myvars(ivar).dim_names{id}) ];
432     for id = 2:length(myvars(ivar).dim_names)
433     dname = myvars(ivar).dim_names{id};
434 mlosch 1.2 if (dname(1) == 'Y') & (myvars(ivar).has_horiz == 1)
435 edhill 1.1 comm = [ comm sprintf(',''%s''','iface') ];
436     end
437     comm = [ comm sprintf(',''%s''',dname) ];
438     end
439     comm = [ comm ' );' ];
440     eval(comm);
441     for iat = 1:length(myvars(ivar).atts.names)
442     comm = sprintf( ...
443     'fonc{''%s''}.%s = nc%s( myvars(ivar).atts.data{iat} );', ...
444     myvars(ivar).name, ...
445     myvars(ivar).atts.names{iat}, ...
446     myvars(ivar).atts.types{iat} );
447     eval(comm);
448     end
449     end
450    
451     % Here is where we need to check the output file size and start
452     % another file in the sequence, if necessary.
453    
454     for ivar = 1:length(myvars)
455     disp(sprintf(' Copying variable: %s',myvars(ivar).name))
456     for itile = 1:length(all_ncf)
457    
458 mlosch 1.2 if (myvars(ivar).has_horiz == 1) | (itile == 1)
459 edhill 1.1
460     clear nct;
461     nct = all_ncf(itile).nc{1};
462     ox_off = all_ncf(itile).exch2_tbasex;
463     oy_off = all_ncf(itile).exch2_tbasey;
464     diml_tin = '';
465     diml_res = '';
466     diml_in = '';
467     diml_out = '';
468     if length(myvars(ivar).dim_names) < 2
469     comm = sprintf( ...
470     'fonc{''%s''}(%s%d) = nct{''%s''}(:);', ...
471     myvars(ivar).name, '1:', myvars(ivar).dim_sizes{1}, ...
472     myvars(ivar).name );
473     % disp([ ' ' comm ]);
474     eval(comm);
475     else
476     for jj = 1:length(myvars(ivar).dim_names)
477     doff = 1;
478     if jj > 1
479     diml_tin = sprintf('%s,',diml_tin);
480     diml_res = sprintf('%s,',diml_res);
481     diml_in = sprintf('%s,',diml_in);
482     diml_out = sprintf('%s,',diml_out);
483     end
484     dnam = myvars(ivar).dim_names{jj};
485     dlen = myvars(ivar).dim_sizes{jj};
486     dlenr = dlen;
487     fchar = myvars(ivar).dim_names{jj}(1);
488     % disp([' fchar = ' fchar ' ' myvars(ivar).dim_names{jj}]);
489     if strcmp(dnam(1),'X') == 1
490     doff = ox_off + doff;
491     dlen = ox_off + dlen;
492     end
493     if strcmp(dnam(1),'Y') == 1
494     diml_res = sprintf('%s%s',diml_res, '[],');
495     diml_in = sprintf('%s%s',diml_in, ':,');
496     diml_out = sprintf('%s%d%s',diml_out,all_ncf(itile).exch2_myFace,',');
497     doff = oy_off + doff;
498     dlen = oy_off + dlen;
499     end
500     diml_tin = sprintf('%s%s',diml_tin, ':');
501     diml_res = sprintf('%s%d',diml_res, dlenr);
502     diml_in = sprintf('%s%s',diml_in, ':');
503     diml_out = sprintf('%s%d%s%d',diml_out,doff,':',dlen);
504     end
505    
506     comm = sprintf( ...
507     'tmp = reshape(nct{''%s''}(%s), %s); fonc{''%s''}(%s) = tmp(%s);', ...
508     myvars(ivar).name, diml_tin, diml_res, myvars(ivar).name, ...
509     diml_out, diml_in );
510     % disp([ ' ' comm ]);
511     eval(comm);
512     end
513    
514     end
515    
516     end
517     end
518     % end
519    
520     fonc = close(fonc);
521    
522     end
523    

  ViewVC Help
Powered by ViewVC 1.1.22