/[MITgcm]/MITgcm/verification/tutorial_global_oce_latlon/diags_matlab/rdmds.m
ViewVC logotype

Contents of /MITgcm/verification/tutorial_global_oce_latlon/diags_matlab/rdmds.m

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.4 - (show annotations) (download)
Mon Apr 25 18:20:40 2011 UTC (13 years ago) by jmc
Branch: MAIN
CVS Tags: checkpoint64y, checkpoint64x, checkpoint64z, checkpoint64q, checkpoint64p, checkpoint64s, checkpoint64r, checkpoint64u, checkpoint64t, checkpoint64w, checkpoint64v, checkpoint64i, checkpoint64h, checkpoint64k, checkpoint64j, checkpoint64m, checkpoint64l, checkpoint64o, checkpoint64n, checkpoint64a, checkpoint64c, checkpoint64b, checkpoint64e, checkpoint64d, checkpoint64g, checkpoint64f, checkpoint63p, checkpoint63q, checkpoint63r, checkpoint63s, checkpoint63l, checkpoint63m, checkpoint63n, checkpoint63o, checkpoint63h, checkpoint63i, checkpoint63j, checkpoint63k, checkpoint63d, checkpoint63e, checkpoint63f, checkpoint63g, checkpoint63a, checkpoint63b, checkpoint63c, checkpoint64, checkpoint65, checkpoint63, checkpoint66g, checkpoint66f, checkpoint66e, checkpoint66d, checkpoint66c, checkpoint66b, checkpoint66a, checkpoint66o, checkpoint66n, checkpoint66m, checkpoint66l, checkpoint66k, checkpoint66j, checkpoint66i, checkpoint66h, checkpoint65z, checkpoint65x, checkpoint65y, checkpoint65r, checkpoint65s, checkpoint65p, checkpoint65q, checkpoint65v, checkpoint65w, checkpoint65t, checkpoint65u, checkpoint65j, checkpoint65k, checkpoint65h, checkpoint65i, checkpoint65n, checkpoint65o, checkpoint65l, checkpoint65m, checkpoint65b, checkpoint65c, checkpoint65a, checkpoint65f, checkpoint65g, checkpoint65d, checkpoint65e, checkpoint62w, checkpoint62z, checkpoint62y, checkpoint62x, HEAD
Changes since 1.3: +180 -64 lines
up-to-date version from MITgcm/utils/matlab

1 function [AA,itrs,MM] = rdmds(fnamearg,varargin)
2 % RDMDS Read MITgcmUV meta/data files
3 %
4 % A = RDMDS(FNAME)
5 % A = RDMDS(FNAME,ITER)
6 % A = RDMDS(FNAME,[ITER1 ITER2 ...])
7 % A = RDMDS(FNAME,NaN)
8 % A = RDMDS(FNAME,Inf)
9 % [A,ITS,M] = RDMDS(FNAME,[...])
10 % A = RDMDS(FNAME,[...],'rec',RECNUM)
11 %
12 % A = RDMDS(FNAME) reads data described by meta/data file format.
13 % FNAME is a string containing the "head" of the file names.
14 %
15 % eg. To load the meta-data files
16 % T.0000002880.000.000.meta, T.0000002880.000.000.data
17 % T.0000002880.001.000.meta, T.0000002880.001.000.data
18 % T.0000002880.002.000.meta, T.0000002880.002.000.data
19 % T.0000002880.003.000.meta, T.0000002880.003.000.data
20 % use
21 % >> A=rdmds('T.0000002880');
22 % >> size(A)
23 % ans =
24 % 64 32 5
25 % eg. To load a multiple record file
26 % >> A=rdmds('pickup.0000002880');
27 % >> size(A)
28 % ans =
29 % 64 32 5 61
30 %
31 %
32 % A = RDMDS(FNAME,ITER) reads data described by meta/data file format.
33 % FNAME is a string containing the "head" of the file name excluding the
34 % 10-digit iterartion number.
35 % ITER is a vector of positive integers that will expand to the 10-digit
36 % number in the file name.
37 % If ITER=NaN, all iterations will be read.
38 % If ITER=Inf, the last (highest) iteration will be read.
39 %
40 % eg. To repeat above operation
41 % >> A=rdmds('T',2880);
42 % eg. To read multiple time steps
43 % >> A=rdmds('T',[0 1440 2880]);
44 % eg. To read all time steps
45 % >> [A,ITS]=rdmds('T',NaN);
46 % eg. To read the last time step
47 % >> [A,IT]=rdmds('T',Inf);
48 % Note: this form can not read files with no iteration count in file name.
49 %
50 %
51 % A = RDMDS(FNAME,[...],'rec',RECNUM) reads individual records from
52 % multiple record files.
53 %
54 % eg. To read a single record from a multi-record file
55 % >> [A,IT]=rdmds('pickup.ckptA',11);
56 % eg. To read several records from a multi-record file
57 % >> [A,IT]=rdmds('pickup',Inf,'rec',[1:5 8 12]);
58 %
59 %
60 % A = RDMDS(FNAME,ITER,MACHINEFORMAT) allows the machine format to be
61 % A = RDMDS(FNAME,MACHINEFORMAT)
62 % specified which MACHINEFORMAT is on of the following strings:
63 % 'n' 'l' 'b' 'd' 'g' 'c' 'a' 's' - see FOPEN for more details
64 %
65
66 % $Header: /u/gcmpack/MITgcm/utils/matlab/rdmds.m,v 1.26 2010/01/10 01:51:03 jmc Exp $
67 % $Name: $
68
69 AA=[];
70 itrs=[];
71 MM=[];
72
73 % Default options
74 ieee='b';
75 fname=fnamearg;
76 userecords=0;
77 recnum=[];
78
79 % Check optional arguments
80 for ind=1:size(varargin,2);
81 arg=varargin{ind};
82 if ischar(arg)
83 if strcmp(arg,'n') | strcmp(arg,'native')
84 ieee='n';
85 elseif strcmp(arg,'l') | strcmp(arg,'ieee-le')
86 ieee='l';
87 elseif strcmp(arg,'b') | strcmp(arg,'ieee-be')
88 ieee='b';
89 elseif strcmp(arg,'c') | strcmp(arg,'cray')
90 ieee='c';
91 elseif strcmp(arg,'a') | strcmp(arg,'ieee-le.l64')
92 ieee='a';
93 elseif strcmp(arg,'s') | strcmp(arg,'ieee-be.l64')
94 ieee='s';
95 elseif strcmp(arg,'rec')
96 userecords=1;
97 else
98 error(['Optional argument ' arg ' is unknown'])
99 end
100 else
101 if userecords==1
102 recnum=arg;
103 elseif isempty(itrs)
104 if isnan(arg)
105 itrs=scanforfiles(fname);
106 disp([ sprintf('Reading %i time levels:',size(itrs,2)) sprintf(' %i',itrs) ]);
107 elseif isinf(arg)
108 itrs=scanforfiles(fname);
109 if isempty(itrs)
110 AA=[];itrs=[];return;
111 end
112 disp([ sprintf('Found %i time levels, reading %i',size(itrs,2),itrs(end)) ]);
113 itrs=itrs(end);
114 % elseif prod(double(arg>=0)) & prod(double(round(arg)==arg))
115 % elseif prod(arg>=0) & prod(round(arg)==arg)
116 elseif min(arg)>=0 & isempty(find(round(arg)~=arg))
117 if arg>=9999999999
118 error(sprintf('Argument %i > 9999999999',arg))
119 end
120 itrs=arg;
121 else
122 error(sprintf('Argument %i must be a positive integer',arg))
123 end
124 else
125 error('Multiple iterations should be specified as a vector')
126 end
127 end
128 end
129
130 if isempty(itrs)
131 itrs=-1;
132 end
133
134 % Loop over each iteration
135 for iter=1:size(itrs,2);
136 if itrs(iter)>=0
137 fname=sprintf('%s.%10.10i',fnamearg,itrs(iter));
138 end
139
140 % Figure out if there is a path in the filename
141 NS=findstr('/',fname);
142 if size(NS)>0
143 Dir=fname(1:NS(end));
144 else
145 Dir='./';
146 end
147
148 % Match name of all meta-files
149 allfiles=dir( sprintf('%s*.meta',fname) );
150
151 if size(allfiles,1)==0
152 disp(sprintf('No files match the search: %s*.meta',fname));
153 %allow partial reads% error('No files found.')
154 end
155
156 % Loop through allfiles
157 for j=1:size(allfiles,1);
158
159 % Read meta- and data-file
160 [A,N,M,mG] = localrdmds([Dir allfiles(j).name],ieee,recnum);
161
162 %- Merge local Meta file content (M) to MM string:
163 if j > 0, %- to comment out this block: "if j < 0" (since j is always > 0)
164 ii=findstr(M,' timeStepNumber');
165 if isempty(ii), ii1=0; ii2=0;
166 else ii1=ii; ii2=ii+min(findstr(M(1+ii:end),'];')); end
167 ii=findstr(M,' timeInterval');
168 if isempty(ii), jj1=0; jj2=0;
169 else jj1=ii; jj2=ii+min(findstr(M(1+ii:end),'];')); end
170 if iter==1 & j==1,
171 MM=M; ind1=0; ind2=0; is1=ii1; js1=jj1; M3='';
172 if ii1*jj1 > 0,
173 %ind1=min(ii1,jj1); ind2=max(ii2,jj2);
174 %if ii1 < jj1, ii3=ii2+1; jj3=jj1-1;
175 %else ii3=jj2+1; jj3=ii1-1; end
176 order=sort([ii1 ii2 jj1 jj2]);
177 ind1=order(1); ii3=order(2)+1; jj3=order(3)-1; ind2=order(4);
178 M2=M(ii1:ii2); M4=M(jj1:jj2); M3=M(ii3:jj3);
179 elseif ii1 > 0,
180 ind1=ii1; ind2=ii2;
181 M2=M(ii1:ii2); M4='';
182 elseif jj1 > 0,
183 ind1=jj1; ind2=jj2;
184 M4=M(jj1:jj2); M2='';
185 end
186 M5=M(1+ind2:end);
187 %fprintf(' ii1,ii2 = %i %i ; jj1,jj2= %i %i ;', ii1,ii2, jj1,jj2);
188 %fprintf(' ii3,jj3= %i %i ; ind1,ind2= %i %i\n',ii3,jj3,ind1,ind2);
189 %fprintf('M(1:ind1)=%s<\n',M(1:ind1));
190 %fprintf(' M2=%s<\n',M2);
191 %fprintf(' M3=%s<\n',M3);
192 %fprintf(' M4=%s<\n',M4);
193 %fprintf(' M5=%s<\n',M5);
194 else
195 if ii1*jj1 > 0,
196 order=sort([ii1 ii2 jj1 jj2]);
197 ind=order(1); ii3=order(2)+1; jj3=order(3)-1; ind2=order(4);
198 else ind=max(ii1,jj1); ind2=ii2+jj2; end
199 compar=(ind == ind1); ii=0;
200 if compar & ind1 == 0, ii=1; compar=strcmp (MM,M); end
201 if compar & ind1 > 0, ii=2; compar=strncmp(MM,M,ind1) ; end
202 if compar & ind1 > 0, ii=3; compar=strcmp(M5,M(1+ind2:end)); end
203 if compar & ii1*jj1 > 0, ii=4; compar=strcmp(M3,M(ii3:jj3)); end
204 if ~compar,
205 fprintf('WARNING: Meta file (%s) is different (%i) from 1rst one:\n', ...
206 allfiles(j).name,ii);
207 fprintf(' it=%i :MM:%s\n',itrs(1),MM);
208 fprintf(' it=%i :M :%s\n\n',itrs(iter),M);
209 elseif ind1 > 0,
210 if ii1 > 0,
211 Mj=M(ii1:ii2); ii=findstr(Mj,'['); Mj=Mj(1+ii:end);
212 % add it-number from Mj to M2 (if different):
213 if isempty(findstr(M2,Mj)), M2=[deblank(M2(1:end-1)),Mj]; end
214 end
215 if jj1 > 0,
216 Mj=M(jj1:jj2); ii=findstr(Mj,'['); Mj=Mj(1+ii:end);
217 % add time interval from Mj to M4 (if different):
218 if isempty(findstr(M4,Mj)), M4=[deblank(M4(1:end-1)),' ;',Mj]; end
219 end
220 end
221 end
222 % save modifications:
223 if ind1>0 & j==size(allfiles,1) & iter==size(itrs,2),
224 if ii1 < jj1, MM=[MM(1:ind1-1),M2,M3,M4,M5];
225 else MM=[MM(1:ind1-1),M4,M3,M2,M5]; end
226 end
227 end
228
229 %- put local data file content in global array AA:
230 bdims=N(1,:);
231 r0=N(2,:);
232 rN=N(3,:);
233 ndims=prod(size(bdims));
234 if j==1 & iter==1, AA=zeros([bdims size(itrs,2)]); end
235 if mG(1)==0 & mG(2)==1,
236 if (ndims == 1)
237 AA(r0(1):rN(1),iter)=A;
238 elseif (ndims == 2)
239 AA(r0(1):rN(1),r0(2):rN(2),iter)=A;
240 elseif (ndims == 3)
241 AA(r0(1):rN(1),r0(2):rN(2),r0(3):rN(3),iter)=A;
242 elseif (ndims == 4)
243 AA(r0(1):rN(1),r0(2):rN(2),r0(3):rN(3),r0(4):rN(4),iter)=A;
244 elseif (ndims == 5)
245 AA(r0(1):rN(1),r0(2):rN(2),r0(3):rN(3),r0(4):rN(4),r0(5):rN(5),iter)=A;
246 else
247 error('Dimension of data set is larger than currently coded. Sorry!')
248 end
249 elseif (ndims == 1)
250 AA(r0(1):rN(1),iter)=A;
251 else
252 %- to debug: do simple stransfert (with a loop on 2nd index);
253 % will need to change this later, to improve efficiency:
254 for i=0:rN(2)-r0(2),
255 if (ndims == 2)
256 AA(r0(1)+i*mG(1):rN(1)+i*mG(1),r0(2)+i*mG(2),iter)=A(:,1+i);
257 elseif (ndims == 3)
258 AA(r0(1)+i*mG(1):rN(1)+i*mG(1),r0(2)+i*mG(2), ...
259 r0(3):rN(3),iter)=A(:,1+i,:);
260 elseif (ndims == 4)
261 AA(r0(1)+i*mG(1):rN(1)+i*mG(1),r0(2)+i*mG(2), ...
262 r0(3):rN(3),r0(4):rN(4),iter)=A(:,1+i,:,:);
263 elseif (ndims == 5)
264 AA(r0(1)+i*mG(1):rN(1)+i*mG(1),r0(2)+i*mG(2), ...
265 r0(3):rN(3),r0(4):rN(4),r0(5):rN(5),iter)=A(:,1+i,:,:,:);
266 else
267 error('Dimension of data set is larger than currently coded. Sorry!')
268 end
269 end
270 end
271
272 end % files
273 end % iterations
274
275 %-------------------------------------------------------------------------------
276
277 function [A,N,M,map2glob] = localrdmds(fname,ieee,recnum)
278
279 mname=strrep(fname,' ','');
280 dname=strrep(mname,'.meta','.data');
281
282 %- set default mapping from tile to global file:
283 map2glob=[0 1];
284
285 % Read and interpret Meta file
286 fid = fopen(mname,'r');
287 if (fid == -1)
288 error(['File' mname ' could not be opened'])
289 end
290
291 % Scan each line of the Meta file
292 allstr=' ';
293 keepgoing = 1;
294 while keepgoing > 0,
295 line = fgetl(fid);
296 if (line == -1)
297 keepgoing=-1;
298 else
299 % Strip out "(PID.TID *.*)" by finding first ")"
300 %old ind=findstr([line ')'],')'); line=line(ind(1)+1:end);
301 ind=findstr(line,')');
302 if size(ind) ~= 0
303 line=line(ind(1)+1:end);
304 end
305 % Remove comments of form //
306 line=[line,' //']; ind=findstr(line,'//'); line=line(1:ind(1)-1);
307 % Add to total string (without starting & ending blanks)
308 while line(1:1) == ' ', line=line(2:end); end
309 if strncmp(line,'map2glob',8), eval(line);
310 else allstr=[allstr,deblank(line),' '];
311 end
312 end
313 end
314
315 % Close meta file
316 fclose(fid);
317
318 % Strip out comments of form /* ... */
319 ind1=findstr(allstr,'/*'); ind2=findstr(allstr,'*/');
320 if size(ind1) ~= size(ind2)
321 error('The /* ... */ comments are not properly paired')
322 end
323 while size(ind1,2) > 0
324 allstr=[deblank(allstr(1:ind1(1)-1)) allstr(ind2(1)+2:end)];
325 %allstr=[allstr(1:ind1(1)-1) allstr(ind2(1)+3:end)];
326 ind1=findstr(allstr,'/*'); ind2=findstr(allstr,'*/');
327 end
328
329 % This is a kludge to catch whether the meta-file is of the
330 % old or new type. nrecords does not exist in the old type.
331 nrecords = NaN;
332
333 %- store the full string for output:
334 M=strrep(allstr,'format','dataprec');
335
336 % Everything in lower case
337 allstr=lower(allstr);
338
339 % Fix the unfortunate choice of 'format'
340 allstr=strrep(allstr,'format','dataprec');
341
342 % Evaluate meta information
343 eval(allstr);
344
345 N=reshape( dimlist , 3 , prod(size(dimlist))/3 );
346 rep=[' dimList = [ ',sprintf('%i ',N(1,:)),']'];
347 if ~isnan(nrecords) & nrecords > 1 & isempty(recnum)
348 N=[N,[nrecords 1 nrecords]'];
349 elseif ~isempty(recnum) & recnum>nrecords
350 error('Requested record number is higher than the number of available records')
351 end
352
353 %- make "dimList" shorter (& fit output array size) in output "M":
354 pat=' dimList = \[(\s*\d+\,?)*\s*\]';
355 M=regexprep(M,pat,rep);
356 % and remove double space within sq.brakets:
357 ind1=findstr(M,'['); ind2=findstr(M,']');
358 if length(ind1) == length(ind2),
359 for i=length(ind1):-1:1, if ind1(i) < ind2(i),
360 M=[M(1:ind1(i)),regexprep(M(ind1(i)+1:ind2(i)-1),'(\s+)',' '),M(ind2(i):end)];
361 end; end
362 else error('The [ ... ] brakets are not properly paired')
363 end
364
365 if isempty(recnum)
366 recnum=1;
367 end
368
369 if isnan(nrecords)
370 % This is the old 'meta' method that used sequential access
371
372 A=allstr;
373 % Open data file
374 fid=fopen(dname,'r',ieee);
375
376 % Read record size in bytes
377 recsz=fread(fid,1,'uint32');
378 ldims=N(3,:)-N(2,:)+1;
379 numels=prod(ldims);
380
381 rat=recsz/numels;
382 if rat == 4
383 A=fread(fid,numels,'real*4');
384 elseif rat == 8
385 A=fread(fid,numels,'real*8');
386 else
387 sprintf(' Implied size in meta-file = %d', numels )
388 sprintf(' Record size in data-file = %d', recsz )
389 error('Ratio between record size and size in meta-file inconsistent')
390 end
391
392 erecsz=fread(fid,1,'uint32');
393 if erecsz ~= recsz
394 sprintf('WARNING: Record sizes at beginning and end of file are inconsistent')
395 end
396
397 fclose(fid);
398
399 A=reshape(A,ldims);
400
401 else
402 % This is the new MDS format that uses direct access
403
404 ldims=N(3,:)-N(2,:)+1;
405 for r=1:size(recnum(:),1);
406 if dataprec == 'float32'
407 A(:,r)=myrdda(dname,ldims,recnum(r),'real*4',ieee);
408 elseif dataprec == 'float64'
409 A(:,r)=myrdda(dname,ldims,recnum(r),'real*8',ieee);
410 else
411 error(['Unrecognized format in meta-file = ' format]);
412 end
413 end
414
415 A=reshape(A,[ldims size(recnum(:),1)]);
416 if size(recnum(:),1)>1
417 N(1,end+1)=size(recnum(:),1);
418 N(2,end)=1;
419 N(3,end)=size(recnum(:),1);
420 end
421
422 end
423
424 %-------------------------------------------------------------------------------
425
426 % result = RDDA( file, dim, irec [options] )
427 %
428 % This routine reads the irec'th record of shape 'dim' from the
429 % direct-access binary file (float or double precision) 'file'.
430 %
431 % Required arguments:
432 %
433 % file - string - name of file to read from
434 % dim - vector - dimensions of the file records and the resulting array
435 % irec - integer - record number in file in which to write data
436 %
437 % Optional arguments (must appear after the required arguments):
438 % prec - string - precision of storage in file. Default = 'real*8'.
439 % ieee - string - IEEE bit-wise representation in file. Default = 'b'.
440 %
441 % 'prec' may take the values:
442 % 'real*4' - floating point, 32 bits.
443 % 'real*8' - floating point, 64 bits - the efault.
444 %
445 % 'ieee' may take values:
446 % 'ieee-be' or 'b' - IEEE floating point with big-endian
447 % byte ordering - the default
448 % 'ieee-le' or 'l' - IEEE floating point with little-endian
449 % byte ordering
450 % 'native' or 'n' - local machine format
451 % 'cray' or 'c' - Cray floating point with big-endian
452 % byte ordering
453 % 'ieee-le.l64' or 'a' - IEEE floating point with little-endian
454 % byte ordering and 64 bit long data type
455 % 'ieee-be.l64' or 's' - IEEE floating point with big-endian byte
456 % ordering and 64 bit long data type.
457 %
458 % eg. T=rdda('t.data',[64 64 32],1);
459 % T=rdda('t.data',[256],4,'real*4');
460 % T=rdda('t.data',[128 64],2,'real*4','b');
461 function [arr] = myrdda(file,N,k,varargin)
462
463 % Defaults
464 WORDLENGTH=8;
465 rtype='real*8';
466 ieee='b';
467
468 % Check optional arguments
469 args=char(varargin);
470 while (size(args,1) > 0)
471 if deblank(args(1,:)) == 'real*4'
472 WORDLENGTH=4;
473 rtype='real*4';
474 elseif deblank(args(1,:)) == 'real*8'
475 WORDLENGTH=8;
476 rtype='real*8';
477 elseif deblank(args(1,:)) == 'n' | deblank(args(1,:)) == 'native'
478 ieee='n';
479 elseif deblank(args(1,:)) == 'l' | deblank(args(1,:)) == 'ieee-le'
480 ieee='l';
481 elseif deblank(args(1,:)) == 'b' | deblank(args(1,:)) == 'ieee-be'
482 ieee='b';
483 elseif deblank(args(1,:)) == 'c' | deblank(args(1,:)) == 'cray'
484 ieee='c';
485 elseif deblank(args(1,:)) == 'a' | deblank(args(1,:)) == 'ieee-le.l64'
486 ieee='a';
487 elseif deblank(args(1,:)) == 's' | deblank(args(1,:)) == 'ieee-be.l64'
488 ieee='s';
489 else
490 error(['Optional argument ' args(1,:) ' is unknown'])
491 end
492 args=args(2:end,:);
493 end
494
495 nnn=prod(N);
496
497 [fid mess]=fopen(file,'r',ieee);
498 if fid == -1
499 error('Error while opening file:\n%s',mess)
500 end
501 st=fseek(fid,nnn*(k-1)*WORDLENGTH,'bof');
502 if st ~= 0
503 mess=ferror(fid);
504 error('There was an error while positioning the file pointer:\n%s',mess)
505 end
506 [arr count]=fread(fid,nnn,rtype);
507 if count ~= nnn
508 error('Not enough data was available to be read: off EOF?')
509 end
510 st=fclose(fid);
511 %arr=reshape(arr,N);
512
513 %
514 function [itrs] = scanforfiles(fname)
515
516 itrs=[];
517 allfiles=dir([fname '.*.001.001.meta']);
518 if isempty(allfiles)
519 allfiles=dir([fname '.*.meta']);
520 ioff=0;
521 else
522 ioff=8;
523 end
524 for k=1:size(allfiles,1);
525 hh=allfiles(k).name;
526 itrs(k)=str2num( hh(end-14-ioff:end-5-ioff) );
527 end
528 itrs=sort(itrs);

  ViewVC Help
Powered by ViewVC 1.1.22