utils/matlab/rdmds.m

function [AA] = rdmds(fnamearg,varargin)
% Read MITgcmUV Meta/Data files
%
% A = RDMDS(FNAME)
% A = RDMDS(FNAME,ITER)
% A = RDMDS(FNAME,[ITER1 ITER2 ...])
%
% A = RDMDS(FNAME) reads data described by meta/data file format.
% FNAME is a string containing the "head" of the file names.
%
% eg. To load the meta-data files
%     T.0000002880.000.000.meta, T.0000002880.000.000.data
%     T.0000002880.001.000.meta, T.0000002880.001.000.data
%     T.0000002880.002.000.meta, T.0000002880.002.000.data
%     T.0000002880.003.000.meta, T.0000002880.003.000.data
% use
%    >> A=rdmds('T.0000002880');

% A = RDMDS(FNAME,ITER) reads data described by meta/data file format.
% FNAME is a string containing the "head" of the file name excluding the
% 10-digit iterartion number.
% ITER is a vector of positive integers that will expand to the 10-digit
% number in the file name.
%
% eg. To repeat above operation
%    >> A=rdmds('T',2880);
% eg. To read multiple time steps
%    >> A=rdmds('T',[0 1440 2880]);
% Note: this form can not read files with no iteration count in file name.
%
% A = RDMDS(FNAME,MACHINEFORMAT)
% A = RDMDS(FNAME,ITER,MACHINEFORMAT) allows the machine format to be
% specified which MACHINEFORMAT is on of the following strings:
%
%   'native'      or 'n' - local machine format - the default
%   'ieee-le'     or 'l' - IEEE floating point with little-endian
%                          byte ordering
%   'ieee-be'     or 'b' - IEEE floating point with big-endian
%                          byte ordering
%   'vaxd'        or 'd' - VAX D floating point and VAX ordering
%   'vaxg'        or 'g' - VAX G floating point and VAX ordering
%   'cray'        or 'c' - Cray floating point with big-endian
%                          byte ordering
%   'ieee-le.l64' or 'a' - IEEE floating point with little-endian
%                          byte ordering and 64 bit long data type
%   'ieee-be.l64' or 's' - IEEE floating point with big-endian byte
%                          ordering and 64 bit long data type.
%
% $Header: /u/gcmpack/models/MITgcmUV/utils/matlab/rdmds.m,v 1.2 2001/05/29 14:01:41 adcroft Exp $

% Default options
ieee='b';
fname=fnamearg;
iters=-1;

% Check optional arguments
for ind=1:size(varargin,2);
 arg=varargin{ind};
 if ischar(arg)
  if strcmp(arg,'n') | strcmp(arg,'native')
   ieee='n';
  elseif strcmp(arg,'l') | strcmp(arg,'ieee-le')
   ieee='l';
  elseif strcmp(arg,'b') | strcmp(arg,'ieee-be')
   ieee='b';
  elseif strcmp(arg,'c') | strcmp(arg,'cray')
   ieee='c';
  elseif strcmp(arg,'a') | strcmp(arg,'ieee-le.l64')
   ieee='a';
  elseif strcmp(arg,'s') | strcmp(arg,'ieee-be.l64')
   ieee='s';
  else
   error(['Optional argument ' arg ' is unknown'])
  end
 else
  if arg>=9999999999
   error(sprintf('Argument %i > 9999999999',arg))
  end
  if prod(arg>=0) & prod(round(arg)==arg)
   iters=arg;
  else
   error(sprintf('Argument %i must be a positive integer',arg))
  end
 end
end

% Loop over each iteration
for iter=1:size(iters,2);
if iters(iter)>=0
 fname=sprintf('%s.%10.10i',fnamearg,iters(iter))
end

% Match name of all meta-files
allfiles=dir( sprintf('%s.*.meta',fname) );

if size(allfiles,1)==0
 disp(sprintf('No files match the search: %s.*.meta',fname));
 error('No files found.')
end

% Loop through allfiles
for j=1:size(allfiles,1);

% Read meta- and data-file
[A,N] = localrdmds(allfiles(j).name,ieee);

bdims=N(1,:);
r0=N(2,:);
rN=N(3,:);
ndims=prod(size(bdims));
if     (ndims == 1)
 AA(r0(1):rN(1),iter)=A;
elseif (ndims == 2)
 AA(r0(1):rN(1),r0(2):rN(2),iter)=A;
elseif (ndims == 3)
 AA(r0(1):rN(1),r0(2):rN(2),r0(3):rN(3),iter)=A;
elseif (ndims == 4)
 AA(r0(1):rN(1),r0(2):rN(2),r0(3):rN(3),r0(4):rN(4),iter)=A;
elseif (ndims == 4)
 AA(r0(1):rN(1),r0(2):rN(2),r0(3):rN(3),r0(4):rN(4),r0(5):rN(5),iter)=A;
else
 error('Dimension of data set is larger than currently coded. Sorry!')
end

end % files
end % iterations

%-------------------------------------------------------------------------------

function [A,N] = localrdmds(fname,ieee)

mname=strrep(fname,' ','');
dname=strrep(mname,'.meta','.data');

% Read and interpret Meta file
fid = fopen(mname,'r');
if (fid == -1)
 error(['File' mname ' could not be opened'])
end

% Scan each line of the Meta file
allstr=' ';
keepgoing = 1;
while keepgoing > 0,
 line = fgetl(fid);
 if (line == -1)
  keepgoing=-1;
 else
% Strip out "(PID.TID *.*)" by finding first ")"
%old  ind=findstr([line ')'],')'); line=line(ind(1)+1:end);
  ind=findstr(line,')');
  if size(ind) ~= 0
    line=line(ind(1)+1:end);
  end
% Remove comments of form //
  line=[line ' //']; ind=findstr(line,'//'); line=line(1:ind(1)-1);
% Add to total string
  allstr=[allstr line];
 end
end

% Close meta file
fclose(fid);

% Strip out comments of form /* ... */
ind1=findstr(allstr,'/*'); ind2=findstr(allstr,'*/');
if size(ind1) ~= size(ind2)
 error('The /* ... */ comments are not properly paired')
end
while size(ind1,2) > 0
 allstr=[allstr(1:ind1(1)-1) allstr(ind2(1)+3:end)];
 ind1=findstr(allstr,'/*'); ind2=findstr(allstr,'*/');
end

% This is a kludge to catch whether the meta-file is of the
% old or new type. nrecords does not exist in the old type.
nrecords = -987;

% Everything in lower case
allstr=lower(allstr);

% Fix the unfortunate choice of 'format'
allstr=strrep(allstr,'format','dataprec');

% Evaluate meta information
eval(allstr);

N=reshape( dimlist , 3 , prod(size(dimlist))/3 );

if nrecords == -987
% This is the old 'meta' method that used sequential access

A=allstr;
% Open data file
fid=fopen(dname,'r',ieee);

% Read record size in bytes
recsz=fread(fid,1,'uint32');
ldims=N(3,:)-N(2,:)+1;
numels=prod(ldims);

rat=recsz/numels;
if rat == 4
 A=fread(fid,numels,'real*4');
elseif rat == 8
 A=fread(fid,numels,'real*8');
else
 sprintf(' Implied size in meta-file = %d', numels )
 sprintf(' Record size in data-file = %d', recsz )
 error('Ratio between record size and size in meta-file inconsistent')
end

erecsz=fread(fid,1,'uint32');
if erecsz ~= recsz
 sprintf('WARNING: Record sizes at beginning and end of file are inconsistent')
end

fclose(fid);

A=reshape(A,ldims);

else
% This is the new MDS format that uses direct access

 ldims=N(3,:)-N(2,:)+1;
 if dataprec == 'float32'
  A=myrdda(dname,ldims,1,'real*4',ieee);
 elseif dataprec == 'float64'
  A=myrdda(dname,ldims,1,'real*8',ieee);
 else
  error(['Unrecognized format in meta-file = ' format]);
 end

end

%-------------------------------------------------------------------------------

% result = RDDA( file, dim, irec [options] )
%
% This routine reads the irec'th record of shape 'dim' from the
% direct-access binary file (float or double precision) 'file'.
%
% Required arguments:
%
%   file  - string  - name of file to read from
%   dim   - vector  - dimensions of the file records and the resulting array
%   irec  - integer - record number in file in which to write data
%
% Optional arguments (must appear after the required arguments):
%   prec  - string  - precision of storage in file. Default = 'real*8'.
%   ieee  - string  - IEEE bit-wise representation in file. Default = 'b'.
%
% 'prec' may take the values:
%       'real*4' - floating point, 32 bits.
%       'real*8' - floating point, 64 bits - the efault.
%
% 'ieee' may take values:
%    'ieee-be'     or 'b' - IEEE floating point with big-endian
%                           byte ordering - the default
%    'ieee-le'     or 'l' - IEEE floating point with little-endian
%                           byte ordering
%    'native'      or 'n' - local machine format
%    'cray'        or 'c' - Cray floating point with big-endian
%                           byte ordering
%    'ieee-le.l64' or 'a' - IEEE floating point with little-endian
%                           byte ordering and 64 bit long data type
%    'ieee-be.l64' or 's' - IEEE floating point with big-endian byte
%                           ordering and 64 bit long data type.
%
% eg.   T=rdda('t.data',[64 64 32],1);
%       T=rdda('t.data',[256],4,'real*4');
%       T=rdda('t.data',[128 64],2,'real*4','b');
function [arr] = myrdda(file,N,k,varargin)

% Defaults
WORDLENGTH=8;
rtype='real*8';
ieee='b';

% Check optional arguments
args=char(varargin);
while (size(args,1) > 0)
 if deblank(args(1,:)) == 'real*4'
  WORDLENGTH=4;
  rtype='real*4';
 elseif deblank(args(1,:)) == 'real*8'
  WORDLENGTH=8;
  rtype='real*8';
 elseif deblank(args(1,:)) == 'n' | deblank(args(1,:)) == 'native'
  ieee='n';
 elseif deblank(args(1,:)) == 'l' | deblank(args(1,:)) == 'ieee-le'
  ieee='l';
 elseif deblank(args(1,:)) == 'b' | deblank(args(1,:)) == 'ieee-be'
  ieee='b';
 elseif deblank(args(1,:)) == 'c' | deblank(args(1,:)) == 'cray'
  ieee='c';
 elseif deblank(args(1,:)) == 'a' | deblank(args(1,:)) == 'ieee-le.l64'
  ieee='a';
 elseif deblank(args(1,:)) == 's' | deblank(args(1,:)) == 'ieee-be.l64'
  ieee='s';
 else
  error(['Optional argument ' args(1,:) ' is unknown'])
 end
 args=args(2:end,:);
end

nnn=prod(N);

[fid mess]=fopen(file,'r',ieee);
if fid == -1
 error('Error while opening file:\n%s',mess)
end
st=fseek(fid,nnn*(k-1)*WORDLENGTH,'bof');
if st ~= 0
 mess=ferror(fid);
 error('There was an error while positioning the file pointer:\n%s',mess)
end
[arr count]=fread(fid,nnn,rtype);
if count ~= nnn
 error('Not enough data was available to be read: off EOF?')
end
st=fclose(fid);
arr=reshape(arr,N);
1	adcroft	1.3	function [AA] = rdmds(fnamearg,varargin)
2			% Read MITgcmUV Meta/Data files
3	adcroft	1.1	%
4	adcroft	1.3	% A = RDMDS(FNAME)
5			% A = RDMDS(FNAME,ITER)
6			% A = RDMDS(FNAME,[ITER1 ITER2 ...])
7	adcroft	1.1	%
8			% A = RDMDS(FNAME) reads data described by meta/data file format.
9			% FNAME is a string containing the "head" of the file names.
10			%
11			% eg. To load the meta-data files
12			% T.0000002880.000.000.meta, T.0000002880.000.000.data
13			% T.0000002880.001.000.meta, T.0000002880.001.000.data
14			% T.0000002880.002.000.meta, T.0000002880.002.000.data
15			% T.0000002880.003.000.meta, T.0000002880.003.000.data
16			% use
17			% >> A=rdmds('T.0000002880');
18	adcroft	1.3
19			% A = RDMDS(FNAME,ITER) reads data described by meta/data file format.
20			% FNAME is a string containing the "head" of the file name excluding the
21			% 10-digit iterartion number.
22			% ITER is a vector of positive integers that will expand to the 10-digit
23			% number in the file name.
24			%
25			% eg. To repeat above operation
26			% >> A=rdmds('T',2880);
27			% eg. To read multiple time steps
28			% >> A=rdmds('T',[0 1440 2880]);
29			% Note: this form can not read files with no iteration count in file name.
30			%
31			% A = RDMDS(FNAME,MACHINEFORMAT)
32			% A = RDMDS(FNAME,ITER,MACHINEFORMAT) allows the machine format to be
33			% specified which MACHINEFORMAT is on of the following strings:
34	adcroft	1.1	%
35			% 'native' or 'n' - local machine format - the default
36			% 'ieee-le' or 'l' - IEEE floating point with little-endian
37			% byte ordering
38			% 'ieee-be' or 'b' - IEEE floating point with big-endian
39			% byte ordering
40			% 'vaxd' or 'd' - VAX D floating point and VAX ordering
41			% 'vaxg' or 'g' - VAX G floating point and VAX ordering
42			% 'cray' or 'c' - Cray floating point with big-endian
43			% byte ordering
44			% 'ieee-le.l64' or 'a' - IEEE floating point with little-endian
45			% byte ordering and 64 bit long data type
46			% 'ieee-be.l64' or 's' - IEEE floating point with big-endian byte
47			% ordering and 64 bit long data type.
48			%
49	adcroft	1.3	% $Header: /u/gcmpack/models/MITgcmUV/utils/matlab/rdmds.m,v 1.2 2001/05/29 14:01:41 adcroft Exp $
50	adcroft	1.1
51			% Default options
52			ieee='b';
53	adcroft	1.3	fname=fnamearg;
54			iters=-1;
55	adcroft	1.1
56			% Check optional arguments
57	adcroft	1.3	for ind=1:size(varargin,2);
58			arg=varargin{ind};
59			if ischar(arg)
60			if strcmp(arg,'n') \| strcmp(arg,'native')
61			ieee='n';
62			elseif strcmp(arg,'l') \| strcmp(arg,'ieee-le')
63			ieee='l';
64			elseif strcmp(arg,'b') \| strcmp(arg,'ieee-be')
65			ieee='b';
66			elseif strcmp(arg,'c') \| strcmp(arg,'cray')
67			ieee='c';
68			elseif strcmp(arg,'a') \| strcmp(arg,'ieee-le.l64')
69			ieee='a';
70			elseif strcmp(arg,'s') \| strcmp(arg,'ieee-be.l64')
71			ieee='s';
72			else
73			error(['Optional argument ' arg ' is unknown'])
74			end
75	adcroft	1.1	else
76	adcroft	1.3	if arg>=9999999999
77			error(sprintf('Argument %i > 9999999999',arg))
78			end
79			if prod(arg>=0) & prod(round(arg)==arg)
80			iters=arg;
81			else
82			error(sprintf('Argument %i must be a positive integer',arg))
83			end
84	adcroft	1.1	end
85	adcroft	1.3	end
86
87			% Loop over each iteration
88			for iter=1:size(iters,2);
89			if iters(iter)>=0
90			fname=sprintf('%s.%10.10i',fnamearg,iters(iter))
91	adcroft	1.1	end
92
93			% Match name of all meta-files
94	adcroft	1.3	allfiles=dir( sprintf('%s.*.meta',fname) );
95	adcroft	1.1
96	adcroft	1.3	if size(allfiles,1)==0
97			disp(sprintf('No files match the search: %s.*.meta',fname));
98			error('No files found.')
99			end
100	adcroft	1.1
101			% Loop through allfiles
102	adcroft	1.3	for j=1:size(allfiles,1);
103	adcroft	1.1
104			% Read meta- and data-file
105	adcroft	1.3	[A,N] = localrdmds(allfiles(j).name,ieee);
106	adcroft	1.1
107			bdims=N(1,:);
108			r0=N(2,:);
109			rN=N(3,:);
110			ndims=prod(size(bdims));
111			if (ndims == 1)
112	adcroft	1.3	AA(r0(1):rN(1),iter)=A;
113	adcroft	1.1	elseif (ndims == 2)
114	adcroft	1.3	AA(r0(1):rN(1),r0(2):rN(2),iter)=A;
115	adcroft	1.1	elseif (ndims == 3)
116	adcroft	1.3	AA(r0(1):rN(1),r0(2):rN(2),r0(3):rN(3),iter)=A;
117	adcroft	1.1	elseif (ndims == 4)
118	adcroft	1.3	AA(r0(1):rN(1),r0(2):rN(2),r0(3):rN(3),r0(4):rN(4),iter)=A;
119			elseif (ndims == 4)
120			AA(r0(1):rN(1),r0(2):rN(2),r0(3):rN(3),r0(4):rN(4),r0(5):rN(5),iter)=A;
121	adcroft	1.1	else
122			error('Dimension of data set is larger than currently coded. Sorry!')
123			end
124
125	adcroft	1.3	end % files
126			end % iterations
127	adcroft	1.1
128			%-------------------------------------------------------------------------------
129
130			function [A,N] = localrdmds(fname,ieee)
131
132			mname=strrep(fname,' ','');
133			dname=strrep(mname,'.meta','.data');
134
135			% Read and interpret Meta file
136			fid = fopen(mname,'r');
137			if (fid == -1)
138			error(['File' mname ' could not be opened'])
139			end
140
141			% Scan each line of the Meta file
142			allstr=' ';
143			keepgoing = 1;
144			while keepgoing > 0,
145			line = fgetl(fid);
146			if (line == -1)
147			keepgoing=-1;
148			else
149			% Strip out "(PID.TID .)" by finding first ")"
150			%old ind=findstr([line ')'],')'); line=line(ind(1)+1:end);
151			ind=findstr(line,')');
152			if size(ind) ~= 0
153			line=line(ind(1)+1:end);
154			end
155			% Remove comments of form //
156			line=[line ' //']; ind=findstr(line,'//'); line=line(1:ind(1)-1);
157			% Add to total string
158			allstr=[allstr line];
159			end
160			end
161
162			% Close meta file
163			fclose(fid);
164
165			% Strip out comments of form /* ... */
166			ind1=findstr(allstr,'/'); ind2=findstr(allstr,'/');
167			if size(ind1) ~= size(ind2)
168			error('The /* ... */ comments are not properly paired')
169			end
170			while size(ind1,2) > 0
171			allstr=[allstr(1:ind1(1)-1) allstr(ind2(1)+3:end)];
172			ind1=findstr(allstr,'/'); ind2=findstr(allstr,'/');
173			end
174
175			% This is a kludge to catch whether the meta-file is of the
176			% old or new type. nrecords does not exist in the old type.
177			nrecords = -987;
178
179			% Everything in lower case
180			allstr=lower(allstr);
181
182			% Fix the unfortunate choice of 'format'
183			allstr=strrep(allstr,'format','dataprec');
184
185			% Evaluate meta information
186			eval(allstr);
187
188			N=reshape( dimlist , 3 , prod(size(dimlist))/3 );
189
190			if nrecords == -987
191			% This is the old 'meta' method that used sequential access
192
193			A=allstr;
194			% Open data file
195			fid=fopen(dname,'r',ieee);
196
197			% Read record size in bytes
198			recsz=fread(fid,1,'uint32');
199			ldims=N(3,:)-N(2,:)+1;
200			numels=prod(ldims);
201
202			rat=recsz/numels;
203			if rat == 4
204			A=fread(fid,numels,'real*4');
205			elseif rat == 8
206			A=fread(fid,numels,'real*8');
207			else
208			sprintf(' Implied size in meta-file = %d', numels )
209			sprintf(' Record size in data-file = %d', recsz )
210			error('Ratio between record size and size in meta-file inconsistent')
211			end
212
213			erecsz=fread(fid,1,'uint32');
214			if erecsz ~= recsz
215			sprintf('WARNING: Record sizes at beginning and end of file are inconsistent')
216			end
217
218			fclose(fid);
219
220			A=reshape(A,ldims);
221
222			else
223			% This is the new MDS format that uses direct access
224
225			ldims=N(3,:)-N(2,:)+1;
226			if dataprec == 'float32'
227			A=myrdda(dname,ldims,1,'real*4',ieee);
228			elseif dataprec == 'float64'
229			A=myrdda(dname,ldims,1,'real*8',ieee);
230			else
231			error(['Unrecognized format in meta-file = ' format]);
232			end
233
234			end
235
236			%-------------------------------------------------------------------------------
237
238			% result = RDDA( file, dim, irec [options] )
239			%
240			% This routine reads the irec'th record of shape 'dim' from the
241			% direct-access binary file (float or double precision) 'file'.
242			%
243			% Required arguments:
244			%
245			% file - string - name of file to read from
246			% dim - vector - dimensions of the file records and the resulting array
247			% irec - integer - record number in file in which to write data
248			%
249			% Optional arguments (must appear after the required arguments):
250			% prec - string - precision of storage in file. Default = 'real*8'.
251			% ieee - string - IEEE bit-wise representation in file. Default = 'b'.
252			%
253			% 'prec' may take the values:
254			% 'real*4' - floating point, 32 bits.
255			% 'real*8' - floating point, 64 bits - the efault.
256			%
257			% 'ieee' may take values:
258			% 'ieee-be' or 'b' - IEEE floating point with big-endian
259			% byte ordering - the default
260			% 'ieee-le' or 'l' - IEEE floating point with little-endian
261			% byte ordering
262			% 'native' or 'n' - local machine format
263			% 'cray' or 'c' - Cray floating point with big-endian
264			% byte ordering
265			% 'ieee-le.l64' or 'a' - IEEE floating point with little-endian
266			% byte ordering and 64 bit long data type
267			% 'ieee-be.l64' or 's' - IEEE floating point with big-endian byte
268			% ordering and 64 bit long data type.
269			%
270			% eg. T=rdda('t.data',[64 64 32],1);
271			% T=rdda('t.data',[256],4,'real*4');
272			% T=rdda('t.data',[128 64],2,'real*4','b');
273			function [arr] = myrdda(file,N,k,varargin)
274
275			% Defaults
276			WORDLENGTH=8;
277			rtype='real*8';
278			ieee='b';
279
280			% Check optional arguments
281			args=char(varargin);
282			while (size(args,1) > 0)
283			if deblank(args(1,:)) == 'real*4'
284			WORDLENGTH=4;
285			rtype='real*4';
286			elseif deblank(args(1,:)) == 'real*8'
287			WORDLENGTH=8;
288			rtype='real*8';
289			elseif deblank(args(1,:)) == 'n' \| deblank(args(1,:)) == 'native'
290			ieee='n';
291			elseif deblank(args(1,:)) == 'l' \| deblank(args(1,:)) == 'ieee-le'
292			ieee='l';
293			elseif deblank(args(1,:)) == 'b' \| deblank(args(1,:)) == 'ieee-be'
294			ieee='b';
295			elseif deblank(args(1,:)) == 'c' \| deblank(args(1,:)) == 'cray'
296			ieee='c';
297			elseif deblank(args(1,:)) == 'a' \| deblank(args(1,:)) == 'ieee-le.l64'
298			ieee='a';
299			elseif deblank(args(1,:)) == 's' \| deblank(args(1,:)) == 'ieee-be.l64'
300			ieee='s';
301			else
302			error(['Optional argument ' args(1,:) ' is unknown'])
303			end
304			args=args(2:end,:);
305			end
306
307			nnn=prod(N);
308
309			[fid mess]=fopen(file,'r',ieee);
310			if fid == -1
311			error('Error while opening file:\n%s',mess)
312			end
313			st=fseek(fid,nnn(k-1)WORDLENGTH,'bof');
314			if st ~= 0
315			mess=ferror(fid);
316			error('There was an error while positioning the file pointer:\n%s',mess)
317			end
318			[arr count]=fread(fid,nnn,rtype);
319			if count ~= nnn
320			error('Not enough data was available to be read: off EOF?')
321			end
322			st=fclose(fid);
323			arr=reshape(arr,N);