matlab_class/gcmfaces_IO/process2nctiles.m

function []=process2nctiles(dirModel,fileModel,fldModel,tileSize);
%process2nctiles(dirModel);
%object : convert MITgcm binary output to netcdf files (tiled) 
%inputs : dirModel is the MITgcm run directory
%           It is expected to contain binaries in 
%           'diags/STATE/', 'diags/TRSP/', etc. as well
%           as the 'available_diagnostics.log' text file.
%         fileModel the file name base e.g. 'state_2d_set1'
%           By default : all variables in e.g. 'state_2d_set1*' 
%           files will be processed, and writen individually to
%           nctiles (tiled netcdf) that will be located in 'nctiles/'
%         fldModel (by default []) can be specified (as e.g. 'ETAN')
%            when fldModel is empty, all fields are processed
%         tileSize (optional) is e.g. [90 90] (by default tiles=faces) 
%output : (netcdf files)

gcmfaces_global;

%listFiles={'state_2d_set1','state_2d_set2','state_3d_set1','state_3d_set2'};
%listFiles={'trsp_3d_set1','trsp_3d_set2','trsp_3d_set3'};
%for ff=1:length(listFiles); process2nctiles('iter12/',listFiles{ff},[],[90 90]); end;

%directory names
listDirs={'STATE/','TRSP/'};
filAvailDiag=[dirModel 'available_diagnostics.log'];
filReadme=[dirModel 'README'];
dirOut=[dirModel 'nctiles/'];
if ~isdir(dirOut); mkdir(dirOut); end;

%seacrh in subdirectories
subDir=[];
for ff=1:length(listDirs);
tmp1=dir([dirModel 'diags/' listDirs{ff} fileModel '*']);
if ~isempty(tmp1); subDir=listDirs{ff}; end;
end;

if isempty(subDir); 
  error(['file ' fileModel ' was not found']);
else;
  dirIn=[dirModel 'diags/' subDir];
  nn=length(dir([dirIn fileModel '*data']));
  fprintf('%s (%d files) was found in \n %s \n',fileModel,nn,dirIn);
end;

%set list of variables to process
if ~isempty(fldModel);
   if ischar(fldModel); listFlds={fldModel};
   else; listFlds=fldModel;
   end;
else;
   meta=read_meta([dirIn fileModel '*']);
   listFlds=meta.fldList;
end;

%determine map of tile indices (by default tiles=faces)
if isempty(whos('tileSize'));
  tileNo=mygrid.XC; 
  for ff=1:mygrid.nFaces; tileNo{ff}(:)=ff; end;
else;
  tileNo=gcmfaces_loc_tile(tileSize(1),tileSize(2));
end;

%now do the actual processing
for vv=1:length(listFlds);
nameDiag=deblank(listFlds{vv}) 

%get meta information
meta=read_meta([dirIn fileModel '*']);
irec=find(strcmp(deblank(meta.fldList),nameDiag));
if length(irec)~=1; error('field not in file\n'); end;

%read time series
myDiag=rdmds2gcmfaces([dirIn fileModel '*'],NaN,'rec',irec);

%set ancilliary time variable
nn=length(size(myDiag{1}));
nn=size(myDiag{1},nn);
%tim=[1:nn];
tim=[1992*ones(nn,1) [1:nn]' 15*ones(nn,1)];
tim=datenum(tim)-datenum([1992 1 0]);
timUnits='days since 1992-1-1 0:0:0';

%get time step axis
[listTimes]=diags_list_times({[dirModel 'diags/STATE/']},{'state_2d_set1'});

%get units and long name from available_diagnostics.log
[avail_diag]=read_avail_diag(filAvailDiag,nameDiag);

%get description of estimate from README
[rdm]=read_readme(filReadme);
disp(rdm');

%convert to MITgcm format (90x1170 array)
myFile=[dirOut nameDiag];%first instance is for subdirectory name
if ~isdir(myFile); mkdir(myFile); end;
myFile=[myFile '/' nameDiag];%second instance is for file name base

%get grid params
[grid_diag]=set_grid_diag(avail_diag);

%apply mask, and convert to land mask
msk=grid_diag.msk;
if length(size(myDiag{1}))==3;
  msk=repmat(msk,[1 1 size(myDiag{1},3)]);
end;
for kk=1:size(myDiag{1},4);
  myDiag(:,:,:,kk)=myDiag(:,:,:,kk).*msk;
end;
clear msk;
%
land=isnan(grid_diag.msk);

%set 'coord' attribute
if avail_diag.nr~=1;
  coord='lon lat dep tim';
else;
  coord='lon lat tim';
end;

%create netcdf file using write2nctiles (works only with old matlab, thus far ...)
doCreate=1; 
dimlist=write2nctiles(myFile,myDiag,doCreate,{'tileNo',tileNo},...
    {'fldName',nameDiag},{'longName',avail_diag.longNameDiag},...
    {'units',avail_diag.units},{'descr',nameDiag},{'coord',coord},{'rdm',rdm});

%determine relevant dimensions
for ff=1:length(dimlist);
  dim.tim{ff}={dimlist{ff}{1}};
  dim.twoD{ff}={dimlist{ff}{end-1:end}};
  if avail_diag.nr~=1;
    dim.threeD{ff}={dimlist{ff}{end-2:end}};
    dim.dep{ff}={dimlist{ff}{end-2}};
  else;
    dim.threeD{ff}=dim.twoD{ff};
    dim.dep{ff}=[];
  end;
end;

%prepare to add fields
doCreate=0;

%now add fields
write2nctiles(myFile,tim,doCreate,{'tileNo',tileNo},...
  {'fldName','tim'},{'longName','time'},...
  {'units',timUnits},{'dimIn',dim.tim});
write2nctiles(myFile,listTimes,doCreate,{'tileNo',tileNo},...
  {'fldName','step'},{'longName','final time step number'},...
  {'units','1'},{'dimIn',dim.tim});
write2nctiles(myFile,grid_diag.lon,doCreate,{'tileNo',tileNo},...
  {'fldName','lon'},{'units','degrees_east'},{'dimIn',dim.twoD});
write2nctiles(myFile,grid_diag.lat,doCreate,{'tileNo',tileNo},...
  {'fldName','lat'},{'units','degrees_north'},{'dimIn',dim.twoD});
write2nctiles(myFile,grid_diag.msk,doCreate,{'tileNo',tileNo},...
  {'fldName','land'},{'units','1'},{'longName','land mask'},{'dimIn',dim.threeD});
write2nctiles(myFile,grid_diag.RAC,doCreate,{'tileNo',tileNo},...
  {'fldName','area'},{'units','m^2'},{'longName','grid cell area'},{'dimIn',dim.twoD});
if isfield(grid_diag,'dep');
    write2nctiles(myFile,grid_diag.dep,doCreate,{'tileNo',tileNo},...
      {'fldName','dep'},{'units','m'},{'dimIn',dim.dep});
    write2nctiles(myFile,grid_diag.dz,doCreate,{'tileNo',tileNo},...
      {'fldName','thic'},{'units','m'},{'dimIn',dim.dep});
end;

end;%for vv=1:length(listFlds);

function [meta]=read_meta(fileName);

%read meta file
tmp1=dir([fileName '*.meta']); tmp1=tmp1(1).name;
tmp2=strfind(fileName,filesep);
if ~isempty(tmp2); tmp2=tmp2(end); else; tmp2=0; end;
tmp1=[fileName(1:tmp2) tmp1]; fid=fopen(tmp1);
while 1;
    tline = fgetl(fid);
    if ~ischar(tline), break, end
    if isempty(whos('tmp3')); tmp3=tline; else; tmp3=[tmp3 ' ' tline]; end;
end
fclose(fid);

%add meta variables to workspace
eval(tmp3);

%reformat to meta structure
meta.dataprec=dataprec;
meta.nDims=nDims;
meta.nFlds=nFlds;
meta.nrecords=nrecords;
meta.fldList=fldList;
meta.dimList=dimList;
if ~isempty(who('timeInterval')); meta.timeInterval=timeInterval; end;
if ~isempty(who('timeStepNumber'));  meta.timeStepNumber=timeStepNumber; end;

%%

function [rdm]=read_readme(filReadme);

gcmfaces_global;

rdm=[];

fid=fopen(filReadme,'rt');
while ~feof(fid);
    nn=length(rdm);
    rdm{nn+1} = fgetl(fid);
end;
fclose(fid);

%%

function [avail_diag]=read_avail_diag(filAvailDiag,nameDiag);

gcmfaces_global;

avail_diag=[];

fid=fopen(filAvailDiag,'rt');
while ~feof(fid);
    tline = fgetl(fid);
    tmp1=8-length(nameDiag); tmp1=repmat(' ',[1 tmp1]);
    tname = ['|' sprintf('%s',nameDiag) tmp1 '|'];
    if ~isempty(strfind(tline,tname));
        %e.g. tline='   235 |SIatmQnt|  1 |       |SM      U1|W/m^2           |Net atmospheric heat flux, >0 decreases theta';
        %
        tmp1=strfind(tline,'|'); tmp1=tmp1(end-1:end);
        avail_diag.units=strtrim(tline(tmp1(1)+1:tmp1(2)-1));
        avail_diag.longNameDiag=tline(tmp1(2)+1:end);
        %
        tmp1=strfind(tline,'|'); tmp1=tmp1(4:5);
        pars=tline(tmp1(1)+1:tmp1(2)-1);
        %
        if strcmp(pars(2),'M'); avail_diag.loc_h='C';
        elseif strcmp(pars(2),'U'); avail_diag.loc_h='W';
        elseif strcmp(pars(2),'V'); avail_diag.loc_h='S';
        end;
        %
        avail_diag.loc_z=pars(9);
        %
        if strcmp(pars(10),'1'); avail_diag.nr=1;
        else; avail_diag.nr=length(mygrid.RC);
        end;
    end;
end;
fclose(fid);

%%

function [grid_diag]=set_grid_diag(avail_diag);

gcmfaces_global;

%switch for non-tracer point values
if strcmp(avail_diag.loc_h,'C');
    grid_diag.lon=mygrid.XC;
    grid_diag.lat=mygrid.YC;
    grid_diag.msk=mygrid.mskC(:,:,1:avail_diag.nr);
elseif strcmp(avail_diag.loc_h,'W');
    grid_diag.lon=mygrid.XG;
    grid_diag.lat=mygrid.YC;
    grid_diag.msk=mygrid.mskW(:,:,1:avail_diag.nr);
elseif strcmp(avail_diag.loc_h,'S');
    grid_diag.lon=mygrid.XC;
    grid_diag.lat=mygrid.YG;
    grid_diag.msk=mygrid.mskS(:,:,1:avail_diag.nr);
end;
grid_diag.RAC=mygrid.RAC;

%vertical grid
if avail_diag.nr~=1;
    if strcmp(avail_diag.loc_z,'M');
        grid_diag.dep=-mygrid.RC;
        grid_diag.dz=mygrid.DRF;
    elseif strcmp(avail_diag.loc_z,'L');
        grid_diag.dep=-mygrid.RF(2:end);
        grid_diag.dz=[mygrid.DRC(2:end) ; 228.25];%quick fix
    else;
        error('unknown vertical grid');
    end;
    grid_diag.dep=reshape(grid_diag.dep,[1 1 avail_diag.nr]);
    grid_diag.dz=reshape(grid_diag.dz,[1 1 avail_diag.nr]);
end;
1	gforget	1.4	function []=process2nctiles(dirModel,fileModel,fldModel,tileSize);
2	gforget	1.1	%process2nctiles(dirModel);
3			%object : convert MITgcm binary output to netcdf files (tiled)
4			%inputs : dirModel is the MITgcm run directory
5			% It is expected to contain binaries in
6			% 'diags/STATE/', 'diags/TRSP/', etc. as well
7			% as the 'available_diagnostics.log' text file.
8			% fileModel the file name base e.g. 'state_2d_set1'
9			% By default : all variables in e.g. 'state_2d_set1*'
10			% files will be processed, and writen individually to
11			% nctiles (tiled netcdf) that will be located in 'nctiles/'
12	gforget	1.4	% fldModel (by default []) can be specified (as e.g. 'ETAN')
13			% when fldModel is empty, all fields are processed
14			% tileSize (optional) is e.g. [90 90] (by default tiles=faces)
15	gforget	1.1	%output : (netcdf files)
16
17			gcmfaces_global;
18
19	gforget	1.5	%listFiles={'state_2d_set1','state_2d_set2','state_3d_set1','state_3d_set2'};
20			%listFiles={'trsp_3d_set1','trsp_3d_set2','trsp_3d_set3'};
21			%for ff=1:length(listFiles); process2nctiles('iter12/',listFiles{ff},[],[90 90]); end;
22
23	gforget	1.1	%directory names
24			listDirs={'STATE/','TRSP/'};
25			filAvailDiag=[dirModel 'available_diagnostics.log'];
26	gforget	1.2	filReadme=[dirModel 'README'];
27	gforget	1.1	dirOut=[dirModel 'nctiles/'];
28			if ~isdir(dirOut); mkdir(dirOut); end;
29
30			%seacrh in subdirectories
31			subDir=[];
32			for ff=1:length(listDirs);
33			tmp1=dir([dirModel 'diags/' listDirs{ff} fileModel '*']);
34			if ~isempty(tmp1); subDir=listDirs{ff}; end;
35			end;
36
37			if isempty(subDir);
38			error(['file ' fileModel ' was not found']);
39			else;
40			dirIn=[dirModel 'diags/' subDir];
41			nn=length(dir([dirIn fileModel '*data']));
42			fprintf('%s (%d files) was found in \n %s \n',fileModel,nn,dirIn);
43			end;
44
45			%set list of variables to process
46	gforget	1.4	if ~isempty(fldModel);
47	gforget	1.1	if ischar(fldModel); listFlds={fldModel};
48			else; listFlds=fldModel;
49			end;
50			else;
51			meta=read_meta([dirIn fileModel '*']);
52			listFlds=meta.fldList;
53			end;
54
55	gforget	1.4	%determine map of tile indices (by default tiles=faces)
56			if isempty(whos('tileSize'));
57			tileNo=mygrid.XC;
58			for ff=1:mygrid.nFaces; tileNo{ff}(:)=ff; end;
59			else;
60			tileNo=gcmfaces_loc_tile(tileSize(1),tileSize(2));
61			end;
62
63	gforget	1.1	%now do the actual processing
64			for vv=1:length(listFlds);
65			nameDiag=deblank(listFlds{vv})
66
67			%get meta information
68			meta=read_meta([dirIn fileModel '*']);
69			irec=find(strcmp(deblank(meta.fldList),nameDiag));
70			if length(irec)~=1; error('field not in file\n'); end;
71
72			%read time series
73			myDiag=rdmds2gcmfaces([dirIn fileModel '*'],NaN,'rec',irec);
74
75	gforget	1.5	%set ancilliary time variable
76	gforget	1.3	nn=length(size(myDiag{1}));
77	gforget	1.5	nn=size(myDiag{1},nn);
78			%tim=[1:nn];
79			tim=[1992ones(nn,1) [1:nn]' 15ones(nn,1)];
80			tim=datenum(tim)-datenum([1992 1 0]);
81			timUnits='days since 1992-1-1 0:0:0';
82	gforget	1.1
83	gforget	1.7	%get time step axis
84			[listTimes]=diags_list_times({[dirModel 'diags/STATE/']},{'state_2d_set1'});
85
86	gforget	1.1	%get units and long name from available_diagnostics.log
87			[avail_diag]=read_avail_diag(filAvailDiag,nameDiag);
88
89	gforget	1.2	%get description of estimate from README
90			[rdm]=read_readme(filReadme);
91			disp(rdm');
92
93	gforget	1.1	%convert to MITgcm format (90x1170 array)
94	gforget	1.4	myFile=[dirOut nameDiag];%first instance is for subdirectory name
95			if ~isdir(myFile); mkdir(myFile); end;
96			myFile=[myFile '/' nameDiag];%second instance is for file name base
97	gforget	1.1
98			%get grid params
99			[grid_diag]=set_grid_diag(avail_diag);
100
101	gforget	1.5	%apply mask, and convert to land mask
102			msk=grid_diag.msk;
103	gforget	1.6	if length(size(myDiag{1}))==3;
104			msk=repmat(msk,[1 1 size(myDiag{1},3)]);
105			end;
106			for kk=1:size(myDiag{1},4);
107			myDiag(:,:,:,kk)=myDiag(:,:,:,kk).*msk;
108			end;
109	gforget	1.5	clear msk;
110			%
111			land=isnan(grid_diag.msk);
112
113			%set 'coord' attribute
114			if avail_diag.nr~=1;
115			coord='lon lat dep tim';
116			else;
117			coord='lon lat tim';
118			end;
119
120	gforget	1.1	%create netcdf file using write2nctiles (works only with old matlab, thus far ...)
121			doCreate=1;
122	gforget	1.4	dimlist=write2nctiles(myFile,myDiag,doCreate,{'tileNo',tileNo},...
123	gforget	1.1	{'fldName',nameDiag},{'longName',avail_diag.longNameDiag},...
124	gforget	1.5	{'units',avail_diag.units},{'descr',nameDiag},{'coord',coord},{'rdm',rdm});
125	gforget	1.1
126	gforget	1.4	%determine relevant dimensions
127			for ff=1:length(dimlist);
128			dim.tim{ff}={dimlist{ff}{1}};
129			dim.twoD{ff}={dimlist{ff}{end-1:end}};
130			if avail_diag.nr~=1;
131			dim.threeD{ff}={dimlist{ff}{end-2:end}};
132			dim.dep{ff}={dimlist{ff}{end-2}};
133	gforget	1.1	else;
134	gforget	1.4	dim.threeD{ff}=dim.twoD{ff};
135			dim.dep{ff}=[];
136	gforget	1.1	end;
137			end;
138
139	gforget	1.4	%prepare to add fields
140			doCreate=0;
141
142	gforget	1.1	%now add fields
143	gforget	1.4	write2nctiles(myFile,tim,doCreate,{'tileNo',tileNo},...
144	gforget	1.5	{'fldName','tim'},{'longName','time'},...
145			{'units',timUnits},{'dimIn',dim.tim});
146	gforget	1.7	write2nctiles(myFile,listTimes,doCreate,{'tileNo',tileNo},...
147			{'fldName','step'},{'longName','final time step number'},...
148			{'units','1'},{'dimIn',dim.tim});
149	gforget	1.4	write2nctiles(myFile,grid_diag.lon,doCreate,{'tileNo',tileNo},...
150	gforget	1.5	{'fldName','lon'},{'units','degrees_east'},{'dimIn',dim.twoD});
151	gforget	1.4	write2nctiles(myFile,grid_diag.lat,doCreate,{'tileNo',tileNo},...
152	gforget	1.5	{'fldName','lat'},{'units','degrees_north'},{'dimIn',dim.twoD});
153	gforget	1.4	write2nctiles(myFile,grid_diag.msk,doCreate,{'tileNo',tileNo},...
154	gforget	1.5	{'fldName','land'},{'units','1'},{'longName','land mask'},{'dimIn',dim.threeD});
155	gforget	1.4	write2nctiles(myFile,grid_diag.RAC,doCreate,{'tileNo',tileNo},...
156	gforget	1.5	{'fldName','area'},{'units','m^2'},{'longName','grid cell area'},{'dimIn',dim.twoD});
157	gforget	1.1	if isfield(grid_diag,'dep');
158	gforget	1.4	write2nctiles(myFile,grid_diag.dep,doCreate,{'tileNo',tileNo},...
159	gforget	1.5	{'fldName','dep'},{'units','m'},{'dimIn',dim.dep});
160	gforget	1.4	write2nctiles(myFile,grid_diag.dz,doCreate,{'tileNo',tileNo},...
161	gforget	1.5	{'fldName','thic'},{'units','m'},{'dimIn',dim.dep});
162	gforget	1.1	end;
163
164			end;%for vv=1:length(listFlds);
165
166			function [meta]=read_meta(fileName);
167
168			%read meta file
169			tmp1=dir([fileName '*.meta']); tmp1=tmp1(1).name;
170	gforget	1.8	tmp2=strfind(fileName,filesep);
171	gforget	1.1	if ~isempty(tmp2); tmp2=tmp2(end); else; tmp2=0; end;
172			tmp1=[fileName(1:tmp2) tmp1]; fid=fopen(tmp1);
173			while 1;
174			tline = fgetl(fid);
175			if ~ischar(tline), break, end
176			if isempty(whos('tmp3')); tmp3=tline; else; tmp3=[tmp3 ' ' tline]; end;
177			end
178			fclose(fid);
179
180			%add meta variables to workspace
181			eval(tmp3);
182
183			%reformat to meta structure
184			meta.dataprec=dataprec;
185			meta.nDims=nDims;
186			meta.nFlds=nFlds;
187			meta.nrecords=nrecords;
188			meta.fldList=fldList;
189			meta.dimList=dimList;
190			if ~isempty(who('timeInterval')); meta.timeInterval=timeInterval; end;
191			if ~isempty(who('timeStepNumber')); meta.timeStepNumber=timeStepNumber; end;
192
193			%%
194
195	gforget	1.2	function [rdm]=read_readme(filReadme);
196
197			gcmfaces_global;
198
199			rdm=[];
200
201			fid=fopen(filReadme,'rt');
202			while ~feof(fid);
203			nn=length(rdm);
204			rdm{nn+1} = fgetl(fid);
205			end;
206			fclose(fid);
207
208			%%
209
210	gforget	1.1	function [avail_diag]=read_avail_diag(filAvailDiag,nameDiag);
211
212			gcmfaces_global;
213
214			avail_diag=[];
215
216			fid=fopen(filAvailDiag,'rt');
217			while ~feof(fid);
218			tline = fgetl(fid);
219			tmp1=8-length(nameDiag); tmp1=repmat(' ',[1 tmp1]);
220			tname = ['\|' sprintf('%s',nameDiag) tmp1 '\|'];
221			if ~isempty(strfind(tline,tname));
222			%e.g. tline=' 235 \|SIatmQnt\| 1 \| \|SM U1\|W/m^2 \|Net atmospheric heat flux, >0 decreases theta';
223			%
224			tmp1=strfind(tline,'\|'); tmp1=tmp1(end-1:end);
225			avail_diag.units=strtrim(tline(tmp1(1)+1:tmp1(2)-1));
226			avail_diag.longNameDiag=tline(tmp1(2)+1:end);
227			%
228			tmp1=strfind(tline,'\|'); tmp1=tmp1(4:5);
229			pars=tline(tmp1(1)+1:tmp1(2)-1);
230			%
231			if strcmp(pars(2),'M'); avail_diag.loc_h='C';
232			elseif strcmp(pars(2),'U'); avail_diag.loc_h='W';
233			elseif strcmp(pars(2),'V'); avail_diag.loc_h='S';
234			end;
235			%
236			avail_diag.loc_z=pars(9);
237			%
238			if strcmp(pars(10),'1'); avail_diag.nr=1;
239			else; avail_diag.nr=length(mygrid.RC);
240			end;
241			end;
242			end;
243			fclose(fid);
244
245			%%
246
247			function [grid_diag]=set_grid_diag(avail_diag);
248
249			gcmfaces_global;
250
251			%switch for non-tracer point values
252			if strcmp(avail_diag.loc_h,'C');
253			grid_diag.lon=mygrid.XC;
254			grid_diag.lat=mygrid.YC;
255			grid_diag.msk=mygrid.mskC(:,:,1:avail_diag.nr);
256			elseif strcmp(avail_diag.loc_h,'W');
257			grid_diag.lon=mygrid.XG;
258			grid_diag.lat=mygrid.YC;
259			grid_diag.msk=mygrid.mskW(:,:,1:avail_diag.nr);
260			elseif strcmp(avail_diag.loc_h,'S');
261			grid_diag.lon=mygrid.XC;
262			grid_diag.lat=mygrid.YG;
263			grid_diag.msk=mygrid.mskS(:,:,1:avail_diag.nr);
264			end;
265			grid_diag.RAC=mygrid.RAC;
266
267			%vertical grid
268			if avail_diag.nr~=1;
269			if strcmp(avail_diag.loc_z,'M');
270			grid_diag.dep=-mygrid.RC;
271			grid_diag.dz=mygrid.DRF;
272			elseif strcmp(avail_diag.loc_z,'L');
273			grid_diag.dep=-mygrid.RF(2:end);
274			grid_diag.dz=[mygrid.DRC(2:end) ; 228.25];%quick fix
275			else;
276			error('unknown vertical grid');
277			end;
278			grid_diag.dep=reshape(grid_diag.dep,[1 1 avail_diag.nr]);
279			grid_diag.dz=reshape(grid_diag.dz,[1 1 avail_diag.nr]);
280			end;