matlab_class/gcmfaces_diags/diags_pre_process.m

function [myswitch]=diags_pre_process(dirModel,dirMat,doInteractive);
%object :      pre-processing for grid, model parameters, budgets,
%              profiles, cost and control, etc if required
%input :       dirModel is the directory containing 'diags/' or 'nctiles/'
%              dirMat is the directory where diagnostics results will be saved
%                     if isempty(dirMat) then [dirModel 'mat/'] is used by default
%(optional)    doInteractive=1 allows users to specify parameters interactively
%                     doInteractive = 0 (default) uses ECCO v4 parameters
%                     and omits budgets and model-data misfits analyses
%output :      myswitch is the set of switches (doBudget, doProfiles, doCost, doCtrl)
%                     that are set here depending on the model output available

gcmfaces_global; global myparms;

dirModel=fullfile(dirModel,filesep);
if isempty(dirMat); dirMat=fullfile(dirModel,'mat',filesep); else; dirMat=fullfile(dirMat,filesep); end;
if isempty(who('doInteractive')); doInteractive=0; end;

%detect which types of files are avaiable
test0=isdir([dirModel 'diags'])|~isempty(dir([dirModel 'state_2d_set1*']));
test1=~isempty(dir([dirModel 'nctiles']))|...
  ~isempty(dir([dirModel 'nctiles_climatology']))|...
  ~isempty(dir([dirModel 'nctiles_monthly']));
if test0&test1&doInteractive;
    myenv.nctiles=input('select to use binaries (0) or nctiles (1) files\n');
elseif test1;
    myenv.nctiles=1;
elseif test0;
    myenv.nctiles=0;
else;
    error('no files (diags/ or nctiles/)  were found\n');
end

if ~myenv.nctiles;%only works with binaries
    dirSnap=fullfile(dirModel,'diags',filesep);
    doBudget=~isempty(dir([dirSnap 'budg2d_snap_set1*']));
    if ~doBudget;
      dirSnap=fullfile(dirModel,'diags',filesep,'BUDG',filesep);
      doBudget=~isempty(dir([dirSnap 'budg2d_snap_set1*']));
    end;
    if ~doBudget;
      dirSnap=dirModel;
      doBudget=~isempty(dir([dirSnap 'budg2d_snap_set1*']));
    end;
    doCtrl=~isempty(dir([dirModel 'xx_atemp.effective.*']))
    doCtrl=doCtrl&~isempty(dir([dirModel 'cap_sigma_tmp2m_degC_eccollc.bin']));
else;
    dirSnap=fullfile(dirModel,'diags',filesep);
    doBudget=0;
    doCost=0;
    doCtrl=0;
end;

doCost=~isempty(dir([dirModel 'barfiles' filesep 'm_eta_day.*.data']));
doCost=doCost|~isempty(dir([dirModel 'm_eta_day.*.data']));
doCost=doCost|~isempty(dir([dirModel 'nctiles_remotesensing']));

doProfiles=~isempty(dir([dirModel 'MITprof' filesep]));
preprocessProfiles=0;
myenv.profiles=fullfile(dirModel,'MITprof',filesep);
if ~doProfiles;
    doProfiles=~isempty(dir([dirModel 'profiles' filesep '*.nc']));
    preprocessProfiles=0;
    myenv.profiles=fullfile(dirModel,'profiles',filesep);
end;
if ~doProfiles;
    doProfiles=~isempty(dir([dirModel 'profiles' filesep]));
    preprocessProfiles=1;
    myenv.profiles=fullfile(dirMat,'profiles',filesep,'output',filesep);
end;

%output switches
myswitch.doBudget=doBudget;
myswitch.doProfiles=doProfiles;
myswitch.doCost=doCost;
myswitch.doCtrl=doCtrl;

fprintf('\n'); disp(myswitch);
if doInteractive;
  test0=input('edit switches (1) or proceed (0)?\n');
  while test0;
    test0=input('type modification (e.g. as ''myswitch.doBudget=1;'') or hit return to stop editing.\n');
    if ~isempty(test0); eval(test0); disp(myswitch); end;
  end;
end;


%set the list of diags directories and files
if myenv.nctiles;
    if isdir(fullfile(dirModel,'nctiles',filesep));
        listSubdirs={fullfile(dirModel,'nctiles',filesep)};
    elseif isdir(fullfile(dirModel,'nctiles_monthly',filesep));
        listSubdirs={fullfile(dirModel,'nctiles_monthly',filesep)};
    elseif isdir(fullfile(dirModel,'nctiles_climatology',filesep));
        listSubdirs={fullfile(dirModel,'nctiles_climatology',filesep)};
    end;
    
    listFiles=dir(listSubdirs{1}); listFiles={listFiles(:).name};
    %remove irrelevant files/dirs
    test1=ones(size(listFiles));
    for kk=1:length(listFiles);
        tmp1=fullfile(listSubdirs{1},listFiles{kk},[listFiles{kk} '*.nc']);
        test1(kk)=~isempty(dir(tmp1));
    end;
    listFiles={listFiles{find(test1)}};
    %store in myparms for later use
    myenv.nctilesdir=listSubdirs{1};
    myenv.nctileslist=listFiles;
    %
    myenv.diagsdir='';
else;
    myenv.diagsdir=fullfile(dirModel,['diags' filesep]);
    if isempty(dir([myenv.diagsdir '*.data']))&...
       isempty(dir([myenv.diagsdir 'STATE' filesep '*.data']));
      myenv.diagsdir=fullfile(dirModel);
    end;
    %
    myenv.nctilesdir='';
    myenv.nctileslist={};
end;
myenv.matdir=fullfile(dirMat);

%0) create dirMat if needed:
if isempty(dir(dirMat)); mkdir(dirMat); end;

%1) pre-processing diags_grid_parms.mat
test0=isempty(dir([dirMat 'diags_grid_parms.mat']));
test1=isempty(dir([dirMat 'lock_mygrid']));

if test0&test1;%this process will do the pre-processing
    fprintf(['pre-processing : started for mygrid, myparms \n']);
    write2file([dirMat 'lock_mygrid'],1);
    %set the list of diags times
    [listTimes]=diags_list_times;
    %set grid and model parameters:
    diags_grid_parms(dirModel,listTimes,doInteractive);
    %save to disk:
    eval(['save ' dirMat 'diags_grid_parms.mat myparms;']);
    delete([dirMat 'lock_mygrid']);
    test1=1;
    fprintf(['pre-processing : completed for mygrid, myparms \n\n']);
end;

%here I should test that files are indeed found (may have been moved)

while ~test1;%this process will wait for pre-processing to complete
    fprintf(['waiting 30s for removal of ' dirMat 'lock_mygrid \n']);
    fprintf(['- That should happen automatically after pre-processing is complete \n']);
    fprintf(['- But if a previous session was interupted, you may need to stop this one, \n ']);
    fprintf(['  remove ' dirMat 'lock_mygrid manually, and start over. \n\n']);
    test1=isempty(dir([dirMat 'lock_mygrid']));
    pause(30);
end;

%here we always reload the grid from dirMat to make sure the same one is used throughout
eval(['load ' dirMat 'diags_grid_parms.mat;']);

%2) pre-processing profiles
test0=isempty(dir([dirMat 'profiles/']));
test1=isempty(dir([dirMat 'lock_profiles']));

if test0&test1&doProfiles&preprocessProfiles;%this process will do the pre-processing
    fprintf(['pre-processing : started for profiles \n']);
    write2file([dirMat 'lock_profiles'],1);
    mkdir([dirMat 'profiles/']);
    mkdir([dirMat 'profiles/output/']);

    listModel=dir([dirModel '*.nc']);
    listModel={listModel(:).name};

    if ~isempty(listModel);
      MITprof_gcm2nc({dirModel,[dirMat 'profiles/output/']},listModel);
    else;
      warning('no MITprof files wer found');
    end;

    delete([dirMat 'lock_profiles']);
    test1=1;
    fprintf(['pre-processing : completed for profiles \n\n']);
end;

while ~test1&doProfiles;%this process will wait for pre-processing to complete
    fprintf(['waiting 30s for removal of ' dirMat 'lock_profiles \n']);
    fprintf(['- That should happen automatically after pre-processing is complete \n']);
    fprintf(['- But if a previous session was interupted, you may need to stop this one, \n ']);
    fprintf(['  remove ' dirMat 'lock_profiles manually, and start over. \n\n']);
    test1=isempty(dir([dirMat 'lock_profiles']));
    pause(30);
end;

%3) budget pre-processing
test0=isempty(dir([dirMat 'BUDG']));
test1=isempty(dir([dirMat 'lock_budg']));%this aims at having only one process do the

if (test0&test1&doBudget);
    fprintf(['pre-processing : started for budget \n']);
    write2file([dirMat 'lock_budg'],1);
    mkdir([dirMat 'BUDG']);
    %compute time derivatives between snapshots that will be
    %compared in budgets with the time mean flux terms
    tmp1=fullfile(dirSnap,'budg3d_snap_set1*meta');
    test3d=~isempty(dir(tmp1));
    tmp1=fullfile(dirSnap,'budg3d_snap_set1*meta');
    test3d=test3d|~isempty(dir(tmp1));
    tmp1=fullfile(dirSnap,'geothermalFlux.bin');
    testGeothermalFlux=~isempty(dir(tmp1));
    diags_diff_snapshots(dirSnap,dirMat,'budg2d_snap_set1');
    if ~test3d;
      diags_diff_snapshots(dirSnap,dirMat,'budg2d_snap_set2');
      if testGeothermalFlux; diags_budg_geothermal(dirSnap,dirMat,'budg2d_snap_set2'); end;
    else;
      diags_diff_snapshots(dirSnap,dirMat,'budg3d_snap_set1');
      if testGeothermalFlux; diags_budg_geothermal(dirSnap,dirMat,'budg3d_snap_set1'); end;
    end;
    budget_list=1;
    for kk=1:length(mygrid.RC);
        tmp1=sprintf('%s/budg2d_snap_set3_%02i*',dirSnap,kk);
        tmp2=~isempty(dir(tmp1));
        if tmp2;
            budget_list=[budget_list kk];
            tmp1=sprintf('budg2d_snap_set3_%02i',kk);
            diags_diff_snapshots(dirSnap,dirMat,tmp1);
           if testGeothermalFlux; diags_budg_geothermal(dirSnap,dirMat,tmp1); end;
        end;
    end;
    eval(['save ' dirMat 'diags_select_budget_list.mat budget_list;']);
    delete([dirMat 'lock_budg']);
    test1=1;
    fprintf(['pre-processing : completed for budget \n\n']);
end;

while ~test1&doBudget;%this process will wait for pre-processing to complete
    fprintf(['waiting 30s more for removal of ' dirMat 'lock_budg \n']);
    fprintf(['- That should happen automatically after pre-processing is complete \n']);
    fprintf(['- But if a previous session was interupted, you may need to stop this one, \n ']);
    fprintf(['  remove ' dirMat 'lock_budg manually, and start over. \n\n']);
    test1=isempty(dir([dirMat 'lock_budg']));
    pause(30);
end;

%set budget list
myparms.budgetList=1;
if ~isempty(dir([dirMat 'diags_select_budget_list.mat']));
    eval(['load ' dirMat 'diags_select_budget_list.mat;']);
    myparms.budgetList=budget_list;
end;
1	function [myswitch]=diags_pre_process(dirModel,dirMat,doInteractive);
2	%object : pre-processing for grid, model parameters, budgets,
3	% profiles, cost and control, etc if required
4	%input : dirModel is the directory containing 'diags/' or 'nctiles/'
5	% dirMat is the directory where diagnostics results will be saved
6	% if isempty(dirMat) then [dirModel 'mat/'] is used by default
7	%(optional) doInteractive=1 allows users to specify parameters interactively
8	% doInteractive = 0 (default) uses ECCO v4 parameters
9	% and omits budgets and model-data misfits analyses
10	%output : myswitch is the set of switches (doBudget, doProfiles, doCost, doCtrl)
11	% that are set here depending on the model output available
12
13	gcmfaces_global; global myparms;
14
15	dirModel=fullfile(dirModel,filesep);
16	if isempty(dirMat); dirMat=fullfile(dirModel,'mat',filesep); else; dirMat=fullfile(dirMat,filesep); end;
17	if isempty(who('doInteractive')); doInteractive=0; end;
18
19	%detect which types of files are avaiable
20	test0=isdir([dirModel 'diags'])\|~isempty(dir([dirModel 'state_2d_set1*']));
21	test1=~isempty(dir([dirModel 'nctiles']))\|...
22	~isempty(dir([dirModel 'nctiles_climatology']))\|...
23	~isempty(dir([dirModel 'nctiles_monthly']));
24	if test0&test1&doInteractive;
25	myenv.nctiles=input('select to use binaries (0) or nctiles (1) files\n');
26	elseif test1;
27	myenv.nctiles=1;
28	elseif test0;
29	myenv.nctiles=0;
30	else;
31	error('no files (diags/ or nctiles/) were found\n');
32	end
33
34	if ~myenv.nctiles;%only works with binaries
35	dirSnap=fullfile(dirModel,'diags',filesep);
36	doBudget=~isempty(dir([dirSnap 'budg2d_snap_set1*']));
37	if ~doBudget;
38	dirSnap=fullfile(dirModel,'diags',filesep,'BUDG',filesep);
39	doBudget=~isempty(dir([dirSnap 'budg2d_snap_set1*']));
40	end;
41	if ~doBudget;
42	dirSnap=dirModel;
43	doBudget=~isempty(dir([dirSnap 'budg2d_snap_set1*']));
44	end;
45	doCtrl=~isempty(dir([dirModel 'xx_atemp.effective.*']))
46	doCtrl=doCtrl&~isempty(dir([dirModel 'cap_sigma_tmp2m_degC_eccollc.bin']));
47	else;
48	dirSnap=fullfile(dirModel,'diags',filesep);
49	doBudget=0;
50	doCost=0;
51	doCtrl=0;
52	end;
53
54	doCost=~isempty(dir([dirModel 'barfiles' filesep 'm_eta_day.*.data']));
55	doCost=doCost\|~isempty(dir([dirModel 'm_eta_day.*.data']));
56	doCost=doCost\|~isempty(dir([dirModel 'nctiles_remotesensing']));
57
58	doProfiles=~isempty(dir([dirModel 'MITprof' filesep]));
59	preprocessProfiles=0;
60	myenv.profiles=fullfile(dirModel,'MITprof',filesep);
61	if ~doProfiles;
62	doProfiles=~isempty(dir([dirModel 'profiles' filesep '*.nc']));
63	preprocessProfiles=0;
64	myenv.profiles=fullfile(dirModel,'profiles',filesep);
65	end;
66	if ~doProfiles;
67	doProfiles=~isempty(dir([dirModel 'profiles' filesep]));
68	preprocessProfiles=1;
69	myenv.profiles=fullfile(dirMat,'profiles',filesep,'output',filesep);
70	end;
71
72	%output switches
73	myswitch.doBudget=doBudget;
74	myswitch.doProfiles=doProfiles;
75	myswitch.doCost=doCost;
76	myswitch.doCtrl=doCtrl;
77
78	fprintf('\n'); disp(myswitch);
79	if doInteractive;
80	test0=input('edit switches (1) or proceed (0)?\n');
81	while test0;
82	test0=input('type modification (e.g. as ''myswitch.doBudget=1;'') or hit return to stop editing.\n');
83	if ~isempty(test0); eval(test0); disp(myswitch); end;
84	end;
85	end;
86
87
88	%set the list of diags directories and files
89	if myenv.nctiles;
90	if isdir(fullfile(dirModel,'nctiles',filesep));
91	listSubdirs={fullfile(dirModel,'nctiles',filesep)};
92	elseif isdir(fullfile(dirModel,'nctiles_monthly',filesep));
93	listSubdirs={fullfile(dirModel,'nctiles_monthly',filesep)};
94	elseif isdir(fullfile(dirModel,'nctiles_climatology',filesep));
95	listSubdirs={fullfile(dirModel,'nctiles_climatology',filesep)};
96	end;
97
98	listFiles=dir(listSubdirs{1}); listFiles={listFiles(:).name};
99	%remove irrelevant files/dirs
100	test1=ones(size(listFiles));
101	for kk=1:length(listFiles);
102	tmp1=fullfile(listSubdirs{1},listFiles{kk},[listFiles{kk} '*.nc']);
103	test1(kk)=~isempty(dir(tmp1));
104	end;
105	listFiles={listFiles{find(test1)}};
106	%store in myparms for later use
107	myenv.nctilesdir=listSubdirs{1};
108	myenv.nctileslist=listFiles;
109	%
110	myenv.diagsdir='';
111	else;
112	myenv.diagsdir=fullfile(dirModel,['diags' filesep]);
113	if isempty(dir([myenv.diagsdir '*.data']))&...
114	isempty(dir([myenv.diagsdir 'STATE' filesep '*.data']));
115	myenv.diagsdir=fullfile(dirModel);
116	end;
117	%
118	myenv.nctilesdir='';
119	myenv.nctileslist={};
120	end;
121	myenv.matdir=fullfile(dirMat);
122
123	%0) create dirMat if needed:
124	if isempty(dir(dirMat)); mkdir(dirMat); end;
125
126	%1) pre-processing diags_grid_parms.mat
127	test0=isempty(dir([dirMat 'diags_grid_parms.mat']));
128	test1=isempty(dir([dirMat 'lock_mygrid']));
129
130	if test0&test1;%this process will do the pre-processing
131	fprintf(['pre-processing : started for mygrid, myparms \n']);
132	write2file([dirMat 'lock_mygrid'],1);
133	%set the list of diags times
134	[listTimes]=diags_list_times;
135	%set grid and model parameters:
136	diags_grid_parms(dirModel,listTimes,doInteractive);
137	%save to disk:
138	eval(['save ' dirMat 'diags_grid_parms.mat myparms;']);
139	delete([dirMat 'lock_mygrid']);
140	test1=1;
141	fprintf(['pre-processing : completed for mygrid, myparms \n\n']);
142	end;
143
144	%here I should test that files are indeed found (may have been moved)
145
146	while ~test1;%this process will wait for pre-processing to complete
147	fprintf(['waiting 30s for removal of ' dirMat 'lock_mygrid \n']);
148	fprintf(['- That should happen automatically after pre-processing is complete \n']);
149	fprintf(['- But if a previous session was interupted, you may need to stop this one, \n ']);
150	fprintf([' remove ' dirMat 'lock_mygrid manually, and start over. \n\n']);
151	test1=isempty(dir([dirMat 'lock_mygrid']));
152	pause(30);
153	end;
154
155	%here we always reload the grid from dirMat to make sure the same one is used throughout
156	eval(['load ' dirMat 'diags_grid_parms.mat;']);
157
158	%2) pre-processing profiles
159	test0=isempty(dir([dirMat 'profiles/']));
160	test1=isempty(dir([dirMat 'lock_profiles']));
161
162	if test0&test1&doProfiles&preprocessProfiles;%this process will do the pre-processing
163	fprintf(['pre-processing : started for profiles \n']);
164	write2file([dirMat 'lock_profiles'],1);
165	mkdir([dirMat 'profiles/']);
166	mkdir([dirMat 'profiles/output/']);
167
168	listModel=dir([dirModel '*.nc']);
169	listModel={listModel(:).name};
170
171	if ~isempty(listModel);
172	MITprof_gcm2nc({dirModel,[dirMat 'profiles/output/']},listModel);
173	else;
174	warning('no MITprof files wer found');
175	end;
176
177	delete([dirMat 'lock_profiles']);
178	test1=1;
179	fprintf(['pre-processing : completed for profiles \n\n']);
180	end;
181
182	while ~test1&doProfiles;%this process will wait for pre-processing to complete
183	fprintf(['waiting 30s for removal of ' dirMat 'lock_profiles \n']);
184	fprintf(['- That should happen automatically after pre-processing is complete \n']);
185	fprintf(['- But if a previous session was interupted, you may need to stop this one, \n ']);
186	fprintf([' remove ' dirMat 'lock_profiles manually, and start over. \n\n']);
187	test1=isempty(dir([dirMat 'lock_profiles']));
188	pause(30);
189	end;
190
191	%3) budget pre-processing
192	test0=isempty(dir([dirMat 'BUDG']));
193	test1=isempty(dir([dirMat 'lock_budg']));%this aims at having only one process do the
194
195	if (test0&test1&doBudget);
196	fprintf(['pre-processing : started for budget \n']);
197	write2file([dirMat 'lock_budg'],1);
198	mkdir([dirMat 'BUDG']);
199	%compute time derivatives between snapshots that will be
200	%compared in budgets with the time mean flux terms
201	tmp1=fullfile(dirSnap,'budg3d_snap_set1*meta');
202	test3d=~isempty(dir(tmp1));
203	tmp1=fullfile(dirSnap,'budg3d_snap_set1*meta');
204	test3d=test3d\|~isempty(dir(tmp1));
205	tmp1=fullfile(dirSnap,'geothermalFlux.bin');
206	testGeothermalFlux=~isempty(dir(tmp1));
207	diags_diff_snapshots(dirSnap,dirMat,'budg2d_snap_set1');
208	if ~test3d;
209	diags_diff_snapshots(dirSnap,dirMat,'budg2d_snap_set2');
210	if testGeothermalFlux; diags_budg_geothermal(dirSnap,dirMat,'budg2d_snap_set2'); end;
211	else;
212	diags_diff_snapshots(dirSnap,dirMat,'budg3d_snap_set1');
213	if testGeothermalFlux; diags_budg_geothermal(dirSnap,dirMat,'budg3d_snap_set1'); end;
214	end;
215	budget_list=1;
216	for kk=1:length(mygrid.RC);
217	tmp1=sprintf('%s/budg2d_snap_set3_%02i*',dirSnap,kk);
218	tmp2=~isempty(dir(tmp1));
219	if tmp2;
220	budget_list=[budget_list kk];
221	tmp1=sprintf('budg2d_snap_set3_%02i',kk);
222	diags_diff_snapshots(dirSnap,dirMat,tmp1);
223	if testGeothermalFlux; diags_budg_geothermal(dirSnap,dirMat,tmp1); end;
224	end;
225	end;
226	eval(['save ' dirMat 'diags_select_budget_list.mat budget_list;']);
227	delete([dirMat 'lock_budg']);
228	test1=1;
229	fprintf(['pre-processing : completed for budget \n\n']);
230	end;
231
232	while ~test1&doBudget;%this process will wait for pre-processing to complete
233	fprintf(['waiting 30s more for removal of ' dirMat 'lock_budg \n']);
234	fprintf(['- That should happen automatically after pre-processing is complete \n']);
235	fprintf(['- But if a previous session was interupted, you may need to stop this one, \n ']);
236	fprintf([' remove ' dirMat 'lock_budg manually, and start over. \n\n']);
237	test1=isempty(dir([dirMat 'lock_budg']));
238	pause(30);
239	end;
240
241	%set budget list
242	myparms.budgetList=1;
243	if ~isempty(dir([dirMat 'diags_select_budget_list.mat']));
244	eval(['load ' dirMat 'diags_select_budget_list.mat;']);
245	myparms.budgetList=budget_list;
246	end;