matlab_class/gcmfaces_diags/diags_pre_process.m

function [myswitch]=diags_pre_process(dirModel,dirMat,doInteractive);
%object :      pre-processing for grid, model parameters, budgets,
%              profiles, cost and control, etc if required
%input :       dirModel is the directory containing 'diags/' or 'nctiles/'
%              dirMat is the directory where diagnostics results will be saved
%                     if isempty(dirMat) then [dirModel 'mat/'] is used by default
%(optional)    doInteractive=1 allows users to specify parameters interactively
%                     doInteractive = 0 (default) uses ECCO v4 parameters
%                     and omits budgets and model-data misfits analyses
%output :      myswitch is the set of switches (doBudget, doProfiles, doCost, doCtrl)
%                     that are set here depending on the model output available

gcmfaces_global; global myparms;

dirModel=fullfile(dirModel,filesep);
if isempty(dirMat); dirMat=fullfile(dirModel,'mat',filesep); else; dirMat=fullfile(dirMat,filesep); end;
if isempty(who('doInteractive')); doInteractive=0; end;

%detect which types of files are avaiable
test0=isdir([dirModel 'diags']);
test1=~isempty(dir([dirModel 'nctiles*']));
if test0&test1&doInteractive;
    myenv.nctiles=input('select to use binaries (0) or nctiles (1) files\n');
elseif test1;
    myenv.nctiles=1;
elseif test0;
    myenv.nctiles=0;
else;
    error('no files (diags/ or nctiles/)  were found\n');
end

if doInteractive&~myenv.nctiles;%only works with binaries
    doBudget=~isempty(dir([dirModel 'diags' filesep 'BUDG' filesep]));
    %doBudget=doBudget|~isempty(dir([dirModel 'diags' filesep 'budg2d_snap_set1*']));
    doBudget=doBudget&~myenv.nctiles
    doCost=~isempty(dir([dirModel 'barfiles' filesep]))
    doCtrl=~isempty(dir([dirModel 'ADXXfiles' filesep]))
else;
    doBudget=0;
    doCost=0;
    doCtrl=0;
end;

doProfiles=~isempty(dir([dirModel 'MITprof' filesep]));
preprocessProfiles=0;
myenv.profiles=fullfile(dirModel,'MITprof',filesep);
if ~doProfiles;
    doProfiles=~isempty(dir([dirModel 'profiles' filesep]));
    preprocessProfiles=1;
    myenv.profiles=fullfile(dirMat,'profiles',filesep,'output',filesep);
end;

%output switches
myswitch.doBudget=doBudget;
myswitch.doProfiles=doProfiles;
myswitch.doCost=doCost;
myswitch.doCtrl=doCtrl;

%set the list of diags directories and files
if myenv.nctiles;
    if isdir(fullfile(dirModel,'nctiles',filesep));
        listSubdirs={fullfile(dirModel,'nctiles',filesep)};
    elseif isdir(fullfile(dirModel,'nctiles_climatology',filesep));
        listSubdirs={fullfile(dirModel,'nctiles_climatology',filesep)};
    end;
    
    listFiles=dir(listSubdirs{1}); listFiles={listFiles(:).name};
    %remove irrelevant files/dirs
    test1=ones(size(listFiles));
    for kk=1:length(listFiles);
        tmp1=fullfile(listSubdirs{1},listFiles{kk},[listFiles{kk} '*.nc']);
        test1(kk)=~isempty(dir(tmp1));
    end;
    listFiles={listFiles{find(test1)}};
    %store in myparms for later use
    myenv.nctilesdir=listSubdirs{1};
    myenv.nctileslist=listFiles;
    %
    myenv.diagsdir='';
else;
    myenv.diagsdir=fullfile(dirModel,['diags' filesep]);
    %
    myenv.nctilesdir='';
    myenv.nctileslist={};
end;
myenv.matdir=fullfile(dirMat);

%0) create dirMat if needed:
if isempty(dir(dirMat)); mkdir(dirMat); end;

%1) pre-processing diags_grid_parms.mat
test0=isempty(dir([dirMat 'diags_grid_parms.mat']));
test1=isempty(dir([dirMat 'lock_mygrid']));

if test0&test1;%this process will do the pre-processing
    fprintf(['pre-processing : started for mygrid, myparms \n']);
    write2file([dirMat 'lock_mygrid'],1);
    %set the list of diags times
    [listTimes]=diags_list_times;
    %set grid and model parameters:
    diags_grid_parms(listTimes,doInteractive);
    %save to disk:
    %if doInteractive;
    eval(['save ' dirMat 'diags_grid_parms.mat mygrid myparms;']);
    %end;
    delete([dirMat 'lock_mygrid']);
    test1=1;
    fprintf(['pre-processing : completed for mygrid, myparms \n\n']);
end;

%here I should test that files are indeed found (may have been moved)

while ~test1;%this process will wait for pre-processing to complete
    fprintf(['waiting 30s for removal of ' dirMat 'lock_mygrid \n']);
    fprintf(['- That should happen automatically after pre-processing is complete \n']);
    fprintf(['- But if a previous session was interupted, you may need to stop this one, \n ']);
    fprintf(['  remove ' dirMat 'lock_mygrid manually, and start over. \n\n']);
    test1=isempty(dir([dirMat 'lock_mygrid']));
    pause(30);
end;

%here we always reload the grid from dirMat to make sure the same one is used throughout
eval(['load ' dirMat 'diags_grid_parms.mat;']);

%2) pre-processing profiles
test0=isempty(dir([dirMat 'profiles/']));
test1=isempty(dir([dirMat 'lock_profiles']));

if test0&test1&doProfiles&preprocessProfiles;%this process will do the pre-processing
    fprintf(['pre-processing : started for profiles \n']);
    write2file([dirMat 'lock_profiles'],1);
    mkdir([dirMat 'profiles/']);
    mkdir([dirMat 'profiles/input/']);
    mkdir([dirMat 'profiles/output/']);
    % NOTE: if on DOS system change ln to mklink
    system(['ln -s /net/nares/raid11/ecco-shared/ecco-version-4/input/input_insitu/*.nc ' dirMat 'profiles/input/.']);
    system(['ln -s ' dirModel '*.nc ' dirMat 'profiles/input/.']);
    if dirModel(1)~='/'; dirModelFull=[pwd '/' dirModel]; else; dirModelFull=dirModel; end;
    % NOTE: if on DOS system change ln to mklink
    system(['ln -s ' dirModelFull 'profiles/*equi.data ' dirMat 'profiles/.']);
    listModel={'argo_feb2013_1992_to_2007*','argo_feb2013_2008_to_2010*',...
        'argo_feb2013_2011_to_2012*','ctd_feb2013*','itp_feb2013*',...
        'seals_feb2013*','xbt_feb2013*','climode_feb2013*'};
    MITprof_gcm2nc([dirMat 'profiles/'],listModel);
    delete([dirMat 'lock_profiles']);
    test1=1;
    fprintf(['pre-processing : completed for profiles \n\n']);
end;

while ~test1&doProfiles;%this process will wait for pre-processing to complete
    fprintf(['waiting 30s for removal of ' dirMat 'lock_profiles \n']);
    fprintf(['- That should happen automatically after pre-processing is complete \n']);
    fprintf(['- But if a previous session was interupted, you may need to stop this one, \n ']);
    fprintf(['  remove ' dirMat 'lock_profiles manually, and start over. \n\n']);
    test1=isempty(dir([dirMat 'lock_profiles']));
    pause(30);
end;

%3) budget pre-processing
test0=isempty(dir([dirMat 'BUDG']));
test1=isempty(dir([dirMat 'lock_budg']));%this aims at having only one process do the

if (test0&test1&doBudget);
    fprintf(['pre-processing : started for budget \n']);
    write2file([dirMat 'lock_budg'],1);
    mkdir([dirMat 'BUDG']);
    %compute time derivatives between snwpshots that will be
    %compared in budgets with the time mean flux terms
    tmp1=fullfile(dirModel,'diags',filesep,'BUDG',filesep,'budg2d_snap_set2*meta');
    test3d=isempty(dir(tmp1))
    diags_diff_snapshots(dirModel,dirMat,'budg2d_snap_set1');
    if ~test3d;
      diags_diff_snapshots(dirModel,dirMat,'budg2d_snap_set2');
    else;
      diags_diff_snapshots(dirModel,dirMat,'budg3d_snap_set1');
    end;
    %diags_diff_snapshots(dirModel,dirMat,'budg3d_snap_set1');
    budget_list=1;
    for kk=1:length(mygrid.RC);
        tmp1=sprintf('%s/diags/BUDG/budg2d_snap_set3_%02i*',dirModel,kk);
        tmp2=~isempty(dir(tmp1));
        if tmp2;
            budget_list=[budget_list kk];
            tmp1=sprintf('budg2d_snap_set3_%02i',kk);
            diags_diff_snapshots(dirModel,dirMat,tmp1);
        end;
    end;
    eval(['save ' dirMat 'diags_select_budget_list.mat budget_list;']);
    delete([dirMat 'lock_budg']);
    test1=1;
    fprintf(['pre-processing : completed for budget \n\n']);
end;

while ~test1&doBudget;%this process will wait for pre-processing to complete
    fprintf(['waiting 30s more for removal of ' dirMat 'lock_budg \n']);
    fprintf(['- That should happen automatically after pre-processing is complete \n']);
    fprintf(['- But if a previous session was interupted, you may need to stop this one, \n ']);
    fprintf(['  remove ' dirMat 'lock_budg manually, and start over. \n\n']);
    test1=isempty(dir([dirMat 'lock_budg']));
    pause(30);
end;

%set budget list
myparms.budgetList=1;
if ~isempty(dir([dirMat 'diags_select_budget_list.mat']));
    eval(['load ' dirMat 'diags_select_budget_list.mat;']);
    myparms.budgetList=budget_list;
end;
1	gforget	1.8	function [myswitch]=diags_pre_process(dirModel,dirMat,doInteractive);
2	gforget	1.10	%object : pre-processing for grid, model parameters, budgets,
3	gforget	1.8	% profiles, cost and control, etc if required
4			%input : dirModel is the directory containing 'diags/' or 'nctiles/'
5			% dirMat is the directory where diagnostics results will be saved
6			% if isempty(dirMat) then [dirModel 'mat/'] is used by default
7			%(optional) doInteractive=1 allows users to specify parameters interactively
8			% doInteractive = 0 (default) uses ECCO v4 parameters
9			% and omits budgets and model-data misfits analyses
10			%output : myswitch is the set of switches (doBudget, doProfiles, doCost, doCtrl)
11			% that are set here depending on the model output available
12	gforget	1.1
13			gcmfaces_global; global myparms;
14
15	gforget	1.12	dirModel=fullfile(dirModel,filesep);
16			if isempty(dirMat); dirMat=fullfile(dirModel,'mat',filesep); else; dirMat=fullfile(dirMat,filesep); end;
17	gforget	1.8	if isempty(who('doInteractive')); doInteractive=0; end;
18	gforget	1.1
19	gforget	1.6	%detect which types of files are avaiable
20			test0=isdir([dirModel 'diags']);
21	gforget	1.11	test1=~isempty(dir([dirModel 'nctiles*']));
22	gforget	1.10	if test0&test1&doInteractive;
23			myenv.nctiles=input('select to use binaries (0) or nctiles (1) files\n');
24	gforget	1.8	elseif test1;
25	gforget	1.10	myenv.nctiles=1;
26	gforget	1.6	elseif test0;
27	gforget	1.10	myenv.nctiles=0;
28	gforget	1.6	else;
29	gforget	1.10	error('no files (diags/ or nctiles/) were found\n');
30	gforget	1.6	end
31
32	gforget	1.12	if doInteractive&~myenv.nctiles;%only works with binaries
33	gforget	1.14	doBudget=~isempty(dir([dirModel 'diags' filesep 'BUDG' filesep]));
34			%doBudget=doBudget\|~isempty(dir([dirModel 'diags' filesep 'budg2d_snap_set1*']));
35			doBudget=doBudget&~myenv.nctiles
36	gforget	1.12	doCost=~isempty(dir([dirModel 'barfiles' filesep]))
37			doCtrl=~isempty(dir([dirModel 'ADXXfiles' filesep]))
38	gforget	1.8	else;
39	gforget	1.10	doBudget=0;
40			doCost=0;
41			doCtrl=0;
42	gforget	1.8	end;
43
44	gforget	1.12	doProfiles=~isempty(dir([dirModel 'MITprof' filesep]));
45	gforget	1.8	preprocessProfiles=0;
46	gforget	1.14	myenv.profiles=fullfile(dirModel,'MITprof',filesep);
47	gforget	1.8	if ~doProfiles;
48	gforget	1.12	doProfiles=~isempty(dir([dirModel 'profiles' filesep]));
49	gforget	1.10	preprocessProfiles=1;
50	gforget	1.14	myenv.profiles=fullfile(dirMat,'profiles',filesep,'output',filesep);
51	gforget	1.8	end;
52	gforget	1.1
53			%output switches
54			myswitch.doBudget=doBudget;
55			myswitch.doProfiles=doProfiles;
56			myswitch.doCost=doCost;
57			myswitch.doCtrl=doCtrl;
58
59	gforget	1.6	%set the list of diags directories and files
60			if myenv.nctiles;
61	gforget	1.14	if isdir(fullfile(dirModel,'nctiles',filesep));
62			listSubdirs={fullfile(dirModel,'nctiles',filesep)};
63			elseif isdir(fullfile(dirModel,'nctiles_climatology',filesep));
64			listSubdirs={fullfile(dirModel,'nctiles_climatology',filesep)};
65	gforget	1.11	end;
66
67	gforget	1.6	listFiles=dir(listSubdirs{1}); listFiles={listFiles(:).name};
68			%remove irrelevant files/dirs
69			test1=ones(size(listFiles));
70			for kk=1:length(listFiles);
71	gforget	1.12	tmp1=fullfile(listSubdirs{1},listFiles{kk},[listFiles{kk} '*.nc']);
72	gforget	1.10	test1(kk)=~isempty(dir(tmp1));
73	gforget	1.6	end;
74			listFiles={listFiles{find(test1)}};
75			%store in myparms for later use
76			myenv.nctilesdir=listSubdirs{1};
77			myenv.nctileslist=listFiles;
78	gforget	1.12	%
79			myenv.diagsdir='';
80			else;
81	gforget	1.14	myenv.diagsdir=fullfile(dirModel,['diags' filesep]);
82	gforget	1.12	%
83			myenv.nctilesdir='';
84			myenv.nctileslist={};
85	gforget	1.6	end;
86	gforget	1.14	myenv.matdir=fullfile(dirMat);
87	gforget	1.6
88	gforget	1.1	%0) create dirMat if needed:
89	gforget	1.7	if isempty(dir(dirMat)); mkdir(dirMat); end;
90	gforget	1.1
91			%1) pre-processing diags_grid_parms.mat
92			test0=isempty(dir([dirMat 'diags_grid_parms.mat']));
93			test1=isempty(dir([dirMat 'lock_mygrid']));
94
95			if test0&test1;%this process will do the pre-processing
96	gforget	1.8	fprintf(['pre-processing : started for mygrid, myparms \n']);
97	gforget	1.1	write2file([dirMat 'lock_mygrid'],1);
98			%set the list of diags times
99	gforget	1.10	[listTimes]=diags_list_times;
100	gforget	1.1	%set grid and model parameters:
101	gforget	1.8	diags_grid_parms(listTimes,doInteractive);
102	gforget	1.1	%save to disk:
103	gforget	1.10	%if doInteractive;
104			eval(['save ' dirMat 'diags_grid_parms.mat mygrid myparms;']);
105			%end;
106	gforget	1.7	delete([dirMat 'lock_mygrid']);
107	gforget	1.1	test1=1;
108	gforget	1.8	fprintf(['pre-processing : completed for mygrid, myparms \n\n']);
109	gforget	1.1	end;
110
111	gforget	1.6	%here I should test that files are indeed found (may have been moved)
112
113	gforget	1.1	while ~test1;%this process will wait for pre-processing to complete
114			fprintf(['waiting 30s for removal of ' dirMat 'lock_mygrid \n']);
115	gforget	1.2	fprintf(['- That should happen automatically after pre-processing is complete \n']);
116			fprintf(['- But if a previous session was interupted, you may need to stop this one, \n ']);
117	gforget	1.14	fprintf([' remove ' dirMat 'lock_mygrid manually, and start over. \n\n']);
118	gforget	1.1	test1=isempty(dir([dirMat 'lock_mygrid']));
119			pause(30);
120			end;
121
122			%here we always reload the grid from dirMat to make sure the same one is used throughout
123	gforget	1.10	eval(['load ' dirMat 'diags_grid_parms.mat;']);
124	gforget	1.1
125			%2) pre-processing profiles
126			test0=isempty(dir([dirMat 'profiles/']));
127			test1=isempty(dir([dirMat 'lock_profiles']));
128
129	gforget	1.8	if test0&test1&doProfiles&preprocessProfiles;%this process will do the pre-processing
130	gforget	1.1	fprintf(['pre-processing : started for profiles \n']);
131			write2file([dirMat 'lock_profiles'],1);
132	gforget	1.7	mkdir([dirMat 'profiles/']);
133	gforget	1.14	mkdir([dirMat 'profiles/input/']);
134			mkdir([dirMat 'profiles/output/']);
135	gforget	1.10	% NOTE: if on DOS system change ln to mklink
136	gforget	1.14	system(['ln -s /net/nares/raid11/ecco-shared/ecco-version-4/input/input_insitu/*.nc ' dirMat 'profiles/input/.']);
137			system(['ln -s ' dirModel '*.nc ' dirMat 'profiles/input/.']);
138	gforget	1.1	if dirModel(1)~='/'; dirModelFull=[pwd '/' dirModel]; else; dirModelFull=dirModel; end;
139	gforget	1.10	% NOTE: if on DOS system change ln to mklink
140	gforget	1.7	system(['ln -s ' dirModelFull 'profiles/*equi.data ' dirMat 'profiles/.']);
141	gforget	1.4	listModel={'argo_feb2013_1992_to_2007','argo_feb2013_2008_to_2010',...
142	gforget	1.3	'argo_feb2013_2011_to_2012','ctd_feb2013','itp_feb2013*',...
143			'seals_feb2013','xbt_feb2013','climode_feb2013*'};
144	gforget	1.1	MITprof_gcm2nc([dirMat 'profiles/'],listModel);
145	gforget	1.7	delete([dirMat 'lock_profiles']);
146	gforget	1.1	test1=1;
147			fprintf(['pre-processing : completed for profiles \n\n']);
148			end;
149
150			while ~test1&doProfiles;%this process will wait for pre-processing to complete
151			fprintf(['waiting 30s for removal of ' dirMat 'lock_profiles \n']);
152	gforget	1.2	fprintf(['- That should happen automatically after pre-processing is complete \n']);
153			fprintf(['- But if a previous session was interupted, you may need to stop this one, \n ']);
154			fprintf([' remove ' dirMat 'lock_profiles manually, and start over. \n\n']);
155	gforget	1.1	test1=isempty(dir([dirMat 'lock_profiles']));
156			pause(30);
157			end;
158
159			%3) budget pre-processing
160	gforget	1.12	test0=isempty(dir([dirMat 'BUDG']));
161	gforget	1.1	test1=isempty(dir([dirMat 'lock_budg']));%this aims at having only one process do the
162
163			if (test0&test1&doBudget);
164			fprintf(['pre-processing : started for budget \n']);
165			write2file([dirMat 'lock_budg'],1);
166	gforget	1.12	mkdir([dirMat 'BUDG']);
167	gforget	1.1	%compute time derivatives between snwpshots that will be
168			%compared in budgets with the time mean flux terms
169	gforget	1.13	tmp1=fullfile(dirModel,'diags',filesep,'BUDG',filesep,'budg2d_snap_set2*meta');
170			test3d=isempty(dir(tmp1))
171	gforget	1.5	diags_diff_snapshots(dirModel,dirMat,'budg2d_snap_set1');
172	gforget	1.13	if ~test3d;
173			diags_diff_snapshots(dirModel,dirMat,'budg2d_snap_set2');
174			else;
175			diags_diff_snapshots(dirModel,dirMat,'budg3d_snap_set1');
176			end;
177	gforget	1.5	%diags_diff_snapshots(dirModel,dirMat,'budg3d_snap_set1');
178	gforget	1.1	budget_list=1;
179			for kk=1:length(mygrid.RC);
180			tmp1=sprintf('%s/diags/BUDG/budg2d_snap_set3_%02i*',dirModel,kk);
181			tmp2=~isempty(dir(tmp1));
182			if tmp2;
183			budget_list=[budget_list kk];
184			tmp1=sprintf('budg2d_snap_set3_%02i',kk);
185	gforget	1.5	diags_diff_snapshots(dirModel,dirMat,tmp1);
186	gforget	1.1	end;
187			end;
188			eval(['save ' dirMat 'diags_select_budget_list.mat budget_list;']);
189	gforget	1.7	delete([dirMat 'lock_budg']);
190	gforget	1.1	test1=1;
191			fprintf(['pre-processing : completed for budget \n\n']);
192			end;
193
194			while ~test1&doBudget;%this process will wait for pre-processing to complete
195			fprintf(['waiting 30s more for removal of ' dirMat 'lock_budg \n']);
196	gforget	1.2	fprintf(['- That should happen automatically after pre-processing is complete \n']);
197			fprintf(['- But if a previous session was interupted, you may need to stop this one, \n ']);
198			fprintf([' remove ' dirMat 'lock_budg manually, and start over. \n\n']);
199	gforget	1.1	test1=isempty(dir([dirMat 'lock_budg']));
200			pause(30);
201			end;
202
203	gforget	1.10	%set budget list
204			myparms.budgetList=1;
205			if ~isempty(dir([dirMat 'diags_select_budget_list.mat']));
206			eval(['load ' dirMat 'diags_select_budget_list.mat;']);
207			myparms.budgetList=budget_list;
208			end;