matlab_class/gcmfaces_diags/diags_pre_process.m

function [myswitch]=diags_pre_process(dirModel,dirMat,doInteractive);
%object :      pre-processing for grid, model parameters, budgets, 
%              profiles, cost and control, etc if required
%input :       dirModel is the directory containing 'diags/' or 'nctiles/'
%              dirMat is the directory where diagnostics results will be saved
%                     if isempty(dirMat) then [dirModel 'mat/'] is used by default
%(optional)    doInteractive=1 allows users to specify parameters interactively
%                     doInteractive = 0 (default) uses ECCO v4 parameters
%                     and omits budgets and model-data misfits analyses
%output :      myswitch is the set of switches (doBudget, doProfiles, doCost, doCtrl)
%                     that are set here depending on the model output available

gcmfaces_global; global myparms;

dirModel=[dirModel '/'];
if isempty(dirMat); dirMat=[dirModel 'mat/']; else; dirMat=[dirMat '/']; end;
if isempty(who('doInteractive')); doInteractive=0; end;

%detect which types of files are avaiable
test0=isdir([dirModel 'diags']);
test1=isdir([dirModel 'nctiles']);
if test0&test1&doInteractive; 
  myenv.nctiles=input('select to use binaries (0) or nctiles (1) files\n');
elseif test1;
  myenv.nctiles=1;
elseif test0;
  myenv.nctiles=0;
else;
  error('no files (diags/ or nctiles/)  were found\n');
end

if doInteractive;
doBudget=~isempty(dir([dirModel 'diags/BUDG/']))
doBudget=doBudget&~myenv.nctiles;
doCost=~isempty(dir([dirModel 'barfiles/']))
doCtrl=~isempty(dir([dirModel 'ADXXfiles/']))
else;
doBudget=0;
doCost=0;
doCtrl=0;
end;

doProfiles=~isempty(dir([dirModel 'MITprof/']));
preprocessProfiles=0;
myenv.profiles=[pwd '/' dirModel 'MITprof/'];
if ~doProfiles;
  doProfiles=~isempty(dir([dirModel 'profiles/']));
  preprocessProfiles=1;
  myenv.profiles=[pwd '/' dirMat 'profiles/output/'];
end;

%output switches
myswitch.doBudget=doBudget;
myswitch.doProfiles=doProfiles;
myswitch.doCost=doCost;
myswitch.doCtrl=doCtrl;

%set the list of diags directories and files
if myenv.nctiles;
    listSubdirs={[pwd '/' dirModel 'nctiles/']};
    listFiles=dir(listSubdirs{1}); listFiles={listFiles(:).name};
    %remove irrelevant files/dirs
    test1=ones(size(listFiles));
    for kk=1:length(listFiles);
      tmp1=[listSubdirs{1} listFiles{kk} '/' listFiles{kk} '*.nc'];
      test1(kk)=~isempty(dir(tmp1));
    end;
    listFiles={listFiles{find(test1)}};
    %store in myparms for later use
    myenv.nctilesdir=listSubdirs{1};
    myenv.nctileslist=listFiles;
end;

%0) create dirMat if needed:
if isempty(dir(dirMat)); mkdir(dirMat); end;

%1) pre-processing diags_grid_parms.mat
test0=isempty(dir([dirMat 'diags_grid_parms.mat']));
test1=isempty(dir([dirMat 'lock_mygrid']));

if test0&test1;%this process will do the pre-processing
    fprintf(['pre-processing : started for mygrid, myparms \n']);
    write2file([dirMat 'lock_mygrid'],1);
    %set the list of diags times
    listSubdirs={[dirMat 'BUDG/' ],[dirModel 'diags/BUDG/' ],[dirModel 'diags/OTHER/' ],...
        [dirModel 'diags/STATE/' ],[dirModel 'diags/TRSP/'],[dirModel 'diags/' ]};
    listFiles={'state_2d_set1','diags_2d_set1','monthly_2d_set1'};
    [listTimes]=diags_list_times(listSubdirs,listFiles);
    %set grid and model parameters:
    diags_grid_parms(listTimes,doInteractive);
    %save to disk:
    if doInteractive; 
      eval(['save ' dirMat 'diags_grid_parms.mat mygrid myparms;']);
    end;
    delete([dirMat 'lock_mygrid']);
    test1=1;
    fprintf(['pre-processing : completed for mygrid, myparms \n\n']);
end;

%here I should test that files are indeed found (may have been moved)

while ~test1;%this process will wait for pre-processing to complete
    fprintf(['waiting 30s for removal of ' dirMat 'lock_mygrid \n']);
    fprintf(['- That should happen automatically after pre-processing is complete \n']);
    fprintf(['- But if a previous session was interupted, you may need to stop this one, \n ']);
    fprintf(['  remove ' dirMat 'lock_grid manually, and start over. \n\n']);
    test1=isempty(dir([dirMat 'lock_mygrid']));
    pause(30);
end;

%here we always reload the grid from dirMat to make sure the same one is used throughout
if doInteractive; eval(['load ' dirMat 'diags_grid_parms.mat;']); end;

%2) pre-processing profiles
test0=isempty(dir([dirMat 'profiles/']));
test1=isempty(dir([dirMat 'lock_profiles']));

if test0&test1&doProfiles&preprocessProfiles;%this process will do the pre-processing
    fprintf(['pre-processing : started for profiles \n']);
    write2file([dirMat 'lock_profiles'],1);
    mkdir([dirMat 'profiles/']);
% NOTE: if on DOS system change ln to mklink
    system(['ln -s /net/nares/raid11/ecco-shared/ecco-version-4/input/input_insitu ' dirMat 'profiles/input']);
    mkdir([dirMat 'profiles/output/']);
    if dirModel(1)~='/'; dirModelFull=[pwd '/' dirModel]; else; dirModelFull=dirModel; end;
% NOTE: if on DOS system change ln to mklink
    system(['ln -s ' dirModelFull 'profiles/*equi.data ' dirMat 'profiles/.']);
    listModel={'argo_feb2013_1992_to_2007*','argo_feb2013_2008_to_2010*',...
        'argo_feb2013_2011_to_2012*','ctd_feb2013*','itp_feb2013*',...
        'seals_feb2013*','xbt_feb2013*','climode_feb2013*'};
    MITprof_gcm2nc([dirMat 'profiles/'],listModel);
    delete([dirMat 'lock_profiles']);
    test1=1;
    fprintf(['pre-processing : completed for profiles \n\n']);
end;

while ~test1&doProfiles;%this process will wait for pre-processing to complete
    fprintf(['waiting 30s for removal of ' dirMat 'lock_profiles \n']);
    fprintf(['- That should happen automatically after pre-processing is complete \n']);
    fprintf(['- But if a previous session was interupted, you may need to stop this one, \n ']);
    fprintf(['  remove ' dirMat 'lock_profiles manually, and start over. \n\n']);
    test1=isempty(dir([dirMat 'lock_profiles']));
    pause(30);
end;

%3) budget pre-processing
test0=isempty(dir([dirMat 'BUDG/']));
test1=isempty(dir([dirMat 'lock_budg']));%this aims at having only one process do the

if (test0&test1&doBudget);
    fprintf(['pre-processing : started for budget \n']);
    write2file([dirMat 'lock_budg'],1);
    mkdir([dirMat 'BUDG/']);
    %compute time derivatives between snwpshots that will be
    %compared in budgets with the time mean flux terms
    diags_diff_snapshots(dirModel,dirMat,'budg2d_snap_set1');
    diags_diff_snapshots(dirModel,dirMat,'budg2d_snap_set2');
    diags_diff_snapshots(dirModel,dirMat,'budg2d_snap_set3');
    %diags_diff_snapshots(dirModel,dirMat,'budg3d_snap_set1');
    budget_list=1;
    for kk=1:length(mygrid.RC);
        tmp1=sprintf('%s/diags/BUDG/budg2d_snap_set3_%02i*',dirModel,kk);
        tmp2=~isempty(dir(tmp1));
        if tmp2;
            budget_list=[budget_list kk];
            tmp1=sprintf('budg2d_snap_set3_%02i',kk);
            diags_diff_snapshots(dirModel,dirMat,tmp1);
        end;
    end;
    eval(['save ' dirMat 'diags_select_budget_list.mat budget_list;']);
    delete([dirMat 'lock_budg']);
    test1=1;
    fprintf(['pre-processing : completed for budget \n\n']);
end;

while ~test1&doBudget;%this process will wait for pre-processing to complete
    fprintf(['waiting 30s more for removal of ' dirMat 'lock_budg \n']);
    fprintf(['- That should happen automatically after pre-processing is complete \n']);
    fprintf(['- But if a previous session was interupted, you may need to stop this one, \n ']);
    fprintf(['  remove ' dirMat 'lock_budg manually, and start over. \n\n']);
    test1=isempty(dir([dirMat 'lock_budg']));
    pause(30);
end;

1	gforget	1.8	function [myswitch]=diags_pre_process(dirModel,dirMat,doInteractive);
2			%object : pre-processing for grid, model parameters, budgets,
3			% profiles, cost and control, etc if required
4			%input : dirModel is the directory containing 'diags/' or 'nctiles/'
5			% dirMat is the directory where diagnostics results will be saved
6			% if isempty(dirMat) then [dirModel 'mat/'] is used by default
7			%(optional) doInteractive=1 allows users to specify parameters interactively
8			% doInteractive = 0 (default) uses ECCO v4 parameters
9			% and omits budgets and model-data misfits analyses
10			%output : myswitch is the set of switches (doBudget, doProfiles, doCost, doCtrl)
11			% that are set here depending on the model output available
12	gforget	1.1
13			gcmfaces_global; global myparms;
14
15			dirModel=[dirModel '/'];
16			if isempty(dirMat); dirMat=[dirModel 'mat/']; else; dirMat=[dirMat '/']; end;
17	gforget	1.8	if isempty(who('doInteractive')); doInteractive=0; end;
18	gforget	1.1
19	gforget	1.6	%detect which types of files are avaiable
20			test0=isdir([dirModel 'diags']);
21			test1=isdir([dirModel 'nctiles']);
22	gforget	1.8	if test0&test1&doInteractive;
23	gforget	1.6	myenv.nctiles=input('select to use binaries (0) or nctiles (1) files\n');
24	gforget	1.8	elseif test1;
25			myenv.nctiles=1;
26	gforget	1.6	elseif test0;
27			myenv.nctiles=0;
28			else;
29			error('no files (diags/ or nctiles/) were found\n');
30			end
31
32	gforget	1.8	if doInteractive;
33	gforget	1.1	doBudget=~isempty(dir([dirModel 'diags/BUDG/']))
34	gforget	1.6	doBudget=doBudget&~myenv.nctiles;
35	gforget	1.1	doCost=~isempty(dir([dirModel 'barfiles/']))
36			doCtrl=~isempty(dir([dirModel 'ADXXfiles/']))
37	gforget	1.8	else;
38			doBudget=0;
39			doCost=0;
40			doCtrl=0;
41			end;
42
43			doProfiles=~isempty(dir([dirModel 'MITprof/']));
44			preprocessProfiles=0;
45			myenv.profiles=[pwd '/' dirModel 'MITprof/'];
46			if ~doProfiles;
47			doProfiles=~isempty(dir([dirModel 'profiles/']));
48			preprocessProfiles=1;
49			myenv.profiles=[pwd '/' dirMat 'profiles/output/'];
50			end;
51	gforget	1.1
52			%output switches
53			myswitch.doBudget=doBudget;
54			myswitch.doProfiles=doProfiles;
55			myswitch.doCost=doCost;
56			myswitch.doCtrl=doCtrl;
57
58	gforget	1.6	%set the list of diags directories and files
59			if myenv.nctiles;
60			listSubdirs={[pwd '/' dirModel 'nctiles/']};
61			listFiles=dir(listSubdirs{1}); listFiles={listFiles(:).name};
62			%remove irrelevant files/dirs
63			test1=ones(size(listFiles));
64			for kk=1:length(listFiles);
65			tmp1=[listSubdirs{1} listFiles{kk} '/' listFiles{kk} '*.nc'];
66			test1(kk)=~isempty(dir(tmp1));
67			end;
68			listFiles={listFiles{find(test1)}};
69			%store in myparms for later use
70			myenv.nctilesdir=listSubdirs{1};
71			myenv.nctileslist=listFiles;
72			end;
73
74	gforget	1.1	%0) create dirMat if needed:
75	gforget	1.7	if isempty(dir(dirMat)); mkdir(dirMat); end;
76	gforget	1.1
77			%1) pre-processing diags_grid_parms.mat
78			test0=isempty(dir([dirMat 'diags_grid_parms.mat']));
79			test1=isempty(dir([dirMat 'lock_mygrid']));
80
81			if test0&test1;%this process will do the pre-processing
82	gforget	1.8	fprintf(['pre-processing : started for mygrid, myparms \n']);
83	gforget	1.1	write2file([dirMat 'lock_mygrid'],1);
84			%set the list of diags times
85			listSubdirs={[dirMat 'BUDG/' ],[dirModel 'diags/BUDG/' ],[dirModel 'diags/OTHER/' ],...
86			[dirModel 'diags/STATE/' ],[dirModel 'diags/TRSP/'],[dirModel 'diags/' ]};
87			listFiles={'state_2d_set1','diags_2d_set1','monthly_2d_set1'};
88			[listTimes]=diags_list_times(listSubdirs,listFiles);
89			%set grid and model parameters:
90	gforget	1.8	diags_grid_parms(listTimes,doInteractive);
91	gforget	1.1	%save to disk:
92	gforget	1.9	if doInteractive;
93			eval(['save ' dirMat 'diags_grid_parms.mat mygrid myparms;']);
94			end;
95	gforget	1.7	delete([dirMat 'lock_mygrid']);
96	gforget	1.1	test1=1;
97	gforget	1.8	fprintf(['pre-processing : completed for mygrid, myparms \n\n']);
98	gforget	1.1	end;
99
100	gforget	1.6	%here I should test that files are indeed found (may have been moved)
101
102	gforget	1.1	while ~test1;%this process will wait for pre-processing to complete
103			fprintf(['waiting 30s for removal of ' dirMat 'lock_mygrid \n']);
104	gforget	1.2	fprintf(['- That should happen automatically after pre-processing is complete \n']);
105			fprintf(['- But if a previous session was interupted, you may need to stop this one, \n ']);
106			fprintf([' remove ' dirMat 'lock_grid manually, and start over. \n\n']);
107	gforget	1.1	test1=isempty(dir([dirMat 'lock_mygrid']));
108			pause(30);
109			end;
110
111			%here we always reload the grid from dirMat to make sure the same one is used throughout
112	gforget	1.9	if doInteractive; eval(['load ' dirMat 'diags_grid_parms.mat;']); end;
113	gforget	1.1
114			%2) pre-processing profiles
115			test0=isempty(dir([dirMat 'profiles/']));
116			test1=isempty(dir([dirMat 'lock_profiles']));
117
118	gforget	1.8	if test0&test1&doProfiles&preprocessProfiles;%this process will do the pre-processing
119	gforget	1.1	fprintf(['pre-processing : started for profiles \n']);
120			write2file([dirMat 'lock_profiles'],1);
121	gforget	1.7	mkdir([dirMat 'profiles/']);
122			% NOTE: if on DOS system change ln to mklink
123			system(['ln -s /net/nares/raid11/ecco-shared/ecco-version-4/input/input_insitu ' dirMat 'profiles/input']);
124			mkdir([dirMat 'profiles/output/']);
125	gforget	1.1	if dirModel(1)~='/'; dirModelFull=[pwd '/' dirModel]; else; dirModelFull=dirModel; end;
126	gforget	1.7	% NOTE: if on DOS system change ln to mklink
127			system(['ln -s ' dirModelFull 'profiles/*equi.data ' dirMat 'profiles/.']);
128	gforget	1.4	listModel={'argo_feb2013_1992_to_2007','argo_feb2013_2008_to_2010',...
129	gforget	1.3	'argo_feb2013_2011_to_2012','ctd_feb2013','itp_feb2013*',...
130			'seals_feb2013','xbt_feb2013','climode_feb2013*'};
131	gforget	1.1	MITprof_gcm2nc([dirMat 'profiles/'],listModel);
132	gforget	1.7	delete([dirMat 'lock_profiles']);
133	gforget	1.1	test1=1;
134			fprintf(['pre-processing : completed for profiles \n\n']);
135			end;
136
137			while ~test1&doProfiles;%this process will wait for pre-processing to complete
138			fprintf(['waiting 30s for removal of ' dirMat 'lock_profiles \n']);
139	gforget	1.2	fprintf(['- That should happen automatically after pre-processing is complete \n']);
140			fprintf(['- But if a previous session was interupted, you may need to stop this one, \n ']);
141			fprintf([' remove ' dirMat 'lock_profiles manually, and start over. \n\n']);
142	gforget	1.1	test1=isempty(dir([dirMat 'lock_profiles']));
143			pause(30);
144			end;
145
146			%3) budget pre-processing
147			test0=isempty(dir([dirMat 'BUDG/']));
148			test1=isempty(dir([dirMat 'lock_budg']));%this aims at having only one process do the
149
150			if (test0&test1&doBudget);
151			fprintf(['pre-processing : started for budget \n']);
152			write2file([dirMat 'lock_budg'],1);
153	gforget	1.7	mkdir([dirMat 'BUDG/']);
154	gforget	1.1	%compute time derivatives between snwpshots that will be
155			%compared in budgets with the time mean flux terms
156	gforget	1.5	diags_diff_snapshots(dirModel,dirMat,'budg2d_snap_set1');
157			diags_diff_snapshots(dirModel,dirMat,'budg2d_snap_set2');
158			diags_diff_snapshots(dirModel,dirMat,'budg2d_snap_set3');
159			%diags_diff_snapshots(dirModel,dirMat,'budg3d_snap_set1');
160	gforget	1.1	budget_list=1;
161			for kk=1:length(mygrid.RC);
162			tmp1=sprintf('%s/diags/BUDG/budg2d_snap_set3_%02i*',dirModel,kk);
163			tmp2=~isempty(dir(tmp1));
164			if tmp2;
165			budget_list=[budget_list kk];
166			tmp1=sprintf('budg2d_snap_set3_%02i',kk);
167	gforget	1.5	diags_diff_snapshots(dirModel,dirMat,tmp1);
168	gforget	1.1	end;
169			end;
170			eval(['save ' dirMat 'diags_select_budget_list.mat budget_list;']);
171	gforget	1.7	delete([dirMat 'lock_budg']);
172	gforget	1.1	test1=1;
173			fprintf(['pre-processing : completed for budget \n\n']);
174			end;
175
176			while ~test1&doBudget;%this process will wait for pre-processing to complete
177			fprintf(['waiting 30s more for removal of ' dirMat 'lock_budg \n']);
178	gforget	1.2	fprintf(['- That should happen automatically after pre-processing is complete \n']);
179			fprintf(['- But if a previous session was interupted, you may need to stop this one, \n ']);
180			fprintf([' remove ' dirMat 'lock_budg manually, and start over. \n\n']);
181	gforget	1.1	test1=isempty(dir([dirMat 'lock_budg']));
182			pause(30);
183			end;
184