profilesMatlabProcessing/profiles_stats/MITprof_stats_load.m

function [MITprof]=MITprof_stats_load(dirData,listData,varCur,varargin);
%[MITprof]=MITprof_stats_load(dirData,listData,varCur,varargin);
%object: loads a series of MITprof files, and computes
%          normalized misfits for one variable
%input:  dirData is the data directory name
%        listData is the data file list (e.g. {'argo_in*'} or {'argo_in*','argo_at*'} )
%        varCur is 'T' or 'S'
%optional :
% EITHER normFactor (optional; double) is the normalization factor (1./prof_?weight by default)
%   OR   varSpec (optional; char) is e.g. 'prof_T' or 'prof_Testim'
%   OR   varSpec,true to remask varSpec according to e.g. prof_T & prof_Testim & prof_Tweight
%output: MITprof.prof is the normalized misfit
%note:   by assumption, all of the files in listData must share the same vertical grid

normFactor=[]; varSpec=''; doRemask=false;
if nargin>3;
    if isnumeric(varargin{1}); normFactor=varargin{1}; end;
    if ischar(varargin{1}); varSpec=varargin{1}; end;
end;
if nargin>4;
    if islogical(varargin{2}); doRemask=varargin{2}; end;
end;

useExtendedProfDepth=0;

%develop listData (that may include wildcards)
listData_bak=listData;
listData={};
for ii=1:length(listData_bak);
    tmp1=dir([dirData listData_bak{ii}]);
    for jj=1:length(tmp1);
        ii2=length(listData)+1;
        listData{ii2}=tmp1(jj).name;
    end;
end;
%avoid duplicates
listData=unique(listData);

%loop over files
for iFile=1:length(listData);
    fileData=dir([dirData listData{iFile}]);
    fileData=fileData.name;
    fprintf(['loading ' varCur ' from ' fileData '\n']);
    MITprofCur=MITprof_load([dirData fileData]);

    if isfield(MITprofCur,'prof_TeccoV4R2')&~isfield(MITprofCur,'prof_Testim');
       MITprofCur.prof_Testim=MITprofCur.prof_TeccoV4R2;
    end;
    if isfield(MITprofCur,'prof_SeccoV4R2')&~isfield(MITprofCur,'prof_Sestim');
       MITprofCur.prof_Sestim=MITprofCur.prof_SeccoV4R2;
    end;
    
    %fixes:
    if ~isfield(MITprofCur,['prof_' varCur]);
        tmp1=NaN*ones(MITprofCur.np,MITprofCur.nr);
        eval(['MITprofCur.prof_' varCur '=tmp1;']);
        eval(['MITprofCur.prof_' varCur 'estim=tmp1;']);
        eval(['MITprofCur.prof_' varCur 'weight=tmp1;']); 
        eval(['MITprofCur.prof_' varCur 'flag=[];']);
    end;
    
    if ~isfield(MITprofCur,['prof_' varCur 'weight']);
        eval(['MITprofCur.prof_' varCur 'weight=1+0*MITprofCur.prof_' varCur ';']);
    end;
    
    %replace weights with normFactor:
    if ~isempty(normFactor);
        eval(['tmp1=MITprofCur.prof_' varCur 'weight;']);
        tmp1(tmp1>0)=normFactor;
        eval(['MITprofCur.prof_' varCur 'weight=tmp1;']);
    end;
    
    %map variable of interest to MITprofCur.prof:
    if isempty(varSpec);        
        eval(['tmp1=(MITprofCur.prof_' varCur 'estim-MITprofCur.prof_' varCur ')' ...
            '.*sqrt(MITprofCur.prof_' varCur 'weight);']);
        tmp1(tmp1==0)=NaN;
    else;
        eval(['tmp1=MITprofCur.' varSpec ';']);
        if doRemask;
            eval(['tmp1(isnan(MITprofCur.prof_' varCur '))=NaN;']);
            eval(['tmp1(isnan(MITprofCur.prof_' varCur 'estim))=NaN;']);
            eval(['tmp1(isnan(MITprofCur.prof_' varCur 'weight))=NaN;']);
            eval(['tmp1(MITprofCur.prof_' varCur 'weight==0)=NaN;']);
        end;
    end;
    %                
    MITprofCur.prof=tmp1;
    
    %remove bad profiles:
    if isfield(MITprofCur,'prof_flag');
        tmp1=MITprofCur.prof_flag; tmp1=isnan(tmp1);
        MITprofCur.prof_flag(tmp1)=0;
        MITprofCur=MITprof_subset(MITprofCur,'flag',0);
        MITprofCur=rmfield(MITprofCur,'prof_flag');
    end;

    %remove un-needed variables:
    listKeep={'prof_YYYYMMDD','prof_HHMMSS','prof_lon','prof_lat',...
        'prof_depth','prof_date','prof_basin','prof_point','prof_descr',...
        'list_descr','nd','np','nr','prof'};
    listField=fieldnames(MITprofCur);
    for iField=1:length(listField);
        if sum(strcmp(listKeep,listField{iField}))==0;
            MITprofCur=rmfield(MITprofCur,listField{iField}); 
        end;
    end;
    
    %bug in 2013b? the following return wrong dates when SS='60'
    %MITprofCur.prof_date=datenum(num2str(MITprofCur.prof_YYYYMMDD*1e6+MITprofCur.prof_HHMMSS),'yyyymmddHHMMSS');
    %old:     ii=find(MITprofCur.prof_date<datenum(1992,1,1)|MITprofCur.prof_date>datenum(2008,12,27)); MITprofCur.prof(ii,:)=NaN;
    
    %extend prof_depth if needed:
    if iFile>1;
        prof_depth=union(MITprof.prof_depth,MITprofCur.prof_depth,'rows');
        for kk=1:2;
            if kk==1; tmpProf=MITprof; else; tmpProf=MITprofCur; end;
            if length(prof_depth)>length(tmpProf.prof_depth);
                tmp1=tmpProf.prof;
                tmp2=NaN*zeros(tmpProf.np,length(prof_depth));
                ii=NaN*length(tmpProf.prof_depth);
                for jj=1:length(tmpProf.prof_depth);
                    ii(jj)=find(prof_depth==tmpProf.prof_depth(jj));
                end;
                tmp2(:,ii)=tmpProf.prof;
                tmpProf.prof=tmp2;
                tmpProf.nr=length(prof_depth);
                tmpProf.prof_depth=prof_depth;
            end;
            if kk==1; MITprof=tmpProf; else; MITprofCur=tmpProf; end;
        end;
    end;
            
    if iFile==1;
        MITprof=MITprofCur;
    else;
        MITprof=MITprof_concat(MITprof,MITprofCur);
    end;
    clear MITprofCur;
end;


1	gforget	1.1	function [MITprof]=MITprof_stats_load(dirData,listData,varCur,varargin);
2			%[MITprof]=MITprof_stats_load(dirData,listData,varCur,varargin);
3	gforget	1.4	%object: loads a series of MITprof files, and computes
4			% normalized misfits for one variable
5			%input: dirData is the data directory name
6			% listData is the data file list (e.g. {'argo_in'} or {'argo_in','argo_at*'} )
7			% varCur is 'T' or 'S'
8	gforget	1.7	%optional :
9			% EITHER normFactor (optional; double) is the normalization factor (1./prof_?weight by default)
10			% OR varSpec (optional; char) is e.g. 'prof_T' or 'prof_Testim'
11	gforget	1.8	% OR varSpec,true to remask varSpec according to e.g. prof_T & prof_Testim & prof_Tweight
12	gforget	1.4	%output: MITprof.prof is the normalized misfit
13			%note: by assumption, all of the files in listData must share the same vertical grid
14	gforget	1.1
15	gforget	1.8	normFactor=[]; varSpec=''; doRemask=false;
16			if nargin>3;
17			if isnumeric(varargin{1}); normFactor=varargin{1}; end;
18			if ischar(varargin{1}); varSpec=varargin{1}; end;
19			end;
20			if nargin>4;
21			if islogical(varargin{2}); doRemask=varargin{2}; end;
22	gforget	1.6	end;
23	gforget	1.1
24	gforget	1.8	useExtendedProfDepth=0;
25	gforget	1.2
26	gforget	1.5	%develop listData (that may include wildcards)
27			listData_bak=listData;
28			listData={};
29			for ii=1:length(listData_bak);
30			tmp1=dir([dirData listData_bak{ii}]);
31			for jj=1:length(tmp1);
32	gforget	1.8	ii2=length(listData)+1;
33			listData{ii2}=tmp1(jj).name;
34	gforget	1.5	end;
35			end;
36			%avoid duplicates
37	gforget	1.8	listData=unique(listData);
38	gforget	1.5
39			%loop over files
40	gforget	1.1	for iFile=1:length(listData);
41			fileData=dir([dirData listData{iFile}]);
42	gforget	1.5	fileData=fileData.name;
43			fprintf(['loading ' varCur ' from ' fileData '\n']);
44	gforget	1.1	MITprofCur=MITprof_load([dirData fileData]);
45	gforget	1.9
46			if isfield(MITprofCur,'prof_TeccoV4R2')&~isfield(MITprofCur,'prof_Testim');
47			MITprofCur.prof_Testim=MITprofCur.prof_TeccoV4R2;
48			end;
49			if isfield(MITprofCur,'prof_SeccoV4R2')&~isfield(MITprofCur,'prof_Sestim');
50			MITprofCur.prof_Sestim=MITprofCur.prof_SeccoV4R2;
51			end;
52	gforget	1.1
53	gforget	1.8	%fixes:
54			if ~isfield(MITprofCur,['prof_' varCur]);
55			tmp1=NaN*ones(MITprofCur.np,MITprofCur.nr);
56			eval(['MITprofCur.prof_' varCur '=tmp1;']);
57			eval(['MITprofCur.prof_' varCur 'estim=tmp1;']);
58			eval(['MITprofCur.prof_' varCur 'weight=tmp1;']);
59			eval(['MITprofCur.prof_' varCur 'flag=[];']);
60	gforget	1.1	end;
61	gforget	1.8
62	gforget	1.7	if ~isfield(MITprofCur,['prof_' varCur 'weight']);
63			eval(['MITprofCur.prof_' varCur 'weight=1+0*MITprofCur.prof_' varCur ';']);
64			end;
65	gforget	1.1
66	gforget	1.8	%replace weights with normFactor:
67			if ~isempty(normFactor);
68	gforget	1.1	eval(['tmp1=MITprofCur.prof_' varCur 'weight;']);
69			tmp1(tmp1>0)=normFactor;
70			eval(['MITprofCur.prof_' varCur 'weight=tmp1;']);
71			end;
72
73	gforget	1.8	%map variable of interest to MITprofCur.prof:
74			if isempty(varSpec);
75			eval(['tmp1=(MITprofCur.prof_' varCur 'estim-MITprofCur.prof_' varCur ')' ...
76			'.*sqrt(MITprofCur.prof_' varCur 'weight);']);
77			tmp1(tmp1==0)=NaN;
78	gforget	1.1	else;
79	gforget	1.8	eval(['tmp1=MITprofCur.' varSpec ';']);
80			if doRemask;
81			eval(['tmp1(isnan(MITprofCur.prof_' varCur '))=NaN;']);
82			eval(['tmp1(isnan(MITprofCur.prof_' varCur 'estim))=NaN;']);
83			eval(['tmp1(isnan(MITprofCur.prof_' varCur 'weight))=NaN;']);
84			eval(['tmp1(MITprofCur.prof_' varCur 'weight==0)=NaN;']);
85			end;
86	gforget	1.6	end;
87	gforget	1.8	%
88	gforget	1.6	MITprofCur.prof=tmp1;
89	gforget	1.3
90	gforget	1.8	%remove bad profiles:
91			if isfield(MITprofCur,'prof_flag');
92			tmp1=MITprofCur.prof_flag; tmp1=isnan(tmp1);
93			MITprofCur.prof_flag(tmp1)=0;
94			MITprofCur=MITprof_subset(MITprofCur,'flag',0);
95			MITprofCur=rmfield(MITprofCur,'prof_flag');
96			end;
97
98			%remove un-needed variables:
99			listKeep={'prof_YYYYMMDD','prof_HHMMSS','prof_lon','prof_lat',...
100			'prof_depth','prof_date','prof_basin','prof_point','prof_descr',...
101			'list_descr','nd','np','nr','prof'};
102			listField=fieldnames(MITprofCur);
103			for iField=1:length(listField);
104			if sum(strcmp(listKeep,listField{iField}))==0;
105			MITprofCur=rmfield(MITprofCur,listField{iField});
106			end;
107	gforget	1.1	end;
108
109	gforget	1.8	%bug in 2013b? the following return wrong dates when SS='60'
110			%MITprofCur.prof_date=datenum(num2str(MITprofCur.prof_YYYYMMDD*1e6+MITprofCur.prof_HHMMSS),'yyyymmddHHMMSS');
111	gforget	1.3	%old: ii=find(MITprofCur.prof_date<datenum(1992,1,1)\|MITprofCur.prof_date>datenum(2008,12,27)); MITprofCur.prof(ii,:)=NaN;
112	gforget	1.1
113	gforget	1.8	%extend prof_depth if needed:
114			if iFile>1;
115			prof_depth=union(MITprof.prof_depth,MITprofCur.prof_depth,'rows');
116			for kk=1:2;
117			if kk==1; tmpProf=MITprof; else; tmpProf=MITprofCur; end;
118			if length(prof_depth)>length(tmpProf.prof_depth);
119			tmp1=tmpProf.prof;
120			tmp2=NaN*zeros(tmpProf.np,length(prof_depth));
121			ii=NaN*length(tmpProf.prof_depth);
122			for jj=1:length(tmpProf.prof_depth);
123			ii(jj)=find(prof_depth==tmpProf.prof_depth(jj));
124			end;
125			tmp2(:,ii)=tmpProf.prof;
126			tmpProf.prof=tmp2;
127			tmpProf.nr=length(prof_depth);
128			tmpProf.prof_depth=prof_depth;
129			end;
130			if kk==1; MITprof=tmpProf; else; MITprofCur=tmpProf; end;
131			end;
132	gforget	1.1	end;
133	gforget	1.8
134	gforget	1.1	if iFile==1;
135			MITprof=MITprofCur;
136			else;
137			MITprof=MITprof_concat(MITprof,MITprofCur);
138			end;
139			clear MITprofCur;
140			end;
141
142