profilesMatlabProcessing/profiles_stats/MITprof_stats_load.m

function [MITprof]=MITprof_stats_load(dirData,listData,varCur,varargin);
%[MITprof]=MITprof_stats_load(dirData,listData,varCur,varargin);
%object: loads a series of MITprof files, and computes
%          normalized misfits for one variable
%input:  dirData is the data directory name
%        listData is the data file list (e.g. {'argo_in*'} or {'argo_in*','argo_at*'} )
%        varCur is 'T' or 'S'
%optional :
% EITHER normFactor (optional; double) is the normalization factor (1./prof_?weight by default)
%   OR   varSpec (optional; char) is e.g. 'prof_T' or 'prof_Testim'
%   OR   varSpec,true to remask varSpec according to e.g. prof_T & prof_Testim & prof_Tweight
%output: MITprof.prof is the normalized misfit
%note:   by assumption, all of the files in listData must share the same vertical grid

normFactor=[]; varSpec=''; doRemask=false;
if nargin>3;
    if isnumeric(varargin{1}); normFactor=varargin{1}; end;
    if ischar(varargin{1}); varSpec=varargin{1}; end;
end;
if nargin>4;
    if islogical(varargin{2}); doRemask=varargin{2}; end;
end;

useExtendedProfDepth=0;

%develop listData (that may include wildcards)
listData_bak=listData;
listData={};
for ii=1:length(listData_bak);
    tmp1=dir([dirData listData_bak{ii}]);
    for jj=1:length(tmp1);
        ii2=length(listData)+1;
        listData{ii2}=tmp1(jj).name;
    end;
end;
%avoid duplicates
listData=unique(listData);

%loop over files
for iFile=1:length(listData);
    fileData=dir([dirData listData{iFile}]);
    fileData=fileData.name;
    fprintf(['loading ' varCur ' from ' fileData '\n']);
    MITprofCur=MITprof_load([dirData fileData]);
    
    %fixes:
    if ~isfield(MITprofCur,['prof_' varCur]);
        tmp1=NaN*ones(MITprofCur.np,MITprofCur.nr);
        eval(['MITprofCur.prof_' varCur '=tmp1;']);
        eval(['MITprofCur.prof_' varCur 'estim=tmp1;']);
        eval(['MITprofCur.prof_' varCur 'weight=tmp1;']); 
        eval(['MITprofCur.prof_' varCur 'flag=[];']);
    end;
    
    if ~isfield(MITprofCur,['prof_' varCur 'weight']);
        eval(['MITprofCur.prof_' varCur 'weight=1+0*MITprofCur.prof_' varCur ';']);
    end;
    
    %replace weights with normFactor:
    if ~isempty(normFactor);
        eval(['tmp1=MITprofCur.prof_' varCur 'weight;']);
        tmp1(tmp1>0)=normFactor;
        eval(['MITprofCur.prof_' varCur 'weight=tmp1;']);
    end;
    
    %map variable of interest to MITprofCur.prof:
    if isempty(varSpec);        
        eval(['tmp1=(MITprofCur.prof_' varCur 'estim-MITprofCur.prof_' varCur ')' ...
            '.*sqrt(MITprofCur.prof_' varCur 'weight);']);
        tmp1(tmp1==0)=NaN;
    else;
        eval(['tmp1=MITprofCur.' varSpec ';']);
        if doRemask;
            eval(['tmp1(isnan(MITprofCur.prof_' varCur '))=NaN;']);
            eval(['tmp1(isnan(MITprofCur.prof_' varCur 'estim))=NaN;']);
            eval(['tmp1(isnan(MITprofCur.prof_' varCur 'weight))=NaN;']);
            eval(['tmp1(MITprofCur.prof_' varCur 'weight==0)=NaN;']);
        end;
    end;
    %                
    MITprofCur.prof=tmp1;
    
    %remove bad profiles:
    if isfield(MITprofCur,'prof_flag');
        tmp1=MITprofCur.prof_flag; tmp1=isnan(tmp1);
        MITprofCur.prof_flag(tmp1)=0;
        MITprofCur=MITprof_subset(MITprofCur,'flag',0);
        MITprofCur=rmfield(MITprofCur,'prof_flag');
    end;

    %remove un-needed variables:
    listKeep={'prof_YYYYMMDD','prof_HHMMSS','prof_lon','prof_lat',...
        'prof_depth','prof_date','prof_basin','prof_point','prof_descr',...
        'list_descr','nd','np','nr','prof'};
    listField=fieldnames(MITprofCur);
    for iField=1:length(listField);
        if sum(strcmp(listKeep,listField{iField}))==0;
            MITprofCur=rmfield(MITprofCur,listField{iField}); 
        end;
    end;
    
    %bug in 2013b? the following return wrong dates when SS='60'
    %MITprofCur.prof_date=datenum(num2str(MITprofCur.prof_YYYYMMDD*1e6+MITprofCur.prof_HHMMSS),'yyyymmddHHMMSS');
    %old:     ii=find(MITprofCur.prof_date<datenum(1992,1,1)|MITprofCur.prof_date>datenum(2008,12,27)); MITprofCur.prof(ii,:)=NaN;
    
    %extend prof_depth if needed:
    if iFile>1;
        prof_depth=union(MITprof.prof_depth,MITprofCur.prof_depth,'rows');
        for kk=1:2;
            if kk==1; tmpProf=MITprof; else; tmpProf=MITprofCur; end;
            if length(prof_depth)>length(tmpProf.prof_depth);
                tmp1=tmpProf.prof;
                tmp2=NaN*zeros(tmpProf.np,length(prof_depth));
                ii=NaN*length(tmpProf.prof_depth);
                for jj=1:length(tmpProf.prof_depth);
                    ii(jj)=find(prof_depth==tmpProf.prof_depth(jj));
                end;
                tmp2(:,ii)=tmpProf.prof;
                tmpProf.prof=tmp2;
                tmpProf.nr=length(prof_depth);
                tmpProf.prof_depth=prof_depth;
            end;
            if kk==1; MITprof=tmpProf; else; MITprofCur=tmpProf; end;
        end;
    end;
            
    if iFile==1;
        MITprof=MITprofCur;
    else;
        MITprof=MITprof_concat(MITprof,MITprofCur);
    end;
    clear MITprofCur;
end;


1	gforget	1.1	function [MITprof]=MITprof_stats_load(dirData,listData,varCur,varargin);
2			%[MITprof]=MITprof_stats_load(dirData,listData,varCur,varargin);
3	gforget	1.4	%object: loads a series of MITprof files, and computes
4			% normalized misfits for one variable
5			%input: dirData is the data directory name
6			% listData is the data file list (e.g. {'argo_in'} or {'argo_in','argo_at*'} )
7			% varCur is 'T' or 'S'
8	gforget	1.7	%optional :
9			% EITHER normFactor (optional; double) is the normalization factor (1./prof_?weight by default)
10			% OR varSpec (optional; char) is e.g. 'prof_T' or 'prof_Testim'
11	gforget	1.8	% OR varSpec,true to remask varSpec according to e.g. prof_T & prof_Testim & prof_Tweight
12	gforget	1.4	%output: MITprof.prof is the normalized misfit
13			%note: by assumption, all of the files in listData must share the same vertical grid
14	gforget	1.1
15	gforget	1.8	normFactor=[]; varSpec=''; doRemask=false;
16			if nargin>3;
17			if isnumeric(varargin{1}); normFactor=varargin{1}; end;
18			if ischar(varargin{1}); varSpec=varargin{1}; end;
19			end;
20			if nargin>4;
21			if islogical(varargin{2}); doRemask=varargin{2}; end;
22	gforget	1.6	end;
23	gforget	1.1
24	gforget	1.8	useExtendedProfDepth=0;
25	gforget	1.2
26	gforget	1.5	%develop listData (that may include wildcards)
27			listData_bak=listData;
28			listData={};
29			for ii=1:length(listData_bak);
30			tmp1=dir([dirData listData_bak{ii}]);
31			for jj=1:length(tmp1);
32	gforget	1.8	ii2=length(listData)+1;
33			listData{ii2}=tmp1(jj).name;
34	gforget	1.5	end;
35			end;
36			%avoid duplicates
37	gforget	1.8	listData=unique(listData);
38	gforget	1.5
39			%loop over files
40	gforget	1.1	for iFile=1:length(listData);
41			fileData=dir([dirData listData{iFile}]);
42	gforget	1.5	fileData=fileData.name;
43			fprintf(['loading ' varCur ' from ' fileData '\n']);
44	gforget	1.1	MITprofCur=MITprof_load([dirData fileData]);
45
46	gforget	1.8	%fixes:
47			if ~isfield(MITprofCur,['prof_' varCur]);
48			tmp1=NaN*ones(MITprofCur.np,MITprofCur.nr);
49			eval(['MITprofCur.prof_' varCur '=tmp1;']);
50			eval(['MITprofCur.prof_' varCur 'estim=tmp1;']);
51			eval(['MITprofCur.prof_' varCur 'weight=tmp1;']);
52			eval(['MITprofCur.prof_' varCur 'flag=[];']);
53	gforget	1.1	end;
54	gforget	1.8
55	gforget	1.7	if ~isfield(MITprofCur,['prof_' varCur 'weight']);
56			eval(['MITprofCur.prof_' varCur 'weight=1+0*MITprofCur.prof_' varCur ';']);
57			end;
58	gforget	1.1
59	gforget	1.8	%replace weights with normFactor:
60			if ~isempty(normFactor);
61	gforget	1.1	eval(['tmp1=MITprofCur.prof_' varCur 'weight;']);
62			tmp1(tmp1>0)=normFactor;
63			eval(['MITprofCur.prof_' varCur 'weight=tmp1;']);
64			end;
65
66	gforget	1.8	%map variable of interest to MITprofCur.prof:
67			if isempty(varSpec);
68			eval(['tmp1=(MITprofCur.prof_' varCur 'estim-MITprofCur.prof_' varCur ')' ...
69			'.*sqrt(MITprofCur.prof_' varCur 'weight);']);
70			tmp1(tmp1==0)=NaN;
71	gforget	1.1	else;
72	gforget	1.8	eval(['tmp1=MITprofCur.' varSpec ';']);
73			if doRemask;
74			eval(['tmp1(isnan(MITprofCur.prof_' varCur '))=NaN;']);
75			eval(['tmp1(isnan(MITprofCur.prof_' varCur 'estim))=NaN;']);
76			eval(['tmp1(isnan(MITprofCur.prof_' varCur 'weight))=NaN;']);
77			eval(['tmp1(MITprofCur.prof_' varCur 'weight==0)=NaN;']);
78			end;
79	gforget	1.6	end;
80	gforget	1.8	%
81	gforget	1.6	MITprofCur.prof=tmp1;
82	gforget	1.3
83	gforget	1.8	%remove bad profiles:
84			if isfield(MITprofCur,'prof_flag');
85			tmp1=MITprofCur.prof_flag; tmp1=isnan(tmp1);
86			MITprofCur.prof_flag(tmp1)=0;
87			MITprofCur=MITprof_subset(MITprofCur,'flag',0);
88			MITprofCur=rmfield(MITprofCur,'prof_flag');
89			end;
90
91			%remove un-needed variables:
92			listKeep={'prof_YYYYMMDD','prof_HHMMSS','prof_lon','prof_lat',...
93			'prof_depth','prof_date','prof_basin','prof_point','prof_descr',...
94			'list_descr','nd','np','nr','prof'};
95			listField=fieldnames(MITprofCur);
96			for iField=1:length(listField);
97			if sum(strcmp(listKeep,listField{iField}))==0;
98			MITprofCur=rmfield(MITprofCur,listField{iField});
99			end;
100	gforget	1.1	end;
101
102	gforget	1.8	%bug in 2013b? the following return wrong dates when SS='60'
103			%MITprofCur.prof_date=datenum(num2str(MITprofCur.prof_YYYYMMDD*1e6+MITprofCur.prof_HHMMSS),'yyyymmddHHMMSS');
104	gforget	1.3	%old: ii=find(MITprofCur.prof_date<datenum(1992,1,1)\|MITprofCur.prof_date>datenum(2008,12,27)); MITprofCur.prof(ii,:)=NaN;
105	gforget	1.1
106	gforget	1.8	%extend prof_depth if needed:
107			if iFile>1;
108			prof_depth=union(MITprof.prof_depth,MITprofCur.prof_depth,'rows');
109			for kk=1:2;
110			if kk==1; tmpProf=MITprof; else; tmpProf=MITprofCur; end;
111			if length(prof_depth)>length(tmpProf.prof_depth);
112			tmp1=tmpProf.prof;
113			tmp2=NaN*zeros(tmpProf.np,length(prof_depth));
114			ii=NaN*length(tmpProf.prof_depth);
115			for jj=1:length(tmpProf.prof_depth);
116			ii(jj)=find(prof_depth==tmpProf.prof_depth(jj));
117			end;
118			tmp2(:,ii)=tmpProf.prof;
119			tmpProf.prof=tmp2;
120			tmpProf.nr=length(prof_depth);
121			tmpProf.prof_depth=prof_depth;
122			end;
123			if kk==1; MITprof=tmpProf; else; MITprofCur=tmpProf; end;
124			end;
125	gforget	1.1	end;
126	gforget	1.8
127	gforget	1.1	if iFile==1;
128			MITprof=MITprofCur;
129			else;
130			MITprof=MITprof_concat(MITprof,MITprofCur);
131			end;
132			clear MITprofCur;
133			end;
134
135