profilesMatlabProcessing/profiles_IO_external/profiles_read_odv.m

function [varargout]=profile_read_odv(dataset,nf,m,varargin)
% read seal data in the odv spreadsheet format
%
% if m=0 :
%  return the information on the nf-th file referenced in
%  dataset.fileInList.
%   [nprofiles,struct_odv]=profile_read_odv(dataset,nf,0);
%       nprofiles: number of profiles in the nf-th file
%       struct_odv: odv data in a struct variable
%
% if m~=0 :
%  return the m-th profile from the nf-th file referenced in
%  dataset.fileInList.
%   profileCur=profile_read_odv(dataset,nf,m,struct_odv);
%
% profileCur = (m-th profile of nf-th file)
%      pnum_txt: '5900841'
%           ymd: 20060101
%           hms: 3207
%           lat: -46.709
%           lon: 137.501
%         direc: 1
%             t: [1x115 single]
%             s: [1x115 single]
%             p: [1x115 single]
%         t_ERR: [1x115 single]
%         s_ERR: [1x115 single]
%     PorZisBAD: 0
%


fileIn=[dataset.dirIn dataset.fileInList(nf).name];

if m==0;

    % return the number of profiles in the file
    % also build an index of first-line prosition of profiles: index_prof
    fid_cur=fopen(fileIn,'rt');
    nprofiles=0;index_prof=[];
    
    % jump comment line at the beginning
    line_cur = fgetl(fid_cur);
    while ~feof(fid_cur) & strcmp(line_cur(1:2),'//')
        line_cur = fgetl(fid_cur);
        continue
    end
    
    % analyse data description line
    I=[0 find(double(line_cur)==9) length(line_cur)+1];
    format=[];
    % column index for (in this order) :
    %   Cruise, Station, Date, Lon, Lat, Dep, Dep_qv, Temp,
    %   Temp_qv, Sal, Sal_qv
    for ii=1:length(I)-1,
        field=line_cur(I(ii)+1:I(ii+1)-1);
        field=field(1:min([6 length(field)]));
        switch field
            case 'Cruise'
                format(1)=ii;
            case 'Statio'
                format(2)=ii;
            case 'yyyy-m'
                format(3)=ii;
            case {'Longit'}
                format(4)=ii;
            case {'Latitu'}
                format(5)=ii;
            case {'Depth '}
                format([6 7])=[ii ii+1];
            case {'Temper'}
                format([8 9])=[ii ii+1];
            case {'Salini'}
                format([10 11])=[ii ii+1];
        end
    end
    
    % init struct_odv
    struct_odv=[];
    
    % read end of the textfile
    nlines=0;nbuffer=100000;niter=0;
    while ~feof(fid_cur);
        niter=niter+1;
        data_odv=cell(length(format)-1,nbuffer);
        for ii=1:nbuffer,
            tline=fgets(fid_cur);
            nlines=nlines+1;
            I_sep=[0 find(double(tline)==9)];
            data_odv{1,ii}=[ tline(I_sep(format(1))+1:I_sep(format(1)+1)-1) '//' ...
                tline(I_sep(format(2))+1:I_sep(format(2)+1)-1) ];
            data_odv{2,ii}=tline(I_sep(format(3))+1:I_sep(format(3)+1)-1);
            for kk=4:length(format),
                I=I_sep(format(kk))+1:I_sep(format(kk)+1)-1;
                if length(I)>1,
                    data_odv{kk-1,ii}=tline(I);
                elseif isempty(I),
                    data_odv{kk-1,ii}='99999';
                else
                    val=tline(I);
                    if val==0,
                        data_odv{kk-1,ii}='99999';
                    else
                        data_odv{kk-1,ii}=val;
                    end
                end
            end
            if feof(fid_cur), break, end
        end
        data_odv=data_odv(:,1:ii);
        data=data_odv(3:length(format)-1,:);
        datanum=sscanf(sprintf('%s,',data{:}), '%g,');
        datanum(datanum==99999)=NaN;
        datanum=reshape(datanum(:),8,ii);

        fprintf('%d lines\n',nlines);
        if niter==1,
            pnum_txt=data_odv(1,:);
            Date=data_odv(2,:);
            Data=datanum;
        else
            pnum_txt=[pnum_txt data_odv(1,:)];
            Date=[Date data_odv(2,:)];
            Data=[Data datanum];
        end
        
    end
    fclose(fid_cur);    

    % get rid of empty indicator string
    [pnum_txt_list,Iprof]=unique(pnum_txt);
    for kk=1:length(pnum_txt),
        if strcmp(pnum_txt_list{kk},'//'),
            pnum_txt_list=pnum_txt_list(setdiff(1:length(pnum_txt_list),kk));
            Iprof(kk)=[];
            break
        end
    end
    
    % sort index of profile
    [Iprof_sort,J]=sort(Iprof);
    pnum_txt_sort=pnum_txt_list(J);
    
    % write index of profile
    Iprof_sort=[Iprof_sort nlines+1];
    for kk=1:length(Iprof_sort)-1,
        for ii=Iprof_sort(kk):Iprof_sort(kk+1)-1,
            struct_odv.Iprof(ii)=kk;
        end
    end
    struct_odv.pnum_txt=pnum_txt;
    struct_odv.Data=Data;
    struct_odv.Date=Date;
    
    % number of profiles
    nprofiles=length(Iprof);

    varargout(1) = {nprofiles};
    varargout(2)= {struct_odv};
    
    
else;%if m==0;
    
    % data_odv: struct of strings: 
    struct_odv=varargin{1};
    
    % profile coordinates
    Iprof=find(struct_odv.Iprof==m);
    pnum_txt=struct_odv.pnum_txt{Iprof(1)};
    lon=struct_odv.Data(1,Iprof(1));
    lat=struct_odv.Data(2,Iprof(1));
    strdate=struct_odv.Date{Iprof(1)};
    if length(strdate)<10, varargout = {[]}; return; end
    ymd=str2num(strdate([1:4 6:7 9:10]));
    hms=0;  
    if length(strdate)==19, hms=str2num(strdate([12:13 15:16 18:19])); end
    if length(strdate)==16, hms=str2num(strdate([12:13 15:16]))*100; end
    if length(strdate)==13, hms=str2num(strdate([12:13]))*10000; end

    %case when necessary information missing: dont retrieve profile
    if isempty(lon)|isempty(lat)|isempty(ymd), varargout = {[]}; return; end
    if lon < 0; lon=lon+360; end;
    
    % get T/S data
    z=struct_odv.Data(3,Iprof);
    z_qv=struct_odv.Data(4,Iprof);z_qv(isempty(z_qv))=1;
    t=struct_odv.Data(5,Iprof);
    t_qv=struct_odv.Data(6,Iprof);t_qv(isempty(t_qv))=1;
    s=struct_odv.Data(7,Iprof);
    s_qv=struct_odv.Data(8,Iprof);s_qv(isempty(s_qv))=1;

    I=find(isnan(z)|z_qv~=0);
    z(I)=[]; t(I)=[]; s(I)=[]; t_qv(I)=[]; s_qv(I)=[];
    t(t_qv~=0)=NaN; s(s_qv~=0)=NaN;
    t_ERR=t*0; s_ERR=s*0; 
    
    direc=2;
    PorZisBAD=0;
    
    profileCur.pnum_txt=pnum_txt;
    profileCur.ymd=ymd; profileCur.hms=hms;
    profileCur.lat=lat; profileCur.lon=lon;
    profileCur.direc=direc;
    profileCur.PorZisBAD=PorZisBAD;
    profileCur.t=t;
    profileCur.s=s;
    profileCur.z=z;
    profileCur.t_ERR=t_ERR;
    profileCur.s_ERR=s_ERR;
    
    varargout = {profileCur};
    
end;


1	roquet	1.2	function [varargout]=profile_read_odv(dataset,nf,m,varargin)
2	roquet	1.1	% read seal data in the odv spreadsheet format
3			%
4			% if m=0 :
5			% return the information on the nf-th file referenced in
6			% dataset.fileInList.
7			% [nprofiles,struct_odv]=profile_read_odv(dataset,nf,0);
8			% nprofiles: number of profiles in the nf-th file
9			% struct_odv: odv data in a struct variable
10			%
11			% if m~=0 :
12			% return the m-th profile from the nf-th file referenced in
13			% dataset.fileInList.
14			% profileCur=profile_read_odv(dataset,nf,m,struct_odv);
15			%
16			% profileCur = (m-th profile of nf-th file)
17			% pnum_txt: '5900841'
18			% ymd: 20060101
19			% hms: 3207
20			% lat: -46.709
21			% lon: 137.501
22			% direc: 1
23			% t: [1x115 single]
24			% s: [1x115 single]
25			% p: [1x115 single]
26			% t_ERR: [1x115 single]
27			% s_ERR: [1x115 single]
28			% PorZisBAD: 0
29			%
30
31
32			fileIn=[dataset.dirIn dataset.fileInList(nf).name];
33
34			if m==0;
35
36			% return the number of profiles in the file
37			% also build an index of first-line prosition of profiles: index_prof
38			fid_cur=fopen(fileIn,'rt');
39			nprofiles=0;index_prof=[];
40
41			% jump comment line at the beginning
42			line_cur = fgetl(fid_cur);
43			while ~feof(fid_cur) & strcmp(line_cur(1:2),'//')
44			line_cur = fgetl(fid_cur);
45			continue
46			end
47
48			% analyse data description line
49			I=[0 find(double(line_cur)==9) length(line_cur)+1];
50			format=[];
51			% column index for (in this order) :
52			% Cruise, Station, Date, Lon, Lat, Dep, Dep_qv, Temp,
53			% Temp_qv, Sal, Sal_qv
54			for ii=1:length(I)-1,
55			field=line_cur(I(ii)+1:I(ii+1)-1);
56			field=field(1:min([6 length(field)]));
57			switch field
58			case 'Cruise'
59			format(1)=ii;
60			case 'Statio'
61			format(2)=ii;
62			case 'yyyy-m'
63			format(3)=ii;
64			case {'Longit'}
65			format(4)=ii;
66			case {'Latitu'}
67			format(5)=ii;
68			case {'Depth '}
69			format([6 7])=[ii ii+1];
70			case {'Temper'}
71			format([8 9])=[ii ii+1];
72			case {'Salini'}
73			format([10 11])=[ii ii+1];
74			end
75			end
76
77			% init struct_odv
78			struct_odv=[];
79
80			% read end of the textfile
81	roquet	1.2	nlines=0;nbuffer=100000;niter=0;
82	roquet	1.1	while ~feof(fid_cur);
83	roquet	1.2	niter=niter+1;
84			data_odv=cell(length(format)-1,nbuffer);
85			for ii=1:nbuffer,
86			tline=fgets(fid_cur);
87	roquet	1.1	nlines=nlines+1;
88			I_sep=[0 find(double(tline)==9)];
89	roquet	1.2	data_odv{1,ii}=[ tline(I_sep(format(1))+1:I_sep(format(1)+1)-1) '//' ...
90			tline(I_sep(format(2))+1:I_sep(format(2)+1)-1) ];
91			data_odv{2,ii}=tline(I_sep(format(3))+1:I_sep(format(3)+1)-1);
92			for kk=4:length(format),
93			I=I_sep(format(kk))+1:I_sep(format(kk)+1)-1;
94			if length(I)>1,
95			data_odv{kk-1,ii}=tline(I);
96			elseif isempty(I),
97			data_odv{kk-1,ii}='99999';
98			else
99			val=tline(I);
100			if val==0,
101			data_odv{kk-1,ii}='99999';
102			else
103			data_odv{kk-1,ii}=val;
104			end
105			end
106	roquet	1.1	end
107			if feof(fid_cur), break, end
108			end
109	roquet	1.2	data_odv=data_odv(:,1:ii);
110			data=data_odv(3:length(format)-1,:);
111			datanum=sscanf(sprintf('%s,',data{:}), '%g,');
112			datanum(datanum==99999)=NaN;
113			datanum=reshape(datanum(:),8,ii);
114
115			fprintf('%d lines\n',nlines);
116			if niter==1,
117			pnum_txt=data_odv(1,:);
118			Date=data_odv(2,:);
119			Data=datanum;
120	roquet	1.1	else
121	roquet	1.2	pnum_txt=[pnum_txt data_odv(1,:)];
122			Date=[Date data_odv(2,:)];
123			Data=[Data datanum];
124	roquet	1.1	end
125
126			end
127	roquet	1.2	fclose(fid_cur);
128
129	roquet	1.1	% get rid of empty indicator string
130	roquet	1.2	[pnum_txt_list,Iprof]=unique(pnum_txt);
131	roquet	1.1	for kk=1:length(pnum_txt),
132	roquet	1.2	if strcmp(pnum_txt_list{kk},'//'),
133			pnum_txt_list=pnum_txt_list(setdiff(1:length(pnum_txt_list),kk));
134	roquet	1.1	Iprof(kk)=[];
135			break
136			end
137			end
138
139			% sort index of profile
140			[Iprof_sort,J]=sort(Iprof);
141	roquet	1.2	pnum_txt_sort=pnum_txt_list(J);
142	roquet	1.1
143			% write index of profile
144			Iprof_sort=[Iprof_sort nlines+1];
145			for kk=1:length(Iprof_sort)-1,
146			for ii=Iprof_sort(kk):Iprof_sort(kk+1)-1,
147			struct_odv.Iprof(ii)=kk;
148			end
149			end
150	roquet	1.2	struct_odv.pnum_txt=pnum_txt;
151			struct_odv.Data=Data;
152			struct_odv.Date=Date;
153
154	roquet	1.1	% number of profiles
155			nprofiles=length(Iprof);
156
157			varargout(1) = {nprofiles};
158			varargout(2)= {struct_odv};
159
160
161			else;%if m==0;
162
163			% data_odv: struct of strings:
164			struct_odv=varargin{1};
165
166			% profile coordinates
167			Iprof=find(struct_odv.Iprof==m);
168			pnum_txt=struct_odv.pnum_txt{Iprof(1)};
169	roquet	1.2	lon=struct_odv.Data(1,Iprof(1));
170			lat=struct_odv.Data(2,Iprof(1));
171	roquet	1.1	strdate=struct_odv.Date{Iprof(1)};
172			if length(strdate)<10, varargout = {[]}; return; end
173			ymd=str2num(strdate([1:4 6:7 9:10]));
174			hms=0;
175			if length(strdate)==19, hms=str2num(strdate([12:13 15:16 18:19])); end
176			if length(strdate)==16, hms=str2num(strdate([12:13 15:16]))*100; end
177			if length(strdate)==13, hms=str2num(strdate([12:13]))*10000; end
178
179			%case when necessary information missing: dont retrieve profile
180			if isempty(lon)\|isempty(lat)\|isempty(ymd), varargout = {[]}; return; end
181			if lon < 0; lon=lon+360; end;
182
183			% get T/S data
184	roquet	1.2	z=struct_odv.Data(3,Iprof);
185			z_qv=struct_odv.Data(4,Iprof);z_qv(isempty(z_qv))=1;
186			t=struct_odv.Data(5,Iprof);
187			t_qv=struct_odv.Data(6,Iprof);t_qv(isempty(t_qv))=1;
188			s=struct_odv.Data(7,Iprof);
189			s_qv=struct_odv.Data(8,Iprof);s_qv(isempty(s_qv))=1;
190	roquet	1.1
191			I=find(isnan(z)\|z_qv~=0);
192			z(I)=[]; t(I)=[]; s(I)=[]; t_qv(I)=[]; s_qv(I)=[];
193			t(t_qv~=0)=NaN; s(s_qv~=0)=NaN;
194			t_ERR=t0; s_ERR=s0;
195
196			direc=2;
197			PorZisBAD=0;
198
199			profileCur.pnum_txt=pnum_txt;
200			profileCur.ymd=ymd; profileCur.hms=hms;
201			profileCur.lat=lat; profileCur.lon=lon;
202			profileCur.direc=direc;
203			profileCur.PorZisBAD=PorZisBAD;
204			profileCur.t=t;
205			profileCur.s=s;
206			profileCur.z=z;
207			profileCur.t_ERR=t_ERR;
208			profileCur.s_ERR=s_ERR;
209
210			varargout = {profileCur};
211
212			end;
213
214
215