profilesMatlabProcessing/profiles_IO_external/profiles_read_odv.m

function [varargout]=profile_read_odv(dataset,nf,m,varargin)
% read seal data in the odv spreadsheet format
%
% if m=0 :
%  return the information on the nf-th file referenced in
%  dataset.fileInList.
%   dataset=profile_read_odv(dataset,nf,0);
%       dataset.nprofiles: number of profiles in the nf-th file
%       dataset.Iprofile: index referencing the profile for each dataline
%
% if m~=0 :
%  return the m-th profile from the nf-th file referenced in
%  dataset.fileInList.
%   profileCur=profile_read_odv(dataset,nf,m);
%
% profileCur = (m-th profile of nf-th file)
%      pnum_txt: '5900841'
%           ymd: 20060101
%           hms: 3207
%           lat: -46.709
%           lon: 137.501
%         direc: 1
%             t: [1x115 single]
%             s: [1x115 single]
%             p: [1x115 single]
%         t_ERR: [1x115 single]
%         s_ERR: [1x115 single]
%         isBAD: 0
%
% IMPORTANT: Use three global variables: pnum Date and Data.

global pnum Date Data

fileIn=[dataset.dirIn dataset.fileInList(nf).name];

if m==0;

    % return the number of profiles in the file
    % also build an index of first-line prosition of profiles: index_prof
    fid_cur=fopen(fileIn,'rt');
    nprofiles=0;index_prof=[];
    
    % jump comment line at the beginning
    line_cur = fgetl(fid_cur);
    while ~feof(fid_cur) & strcmp(line_cur(1:2),'//')
        line_cur = fgetl(fid_cur);
        continue
    end
    
    % analyse data description line
    I=[0 find(double(line_cur)==9) length(line_cur)+1];
    format=[];
    % column index for (in this order) :
    %   Cruise, Station, Date, Lon, Lat, Dep, Dep_qv, Temp,
    %   Temp_qv, Sal, Sal_qv
    for ii=1:length(I)-1,
        field=line_cur(I(ii)+1:I(ii+1)-1);
        field=field(1:min([6 length(field)]));
        switch field
            case 'Cruise'
                format(1)=ii;
            case 'Statio'
                format(2)=ii;
            case 'yyyy-m'
                format(3)=ii;
            case {'Longit'}
                format(4)=ii;
            case {'Latitu'}
                format(5)=ii;
            case {'Depth '}
                format([6 7])=[ii ii+1];
            case {'Temper'}
                format([8 9])=[ii ii+1];
            case {'Salini'}
                format([10 11])=[ii ii+1];
        end
    end
    
    % read end of the textfile
    nlines=0;nbuffer=100000;niter=0;
    while ~feof(fid_cur);
        niter=niter+1;
        data_odv=cell(length(format)-1,nbuffer);
        for ii=1:nbuffer,
            tline=fgets(fid_cur);
            nlines=nlines+1;
            I_sep=[0 find(double(tline)==9)];
            data_odv{1,ii}=[ tline(I_sep(format(1))+1:I_sep(format(1)+1)-1) '//' ...
                tline(I_sep(format(2))+1:I_sep(format(2)+1)-1) ];
            data_odv{2,ii}=tline(I_sep(format(3))+1:I_sep(format(3)+1)-1);
            for kk=4:length(format),
                I=I_sep(format(kk))+1:I_sep(format(kk)+1)-1;
                if length(I)>1,
                    data_odv{kk-1,ii}=tline(I);
                elseif isempty(I),
                    data_odv{kk-1,ii}='99999';
                else
                    val=tline(I);
                    if val==0,
                        data_odv{kk-1,ii}='99999';
                    else
                        data_odv{kk-1,ii}=val;
                    end
                end
            end
            if feof(fid_cur), break, end
        end
        data_odv=data_odv(:,1:ii);
        data=data_odv(3:length(format)-1,:);
        datanum=sscanf(sprintf('%s,',data{:}), '%g,');
        datanum(datanum==99999)=NaN;
        
        %no salinity case
        if dataset.inclS,
            datanum=reshape(datanum(:),8,ii);
        else
            datanum=reshape(datanum(:),6,ii);
        end

        fprintf('%d lines\n',nlines);
        if niter==1,
            pnum=data_odv(1,:);
            Date=data_odv(2,:);
            Data=datanum;
        else
            pnum=[pnum data_odv(1,:)];
            Date=[Date data_odv(2,:)];
            Data=[Data datanum];
        end
        
    end
    fclose(fid_cur);    

    % get rid of empty indicator string
    [pnum_txt_list,Iprof]=unique(pnum);
    if size(Iprof,2)==1; Iprof=Iprof'; end;
    for kk=1:length(pnum),
        if strcmp(pnum_txt_list{kk},'//'),
            pnum_txt_list=pnum_txt_list(setdiff(1:length(pnum_txt_list),kk));
            Iprof(kk)=[];
            break
        end
    end
    
    % sort index of profile
    [Iprof_sort,J]=sort(Iprof);
    pnum_txt_sort=pnum_txt_list(J);
    
    % write index of profile
    Iprof_sort=[Iprof_sort nlines+1];
    Iprofile=zeros(nlines,1);
    for kk=1:length(Iprof_sort)-1,
        for ii=Iprof_sort(kk):Iprof_sort(kk+1)-1,
            Iprofile(ii)=kk;
        end
    end
    dataset.Iprofile=Iprofile;
    
    % number of profiles
    dataset.nprofiles=length(Iprof);

    varargout(1) = {dataset};
    
    
else;%if m==0;
    
    % profile coordinates
    Iprof=find(dataset.Iprofile==m);
    pnum_txt=pnum{Iprof(1)};
    lon=Data(1,Iprof(1));
    lat=Data(2,Iprof(1));
    strdate=Date{Iprof(1)};
    if length(strdate)<10, varargout = {[]}; return; end
    ymd=str2num(strdate([1:4 6:7 9:10]));
    hms=0;  
    if length(strdate)==19, hms=str2num(strdate([12:13 15:16 18:19])); end
    if length(strdate)==16, hms=str2num(strdate([12:13 15:16]))*100; end
    if length(strdate)==13, hms=str2num(strdate([12:13]))*10000; end

    %case when necessary information missing: dont retrieve profile
    if isempty(lon)|isempty(lat)|isempty(ymd), varargout = {[]}; return; end
    if lon < 0; lon=lon+360; end;
    
    % get T/S data
    z=Data(3,Iprof);
    z_qv=Data(4,Iprof);z_qv(isempty(z_qv))=1;
    t=Data(5,Iprof);
    t_qv=Data(6,Iprof);t_qv(isempty(t_qv))=1;
    if dataset.inclS,
        s=Data(7,Iprof);
        s_qv=Data(8,Iprof);s_qv(isempty(s_qv))=1;
    else
        s=t*NaN;  s_qv=t*NaN;   s_ERR=t*NaN;
    end
    
    I=find(isnan(z)|z_qv~=0);
    z(I)=[]; t(I)=[]; t_qv(I)=[]; 
    t(t_qv~=0)=NaN; 
    t_ERR=t*0;
    if dataset.inclS,
        s_qv(I)=[];    s(I)=[]; 
        s(s_qv~=0)=NaN;     s_ERR=s*0;
    end
    z_ERR=t*0;
    
    direc=2;
    isBAD=0;
    
    profileCur.pnum_txt=pnum_txt;
    profileCur.ymd=ymd; profileCur.hms=hms;
    profileCur.lat=lat; profileCur.lon=lon;
    profileCur.direc=direc;
    profileCur.isBAD=isBAD;
    profileCur.T=t;
    profileCur.S=s;
    profileCur.depth=z;
    profileCur.T_ERR=t_ERR;
    profileCur.S_ERR=s_ERR;
    profileCur.depth_ERR=z_ERR;
    
    varargout = {profileCur};
    
end;


1	roquet	1.2	function [varargout]=profile_read_odv(dataset,nf,m,varargin)
2	roquet	1.1	% read seal data in the odv spreadsheet format
3			%
4			% if m=0 :
5			% return the information on the nf-th file referenced in
6			% dataset.fileInList.
7	roquet	1.4	% dataset=profile_read_odv(dataset,nf,0);
8			% dataset.nprofiles: number of profiles in the nf-th file
9			% dataset.Iprofile: index referencing the profile for each dataline
10	roquet	1.1	%
11			% if m~=0 :
12			% return the m-th profile from the nf-th file referenced in
13			% dataset.fileInList.
14	roquet	1.4	% profileCur=profile_read_odv(dataset,nf,m);
15	roquet	1.1	%
16			% profileCur = (m-th profile of nf-th file)
17			% pnum_txt: '5900841'
18			% ymd: 20060101
19			% hms: 3207
20			% lat: -46.709
21			% lon: 137.501
22			% direc: 1
23			% t: [1x115 single]
24			% s: [1x115 single]
25			% p: [1x115 single]
26			% t_ERR: [1x115 single]
27			% s_ERR: [1x115 single]
28	gforget	1.7	% isBAD: 0
29	roquet	1.1	%
30	roquet	1.4	% IMPORTANT: Use three global variables: pnum Date and Data.
31	roquet	1.1
32	roquet	1.4	global pnum Date Data
33	roquet	1.1
34			fileIn=[dataset.dirIn dataset.fileInList(nf).name];
35
36			if m==0;
37
38			% return the number of profiles in the file
39			% also build an index of first-line prosition of profiles: index_prof
40			fid_cur=fopen(fileIn,'rt');
41			nprofiles=0;index_prof=[];
42
43			% jump comment line at the beginning
44			line_cur = fgetl(fid_cur);
45			while ~feof(fid_cur) & strcmp(line_cur(1:2),'//')
46			line_cur = fgetl(fid_cur);
47			continue
48			end
49
50			% analyse data description line
51			I=[0 find(double(line_cur)==9) length(line_cur)+1];
52			format=[];
53			% column index for (in this order) :
54			% Cruise, Station, Date, Lon, Lat, Dep, Dep_qv, Temp,
55			% Temp_qv, Sal, Sal_qv
56			for ii=1:length(I)-1,
57			field=line_cur(I(ii)+1:I(ii+1)-1);
58			field=field(1:min([6 length(field)]));
59			switch field
60			case 'Cruise'
61			format(1)=ii;
62			case 'Statio'
63			format(2)=ii;
64			case 'yyyy-m'
65			format(3)=ii;
66			case {'Longit'}
67			format(4)=ii;
68			case {'Latitu'}
69			format(5)=ii;
70			case {'Depth '}
71			format([6 7])=[ii ii+1];
72			case {'Temper'}
73			format([8 9])=[ii ii+1];
74			case {'Salini'}
75			format([10 11])=[ii ii+1];
76			end
77			end
78
79			% read end of the textfile
80	roquet	1.2	nlines=0;nbuffer=100000;niter=0;
81	roquet	1.1	while ~feof(fid_cur);
82	roquet	1.2	niter=niter+1;
83			data_odv=cell(length(format)-1,nbuffer);
84			for ii=1:nbuffer,
85			tline=fgets(fid_cur);
86	roquet	1.1	nlines=nlines+1;
87			I_sep=[0 find(double(tline)==9)];
88	roquet	1.2	data_odv{1,ii}=[ tline(I_sep(format(1))+1:I_sep(format(1)+1)-1) '//' ...
89			tline(I_sep(format(2))+1:I_sep(format(2)+1)-1) ];
90			data_odv{2,ii}=tline(I_sep(format(3))+1:I_sep(format(3)+1)-1);
91			for kk=4:length(format),
92			I=I_sep(format(kk))+1:I_sep(format(kk)+1)-1;
93			if length(I)>1,
94			data_odv{kk-1,ii}=tline(I);
95			elseif isempty(I),
96			data_odv{kk-1,ii}='99999';
97			else
98			val=tline(I);
99			if val==0,
100			data_odv{kk-1,ii}='99999';
101			else
102			data_odv{kk-1,ii}=val;
103			end
104			end
105	roquet	1.1	end
106			if feof(fid_cur), break, end
107			end
108	roquet	1.2	data_odv=data_odv(:,1:ii);
109			data=data_odv(3:length(format)-1,:);
110			datanum=sscanf(sprintf('%s,',data{:}), '%g,');
111			datanum(datanum==99999)=NaN;
112	roquet	1.4
113			%no salinity case
114			if dataset.inclS,
115			datanum=reshape(datanum(:),8,ii);
116			else
117			datanum=reshape(datanum(:),6,ii);
118			end
119	roquet	1.2
120			fprintf('%d lines\n',nlines);
121			if niter==1,
122	roquet	1.3	pnum=data_odv(1,:);
123	roquet	1.2	Date=data_odv(2,:);
124			Data=datanum;
125	roquet	1.1	else
126	roquet	1.3	pnum=[pnum data_odv(1,:)];
127	roquet	1.2	Date=[Date data_odv(2,:)];
128			Data=[Data datanum];
129	roquet	1.1	end
130
131			end
132	roquet	1.2	fclose(fid_cur);
133
134	roquet	1.1	% get rid of empty indicator string
135	roquet	1.3	[pnum_txt_list,Iprof]=unique(pnum);
136	gforget	1.5	if size(Iprof,2)==1; Iprof=Iprof'; end;
137	roquet	1.3	for kk=1:length(pnum),
138	roquet	1.2	if strcmp(pnum_txt_list{kk},'//'),
139			pnum_txt_list=pnum_txt_list(setdiff(1:length(pnum_txt_list),kk));
140	roquet	1.1	Iprof(kk)=[];
141			break
142			end
143			end
144
145			% sort index of profile
146			[Iprof_sort,J]=sort(Iprof);
147	roquet	1.2	pnum_txt_sort=pnum_txt_list(J);
148	roquet	1.1
149			% write index of profile
150			Iprof_sort=[Iprof_sort nlines+1];
151	roquet	1.3	Iprofile=zeros(nlines,1);
152	roquet	1.1	for kk=1:length(Iprof_sort)-1,
153			for ii=Iprof_sort(kk):Iprof_sort(kk+1)-1,
154	roquet	1.3	Iprofile(ii)=kk;
155	roquet	1.1	end
156			end
157	roquet	1.4	dataset.Iprofile=Iprofile;
158	roquet	1.2
159	roquet	1.1	% number of profiles
160	roquet	1.4	dataset.nprofiles=length(Iprof);
161	roquet	1.1
162	roquet	1.4	varargout(1) = {dataset};
163	roquet	1.1
164
165			else;%if m==0;
166
167			% profile coordinates
168	roquet	1.4	Iprof=find(dataset.Iprofile==m);
169	roquet	1.3	pnum_txt=pnum{Iprof(1)};
170			lon=Data(1,Iprof(1));
171			lat=Data(2,Iprof(1));
172			strdate=Date{Iprof(1)};
173	roquet	1.1	if length(strdate)<10, varargout = {[]}; return; end
174			ymd=str2num(strdate([1:4 6:7 9:10]));
175			hms=0;
176			if length(strdate)==19, hms=str2num(strdate([12:13 15:16 18:19])); end
177			if length(strdate)==16, hms=str2num(strdate([12:13 15:16]))*100; end
178			if length(strdate)==13, hms=str2num(strdate([12:13]))*10000; end
179
180			%case when necessary information missing: dont retrieve profile
181			if isempty(lon)\|isempty(lat)\|isempty(ymd), varargout = {[]}; return; end
182			if lon < 0; lon=lon+360; end;
183
184			% get T/S data
185	roquet	1.3	z=Data(3,Iprof);
186			z_qv=Data(4,Iprof);z_qv(isempty(z_qv))=1;
187			t=Data(5,Iprof);
188			t_qv=Data(6,Iprof);t_qv(isempty(t_qv))=1;
189	roquet	1.4	if dataset.inclS,
190			s=Data(7,Iprof);
191			s_qv=Data(8,Iprof);s_qv(isempty(s_qv))=1;
192			else
193			s=tNaN; s_qv=tNaN; s_ERR=t*NaN;
194			end
195
196	roquet	1.1	I=find(isnan(z)\|z_qv~=0);
197	roquet	1.4	z(I)=[]; t(I)=[]; t_qv(I)=[];
198			t(t_qv~=0)=NaN;
199			t_ERR=t*0;
200			if dataset.inclS,
201			s_qv(I)=[]; s(I)=[];
202			s(s_qv~=0)=NaN; s_ERR=s*0;
203			end
204	gforget	1.6	z_ERR=t*0;
205	roquet	1.1
206			direc=2;
207	gforget	1.7	isBAD=0;
208	roquet	1.1
209			profileCur.pnum_txt=pnum_txt;
210			profileCur.ymd=ymd; profileCur.hms=hms;
211			profileCur.lat=lat; profileCur.lon=lon;
212			profileCur.direc=direc;
213	gforget	1.7	profileCur.isBAD=isBAD;
214	gforget	1.6	profileCur.T=t;
215			profileCur.S=s;
216			profileCur.depth=z;
217			profileCur.T_ERR=t_ERR;
218			profileCur.S_ERR=s_ERR;
219			profileCur.depth_ERR=z_ERR;
220	roquet	1.1
221			varargout = {profileCur};
222
223			end;
224
225
226