/[MITgcm]/MITgcm/utils/scripts/gluemnc
ViewVC logotype

Contents of /MITgcm/utils/scripts/gluemnc

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.16 - (show annotations) (download)
Thu Nov 10 14:26:16 2011 UTC (10 years, 6 months ago) by mlosch
Branch: MAIN
CVS Tags: checkpoint64y, checkpoint64x, checkpoint64z, checkpoint64q, checkpoint64p, checkpoint64s, checkpoint64r, checkpoint64u, checkpoint64t, checkpoint64w, checkpoint64v, checkpoint64i, checkpoint64h, checkpoint64k, checkpoint64j, checkpoint64m, checkpoint64l, checkpoint64o, checkpoint64n, checkpoint64a, checkpoint64c, checkpoint64b, checkpoint64e, checkpoint64d, checkpoint64g, checkpoint64f, checkpoint63p, checkpoint63q, checkpoint63r, checkpoint63s, checkpoint63l, checkpoint63m, checkpoint63n, checkpoint63o, checkpoint63h, checkpoint63i, checkpoint63j, checkpoint63k, checkpoint63f, checkpoint63g, checkpoint64, checkpoint65, checkpoint66g, checkpoint66f, checkpoint66e, checkpoint66d, checkpoint66c, checkpoint66b, checkpoint66a, checkpoint66o, checkpoint66n, checkpoint66m, checkpoint66l, checkpoint66k, checkpoint66j, checkpoint66i, checkpoint66h, checkpoint65z, checkpoint65x, checkpoint65y, checkpoint65r, checkpoint65s, checkpoint65p, checkpoint65q, checkpoint65v, checkpoint65w, checkpoint65t, checkpoint65u, checkpoint65j, checkpoint65k, checkpoint65h, checkpoint65i, checkpoint65n, checkpoint65o, checkpoint65l, checkpoint65m, checkpoint65b, checkpoint65c, checkpoint65a, checkpoint65f, checkpoint65g, checkpoint65d, checkpoint65e, HEAD
Changes since 1.15: +49 -26 lines
change the way variables without record dimensions are handled so that
files that have variables with and without record variables can be
glued: add a record dimension with a new unique name to be able to
differentiate between variable with and without a-prior record
dimensions. Remove this record dimension before adding the variable to
the *.glob.nc file. This procedure automatically works also for
grid.*.nc files that do not have any record dimension.

1 #! /usr/bin/env bash
2 #
3 # $Header: /u/gcmpack/MITgcm/utils/scripts/gluemnc,v 1.14 2011/10/25 15:40:19 mlosch Exp $
4 # $Name: $
5
6 # This is a shell script to combine multiple MITgcm mnc output files from
7 # different tiles into one global file.
8 # All of the variables should be in one directory, where this script is run.
9 #
10 # To combine all state.0000000000.t*.nc files,
11 # gluemnc state.0000000000.*.nc
12 # This will result in an output file state.0000000000.glob.nc
13 # Where glob is for global.
14 #
15 # You can even combine all mnc files, use
16 # gluemnc *.nc
17 # This will result in a series of global files,
18 # state.0000000000.glob.nc state.0000000100.glob.nc, ...
19 # grid.0000000000.glob.nc grid.0000000100.glob.nc, ...
20 # diag.0000000000.glob.nc diag.0000000100.glob.nc, ...
21 #
22 # A lot of hard drive activity is needed. If you have a fast drive
23 # export TMPDIR=<path of hard drive>. On some high-performance
24 # systems, this is already done for you.
25 #
26 # **********WARNINGS**********
27 # This will probably not work at all with exch2/cubed sphere.
28 # In that case, you probably can assemble all of the tiles on a face,
29 # but combining faces is currently not implemented.
30 #
31 # Be sure you have enough disk space for the copies! In this version
32 # nothing is done to assure all of the data is copied.
33 #
34 # Be careful! It will be easy to exceed the 2 GB limit for the old 32-bit
35 # version of netcdf. If you do not have large-file support or 64-bit netcdf
36 # you will have to be clever in dividing up your tiled files,
37 # e.g., along the time dimension before combining to global files.
38 # The nco operator ncks is adept at shortening files to fewer snapshots.
39 # *****************************
40 #
41 # Good luck and happy gluing,
42 # Baylor Fox-Kemper
43
44 DEBUG="--dbg_lvl=0"
45 LOGFILE="/dev/null"
46
47 DIRORIG=`pwd`
48
49 if [ ! ${#TMPDIR} -gt 0 ]; then
50 TMPDIR=$DIRORIG
51 fi
52
53 export DIRNAME="$TMPDIR/gluedir.$RANDOM"
54 mkdir $DIRNAME
55
56 echo Using temporary directory $DIRNAME
57
58 if [ -f xplodemnc ]; then
59 cp xplodemnc $DIRNAME
60 else
61 cp `which xplodemnc` $DIRNAME
62 fi
63
64 # find an unambiguous name for a new record dimension
65 myrecname=record`echo $DIRNAME | awk -F. '{print $NF}'`
66
67 cd $DIRNAME
68
69 inone=$1
70 inone=${1:?"You must input mnc filenames to be glued"}
71
72 for somefile in $@
73 do
74 ln -s $DIRORIG/$somefile .
75 if [ ! -s $somefile ]; then
76 echo "Error: $somefile is missing or empty"
77 exit 1
78 fi
79 done
80
81 prels=${@%.t???.nc}
82
83 for somepre in $prels
84 do
85 inls=0
86 for somepres in $sprels
87 do
88 if [ "$somepre" = "$somepres" ]; then
89 inls=1
90 fi
91 done
92 if [ "$inls" = "0" ]; then
93 sprels=$sprels" "$somepre
94 fi
95 done
96
97 prels=$sprels
98
99 # ML: determine the coordinate variable (this is hack for the unlikely
100 # case that we do not have X or Y as coordinate variables; this can
101 # happen, when only U-point or V-point variables are written to a
102 # diagnostics stream; I do not know if this always works, but it works for me)
103 echo Determine a usable coordinate variable
104 somefile=${prels}.t001.nc
105 # first try X and Y
106 Xcoord=X
107 Ycoord=Y
108 Xtest=$(ncdump -vX -l 10000 $somefile | grep "X = ")
109 Ytest=$(ncdump -vY -l 10000 $somefile | grep "Y = ")
110 if [ ${#Xtest} = 0 ]; then
111 echo "X not found, trying Xp1"
112 Xtest=$(ncdump -vXp1 -l 10000 $somefile | grep "Xp1 = ")
113 Xcoord=Xp1
114 fi
115 if [ ${#Xtest} = 0 ]; then
116 echo "no X-coordinate found"
117 Xcoord=
118 fi
119 if [ ${#Ytest} = 0 ]; then
120 echo "Y not found, trying Yp1"
121 Ytest=$(ncdump -vYp1 -l 10000 $somefile | grep "Yp1 = ")
122 Ycoord=Yp1
123 fi
124 if [ ${#Ytest} = 0 ]; then
125 echo "no Y-coordinate found"
126 Ycoord=
127 fi
128 if [ ${#Xcoord} = 0 ]; then
129 echo cannot continue
130 exit
131 fi
132 if [ ${#Ycoord} = 0 ]; then
133 echo cannot continue
134 exit
135 fi
136
137 for somepre in $prels
138 do
139 echo Making $somepre.glob.nc...
140 Xsls=
141 Ysls=
142
143 for somefile in $@
144 do
145 if [ "${somefile%.t???.nc}" = "$somepre" ]; then
146 echo Scanning $somefile...
147 Xs=$(ncdump -v${Xcoord} -l 10000 $somefile | grep "${Xcoord} = ")
148 Xs=${Xs#*;}
149 Xs=${Xs%;*}
150 Xs=$(echo $Xs | sed s/' '//g)
151 Xsls=$Xsls$Xs" "
152
153 Ys=$(ncdump -v${Ycoord} -l 10000 $somefile | grep "${Ycoord} = ")
154 Ys=${Ys#*;}
155 Ys=${Ys%;*}
156 Ys=$(echo $Ys | sed s/' '//g)
157 Ysls=$Ysls$Ys" "
158 fi
159 done
160
161 sYsls=
162 sXsls=
163
164 # Determine all the X locations
165 countx=0
166 for someXs in $Xsls
167 do
168 inls=0
169 for somesXs in $sXsls
170 do
171 if [ "$someXs" = "$somesXs" ]; then
172 inls=1
173 fi
174 done
175 if [ "$inls" = "0" ]; then
176 sXsls=$sXsls$someXs" "
177 countx=$((countx))+1
178 fi
179 done
180 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
181 echo $((countx)) tiles found in x-direction.
182 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
183
184 # Determine all the Y locations
185 county=0
186 for someYs in $Ysls
187 do
188 inls=0
189 for somesYs in $sYsls
190 do
191 if [ "$someYs" = "$somesYs" ]; then
192 inls=1
193 fi
194 done
195 if [ "$inls" = "0" ]; then
196 sYsls=$sYsls$someYs" "
197 county=$((county))+1
198 fi
199 done
200 echo YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY
201 echo $((county)) tiles found in y-direction.
202 echo YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY
203
204 countyy=1000
205 countxx=1000
206
207 cntls=
208 for someX in $sXsls
209 do
210 countxx=$((countxx+1))
211 cntls=$cntls$countxx" "
212 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
213 echo Prepping X tile $((countxx-1000))
214 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
215
216 for somefile in $@
217 do
218 if [ "${somefile%.t???.nc}" = "$somepre" ]; then
219 Xs=$(ncdump -v${Xcoord} -l 10000 $somefile | grep "${Xcoord} = ")
220 Xs=${Xs#*;}
221 Xs=${Xs%;*}
222 Xs=$(echo $Xs | sed s/' '//g)
223
224 if [ "$someX" = $Xs ]; then
225 ./xplodemnc $somefile
226 if [ -f iter.$somefile ]; then
227 mv iter.$somefile iter.${somefile%t???.nc}glob.nc
228 fi
229 for somesplit in $(ls *.$somefile)
230 do
231 # Added to account for grid files with no T dimension defined:
232 # Search for the unlimited dimension and get its name, assuming
233 # that its first character is a letter from the alphabet, the "tr"
234 # removes the blank characters
235 recname=$(ncdump -h $somesplit \
236 | sed -n 's/\([a-z,A-Z]*\) = \(UNLIMITED\) .*/\1/p' \
237 | tr -d ' \t' )
238 if [[ -z "$recname" ]]; then
239 echo "No record dimension found, adding one now: "$myrecname
240 ncecat $DEBUG -O -u $myrecname $somesplit $somesplit > $LOGFILE
241 recname=$myrecname
242 fi
243 withY=$(ncdump -h $somesplit | grep "Y =")
244 if [ ${#withY} -gt 1 ]; then
245 echo Changing Y to record variable in $somesplit
246 ncpdq $DEBUG -O -a Y,$recname $somesplit $somesplit > $LOGFILE
247 mv $somesplit i$countxx.$somesplit
248 fi
249
250 if [ -f $somesplit ]; then
251 withYp1=$(ncdump -h $somesplit | grep "Yp1 =")
252 if [ ${#withYp1} -gt 1 ]; then
253 Yp1len=${withYp1#*= }
254 Yp1len=$((${Yp1len% ;}-1))
255 # Strip off the repeated value in Yp1
256 echo Changing Yp1 to record variable in $somesplit
257 ncpdq $DEBUG -O -a Yp1,$recname -F -d Yp1,1,$Yp1len $somesplit $somesplit > $LOGFILE
258 mv $somesplit i$countxx.$somesplit
259 fi
260 fi
261 done
262 fi
263 fi
264 done
265 done
266 echo Tile names $cntls
267 for countxx in $cntls
268 do
269 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
270 echo Combining X tile $((countxx-1000))
271 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
272 varls=
273 cxfilels=$(ls i$countxx.*)
274 oldvar=
275 for somefile in $cxfilels
276 do
277 varname=`echo $somefile | sed 's/^i[0-9][0-9][0-9][0-9]\.//; s/\..*nc//'`
278 if [ "$varname" = "$oldvar" ]; then
279 echo $varname repeated
280 else
281 varls=$varls$varname" "
282 fi
283 oldvar=$varname
284 done
285
286 echo Found these variables to combine: $varls
287
288 for somevar in $varls
289 do
290 echo YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY
291 echo Combining $somevar files in Y
292 echo YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY
293
294 filelist=(`ls i$countxx.$somevar.$somepre.*.nc`)
295 withY=$(ncdump -h ${filelist[0]} | grep "Y =")
296 withYp1=$(ncdump -h ${filelist[0]} | grep "Yp1 =")
297
298 ncrcat $DEBUG i$countxx.$somevar.$somepre.*.nc $somevar.$somepre.gloy.$countxx.nc > $LOGFILE
299 echo Just combined $countxx.$somevar
300 rm i$countxx.$somevar.$somepre.t???.nc
301
302 # Recover the name of the record variable, there are two possibilities:
303 # T (MITgcm convention) and $myrecname.
304 # I must admit that this could be more elegant.
305 recname=$(ncdump -h $somevar.$somepre.gloy.$countxx.nc | grep $myrecname)
306 if [[ -z $recname ]]; then
307 recname=T
308 else
309 recname=$myrecname
310 fi
311 if [ ${#withY} -gt 1 ]; then
312 echo Changing $recname to record variable in $somevar.$somepre.gloy.$countxx.nc
313 ncpdq $DEBUG -O -a $recname,Y $somevar.$somepre.gloy.$countxx.nc $somevar.$somepre.gloy.$countxx.nc > $LOGFILE
314 fi
315
316 if [ ${#withYp1} -gt 1 ]; then
317 echo Changing $recname to record variable in $somevar.$somepre.gloy.$countxx.nc
318 ncpdq $DEBUG -O -a $recname,Yp1 $somevar.$somepre.gloy.$countxx.nc $somevar.$somepre.gloy.$countxx.nc > $LOGFILE
319 fi
320 done
321 done
322 for somevar in $varls
323 do
324 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
325 echo Combining $somevar files in X...
326 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
327 for somegloy in $(ls $somevar.$somepre.gloy.*.nc)
328 do
329 withX=$(ncdump -h $somegloy | grep "X =")
330 withXp1=$(ncdump -h $somegloy | grep "Xp1 =")
331 recname=$(ncdump -h $somegloy \
332 | sed -n 's/\([a-z,A-Z]*\) = \(UNLIMITED\) .*/\1/p' \
333 | tr -d ' \t' )
334
335 if [ ${#withX} -gt 1 ]; then
336 echo Changing X to record variable in $somegloy
337 ncpdq $DEBUG -O -a X,$recname $somegloy $somegloy > $LOGFILE
338 fi
339
340 if [ ${#withXp1} -gt 1 ]; then
341 Xp1len=${withXp1#*= }
342 Xp1len=$((${Xp1len% ;}-1))
343 # Strip off the repeated value in Xp1
344 echo Changing Xp1 to record variable in $somegloy
345 echo ncpdq $DEBUG -O -a Xp1,$recname -F -d Xp1,1,$Xp1len $somegloy $somegloy > $LOGFILE
346 ncpdq $DEBUG -O -a Xp1,$recname -F -d Xp1,1,$Xp1len $somegloy $somegloy > $LOGFILE
347 fi
348 done
349 echo Combining $somevar.gloy files...
350 ncrcat $DEBUG $somevar.$somepre.gloy.*.nc $somevar.$somepre.glob.nc > $LOGFILE
351 # rm $somevar.$somepre.gloy.*.nc
352
353 # recname is still valid, so change back to it without testing for it again
354 if [ ${#withX} -gt 1 ]; then
355 echo Changing $recname to record variable in $somevar.$somepre.glob.nc
356 ncpdq $DEBUG -O -a $recname,X $somevar.$somepre.glob.nc $somevar.$somepre.glob.nc > $LOGFILE
357 fi
358
359 if [ ${#withXp1} -gt 1 ]; then
360 echo Changing $recname to record variable in $somevar.$somepre.glob.nc
361 ncpdq $DEBUG -O -a $recname,Xp1 $somevar.$somepre.glob.nc $somevar.$somepre.glob.nc > $LOGFILE
362 fi
363 if [ "$recname" = "$myrecname" ]; then
364 # only for variables that did not have a record dimension to begin with
365 echo "removing record dimension $recname from $somevar.$somepre.glob.nc"
366 ncwa $DEBUG -O -a $recname $somevar.$somepre.glob.nc $somevar.$somepre.glob.nc
367 fi
368 ncks $DEBUG -A $somevar.$somepre.glob.nc $somepre.glob.nc > $LOGFILE
369 # rm $somevar.$somepre.glob.nc
370 done
371 if [ -f iter.$somepre.glob.nc ]; then
372 ncks $DEBUG -A iter.$somepre.glob.nc $somepre.glob.nc > $LOGFILE
373 fi
374 # rm iter.$somepre.glob.nc
375
376 # another hack by rpa to accomodate grid.nc files
377 # (there are several variables with just Z dimension that we want to keep)
378 # varsz=$( ncdump -h $somepre.t001.nc | sed -n 's/^\s*\(double\|float\).* \(\w*\)(Z\w*).*/\2/p' )
379 # The OR ("\|") and "\s", "\w" only work for GNU-sed, but not for
380 # BSD-sed or SunOS-sed, therefore we need to use some work-arounds:
381 varsz=$( ncdump -h $somepre.t001.nc | egrep "double|float" \
382 | grep -v , | sed -n 's/.* \(.*\)(Z.*).*/\1/p' )
383 fixed=
384 for varz in $varsz
385 do
386 # check to make sure the variable does not already exist in the glob file
387 if [[ -z $( ncdump -h $somepre.glob.nc | grep " $varz(" ) ]]
388 then
389 echo "Adding variable $varz to $somepre.glob.nc"
390 ncks $DEBUG -A -v $varz $somepre.t001.nc $somepre.glob.nc > $LOGFILE
391 fixed='yes'
392 fi
393 done
394
395 cp $somepre.glob.nc $DIRORIG
396 done
397
398
399 cd $DIRORIG
400 rm -rf $DIRNAME

  ViewVC Help
Powered by ViewVC 1.1.22