/[MITgcm]/MITgcm/utils/scripts/gluemnc
ViewVC logotype

Contents of /MITgcm/utils/scripts/gluemnc

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.8 - (show annotations) (download)
Wed Nov 8 16:22:33 2006 UTC (17 years, 5 months ago) by baylor
Branch: MAIN
CVS Tags: checkpoint58u_post, checkpoint58w_post, checkpoint60, checkpoint61, checkpoint58r_post, checkpoint58x_post, checkpoint58t_post, checkpoint59q, checkpoint59p, checkpoint59r, checkpoint59e, checkpoint59d, checkpoint59g, checkpoint59f, checkpoint59a, checkpoint59c, checkpoint59b, checkpoint59m, checkpoint59l, checkpoint59o, checkpoint59n, checkpoint59i, checkpoint59h, checkpoint59k, checkpoint59j, checkpoint59, checkpoint58y_post, checkpoint58v_post, checkpoint58s_post, checkpoint61f, checkpoint61g, checkpoint61d, checkpoint61e, checkpoint61b, checkpoint61c, checkpoint61a, checkpoint61j, checkpoint61k, checkpoint61h, checkpoint61i
Changes since 1.7: +49 -28 lines
Latest more extensively tested version.  Now capable of specifying a high-speed hard drive with TMPDIR.  This makes the gluing go faster on high-performance machines with a high-speed per-node drive.

1 #!/bin/bash
2 # This is a shell script to combine multiple MITgcm mnc output files from
3 # different tiles into one global file.
4 # All of the variables should be in one directory, where this script is run.
5 #
6 # To combine all state.0000000000.t*.nc files,
7 # gluemnc state.0000000000.*.nc
8 # This will result in an output file state.0000000000.glob.nc
9 # Where glob is for global.
10 #
11 # You can even combine all mnc files, use
12 # gluemnc *.nc
13 # This will result in a series of global files,
14 # state.0000000000.glob.nc state.0000000100.glob.nc, ...
15 # grid.0000000000.glob.nc grid.0000000100.glob.nc, ...
16 # diag.0000000000.glob.nc diag.0000000100.glob.nc, ...
17 #
18 # A lot of hard drive activity is needed. If you have a fast drive
19 # export TMPDIR=<path of hard drive>. On some high-performance
20 # systems, this is already done for you.
21 #
22 # **********WARNINGS**********
23 # This will probably not work at all with exch2/cubed sphere.
24 # In that case, you probably can assemble all of the tiles on a face,
25 # but combining faces is currently not implemented.
26 #
27 # Be sure you have enough disk space for the copies! In this version
28 # nothing is done to assure all of the data is copied.
29 #
30 # Be careful! It will be easy to exceed the 2 GB limit for the old 32-bit
31 # version of netcdf. If you do not have large-file support or 64-bit netcdf
32 # you will have to be clever in dividing up your tiled files,
33 # e.g., along the time dimension before combining to global files.
34 # The nco operator ncks is adept at shortening files to fewer snapshots.
35 # *****************************
36 #
37 # Good luck and happy gluing,
38 # Baylor Fox-Kemper
39
40 DEBUG="--dbg_lvl=0"
41 LOGFILE="/dev/null"
42
43 DIRORIG=`pwd`
44
45 if [ ! ${#TMPDIR} -gt 0 ]; then
46 TMPDIR=$DIRORIG
47 fi
48
49 export DIRNAME="$TMPDIR/gluedir.$RANDOM"
50 mkdir $DIRNAME
51
52 echo Using temporary directory $DIRNAME
53
54 if [ -f xplodemnc ]; then
55 cp xplodemnc $DIRNAME
56 else
57 cp `which xplodemnc` $DIRNAME
58 fi
59
60 cd $DIRNAME
61
62 inone=$1
63 inone=${1:?"You must input mnc filenames to be glued"}
64
65 for somefile in $@
66 do
67 ln -s $DIRORIG/$somefile .
68 if [ ! -s $somefile ]; then
69 echo "Error: $somefile is missing or empty"
70 exit 1
71 fi
72 done
73
74 prels=${@%.t???.nc}
75
76 for somepre in $prels
77 do
78 inls=0
79 for somepres in $sprels
80 do
81 if [ "$somepre" = "$somepres" ]; then
82 inls=1
83 fi
84 done
85 if [ "$inls" = "0" ]; then
86 sprels=$sprels" "$somepre
87 fi
88 done
89
90 prels=$sprels
91
92 for somepre in $prels
93 do
94 echo Making $somepre.glob.nc...
95 Xsls=
96 Ysls=
97
98 for somefile in $@
99 do
100 if [ "${somefile%.t???.nc}" = "$somepre" ]; then
101 echo Scanning $somefile...
102 Xs=$(ncdump -vX -l 10000 $somefile | grep "X = ")
103 Xs=${Xs#*;}
104 Xs=${Xs%;*}
105 Xs=$(echo $Xs | sed s/' '//g)
106 Xsls=$Xsls$Xs" "
107
108 Ys=$(ncdump -vY -l 10000 $somefile | grep "Y = ")
109 Ys=${Ys#*;}
110 Ys=${Ys%;*}
111 Ys=$(echo $Ys | sed s/' '//g)
112 Ysls=$Ysls$Ys" "
113 fi
114 done
115
116 sYsls=
117 sXsls=
118
119 # Determine all the X locations
120 countx=0
121 for someXs in $Xsls
122 do
123 inls=0
124 for somesXs in $sXsls
125 do
126 if [ "$someXs" = "$somesXs" ]; then
127 inls=1
128 fi
129 done
130 if [ "$inls" = "0" ]; then
131 sXsls=$sXsls$someXs" "
132 countx=$((countx))+1
133 fi
134 done
135 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
136 echo $((countx)) tiles found in x-direction.
137 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
138
139 # Determine all the Y locations
140 county=0
141 for someYs in $Ysls
142 do
143 inls=0
144 for somesYs in $sYsls
145 do
146 if [ "$someYs" = "$somesYs" ]; then
147 inls=1
148 fi
149 done
150 if [ "$inls" = "0" ]; then
151 sYsls=$sYsls$someYs" "
152 county=$((county))+1
153 fi
154 done
155 echo YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY
156 echo $((county)) tiles found in y-direction.
157 echo YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY
158
159 countyy=1000
160 countxx=1000
161
162 cntls=
163 for someX in $sXsls
164 do
165 countxx=$((countxx+1))
166 cntls=$cntls$countxx" "
167 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
168 echo Prepping X tile $((countxx-1000))
169 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
170
171 for somefile in $@
172 do
173 if [ "${somefile%.t???.nc}" = "$somepre" ]; then
174 Xs=$(ncdump -vX -l 10000 $somefile | grep "X = ")
175 Xs=${Xs#*;}
176 Xs=${Xs%;*}
177 Xs=$(echo $Xs | sed s/' '//g)
178
179 if [ "$someX" = $Xs ]; then
180 ./xplodemnc $somefile
181 mv iter.$somefile iter.${somefile%t???.nc}glob.nc
182 for somesplit in $(ls *.$somefile)
183 do
184 withY=$(ncdump -h $somesplit | grep "Y =")
185 if [ ${#withY} -gt 1 ]; then
186 echo Changing Y to record variable in $somesplit
187 ncpdq $DEBUG -O -a Y,T $somesplit $somesplit > $LOGFILE
188 mv $somesplit $countxx.$somesplit
189 fi
190
191 if [ -f $somesplit ]; then
192 withYp1=$(ncdump -h $somesplit | grep "Yp1 =")
193 if [ ${#withYp1} -gt 1 ]; then
194 Yp1len=${withYp1#*= }
195 Yp1len=$((${Yp1len% ;}-1))
196 # Strip off the repeated value in Yp1
197 echo Changing Yp1 to record variable in $somesplit
198 ncpdq $DEBUG -O -a Yp1,T -F -d Yp1,1,$Yp1len $somesplit $somesplit > $LOGFILE
199 mv $somesplit $countxx.$somesplit
200 fi
201 fi
202 done
203 fi
204 fi
205 done
206 done
207 echo Tile names $cntls
208 for countxx in $cntls
209 do
210 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
211 echo Combining X tile $((countxx-1000))
212 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
213 varls=
214 cxfilels=$(ls $countxx.*)
215 oldvar=
216 for somefile in $cxfilels
217 do
218 varname=${somefile%.*}
219 varname=${varname%.*}
220 varname=${varname%.*}
221 varname=${varname%.*}
222 varname=${varname#*.}
223 if [ "$varname" = "$oldvar" ]; then
224 echo $varname repeated
225 else
226 varls=$varls$varname" "
227 fi
228 oldvar=$varname
229 done
230
231 echo Found these variables to combine: $varls
232
233 for somevar in $varls
234 do
235 echo YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY
236 echo Combining $somevar files in Y
237 echo YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY
238
239 withY=$(ncdump -h $countxx.$somevar.$somepre.*.nc | grep "Y =")
240 withYp1=$(ncdump -h $countxx.$somevar.$somepre.*.nc | grep "Yp1 =")
241
242 ncrcat $DEBUG $countxx.$somevar.$somepre.*.nc $somevar.$somepre.gloy.$countxx.nc > $LOGFILE
243 echo Just combined $countxx.$somevar
244 rm $countxx.$somevar.$somepre.t???.nc
245
246 if [ ${#withY} -gt 1 ]; then
247 echo Changing T to record variable in $somevar.$somepre.gloy.$countxx.nc
248 ncpdq $DEBUG -O -a T,Y $somevar.$somepre.gloy.$countxx.nc $somevar.$somepre.gloy.$countxx.nc > $LOGFILE
249 fi
250
251 if [ ${#withYp1} -gt 1 ]; then
252 echo Changing T to record variable in $somevar.$somepre.gloy.$countxx.nc
253 ncpdq $DEBUG -O -a T,Yp1 $somevar.$somepre.gloy.$countxx.nc $somevar.$somepre.gloy.$countxx.nc > $LOGFILE
254 fi
255 done
256 done
257 for somevar in $varls
258 do
259 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
260 echo Combining $somevar files in X...
261 echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
262 for somegloy in $(ls $somevar.$somepre.gloy.*.nc)
263 do
264 withX=$(ncdump -h $somegloy | grep "X =")
265 withXp1=$(ncdump -h $somegloy | grep "Xp1 =")
266
267 if [ ${#withX} -gt 1 ]; then
268 echo Changing X to record variable in $somegloy
269 ncpdq $DEBUG -O -a X,T $somegloy $somegloy > $LOGFILE
270 fi
271
272 if [ ${#withXp1} -gt 1 ]; then
273 Xp1len=${withXp1#*= }
274 Xp1len=$((${Xp1len% ;}-1))
275 # Strip off the repeated value in Xp1
276 echo Changing Xp1 to record variable in $somegloy
277 echo ncpdq $DEBUG -O -a Xp1,T -F -d Xp1,1,$Xp1len $somegloy $somegloy > $LOGFILE
278 ncpdq $DEBUG -O -a Xp1,T -F -d Xp1,1,$Xp1len $somegloy $somegloy > $LOGFILE
279 fi
280 done
281 echo Combining $somevar.gloy files...
282 ncrcat $DEBUG $somevar.$somepre.gloy.*.nc $somevar.$somepre.glob.nc > $LOGFILE
283 # rm $somevar.$somepre.gloy.*.nc
284
285 if [ ${#withX} -gt 1 ]; then
286 echo Changing T to record variable in $somevar.$somepre.glob.nc
287 ncpdq $DEBUG -O -a T,X $somevar.$somepre.glob.nc $somevar.$somepre.glob.nc > $LOGFILE
288 fi
289
290 if [ ${#withXp1} -gt 1 ]; then
291 echo Changing T to record variable in $somevar.$somepre.glob.nc
292 ncpdq $DEBUG -O -a T,Xp1 $somevar.$somepre.glob.nc $somevar.$somepre.glob.nc > $LOGFILE
293 fi
294 ncrcat $DEBUG -A $somevar.$somepre.glob.nc $somepre.glob.nc > $LOGFILE
295 # rm $somevar.$somepre.glob.nc
296 done
297 ncrcat $DEBUG -A iter.$somepre.glob.nc $somepre.glob.nc > $LOGFILE
298 # rm iter.$somepre.glob.nc
299
300 cp $somepre.glob.nc $DIRORIG
301 done
302
303
304 cd $DIRORIG
305 rm -rf $DIRNAME

  ViewVC Help
Powered by ViewVC 1.1.22