/[MITgcm]/MITgcm_contrib/llc_hires/llc_1080/readme.txt
ViewVC logotype

Contents of /MITgcm_contrib/llc_hires/llc_1080/readme.txt

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.16 - (show annotations) (download)
Sat Mar 2 19:31:13 2019 UTC (6 years, 1 month ago) by dimitri
Branch: MAIN
CVS Tags: HEAD
Changes since 1.15: +2 -3 lines
File MIME type: text/plain
updated asyncio code from Bron Nelson, which can now be use to read pickup files

1 ==============
2 # Interactive 90x90x1342 tile configuration with latest MITgcm, no asyncio
3 cd ~/llc_1080
4 cvs co MITgcm_contrib/llc_hires/llc_1080
5 git clone https://github.com/MITgcm/MITgcm.git
6 qsub -I -q long -l select=48:ncpus=28:model=bro,walltime=120:00:00 -m abe
7 module purge
8 module load comp-intel/2016.2.181 mpi-sgi/mpt.2.14r19 hdf4/4.2.12 hdf5/1.8.18_mpt netcdf/4.4.1.1_mpt
9 cd ~/llc_1080/MITgcm
10 mkdir build run
11 lfs setstripe -c -1 run
12 cd build
13 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h
14 ../tools/genmake2 -of \
15 ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
16 -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
17 make depend
18 make -j 56
19 cd ../run_test_noasync
20 cp ../build/mitgcmuv .
21 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
22 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
23 ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
24 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
25 mv data.exch2_90x90x1342 data.exch2
26 mpiexec -n 1342 ./mitgcmuv
27
28 ==============
29 # Interactive 90x90x1342 tile configuration with latest MITgcm, with asyncio
30 cd ~/llc_1080
31 cvs co MITgcm_contrib/llc_hires/llc_1080
32 git clone https://github.com/MITgcm/MITgcm.git
33 qsub -I -q long -l select=52:ncpus=28:model=bro,walltime=120:00:00 -m abe
34 module purge
35 module load comp-intel/2016.2.181 mpi-sgi/mpt.2.14r19 hdf4/4.2.12 hdf5/1.8.18_mpt netcdf/4.4.1.1_mpt
36 cd ~/llc_1080/MITgcm
37 mkdir build run
38 lfs setstripe -c -1 run
39 cd build
40 ../tools/genmake2 -of \
41 ../../MITgcm_contrib/llc_hires/llc_1080/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
42 '../../MITgcm_contrib/llc_hires/llc_1080/code ../../MITgcm_contrib/llc_hires/llc_1080/code-async'
43 make depend
44 make -j 56
45 cd ../run_test_async
46 cp ../build/mitgcmuv .
47 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
48 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
49 ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
50 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
51 mv data.exch2_90x90x1342 data.exch2
52 mpiexec -n 1442 ./mitgcmuv
53
54 ==============
55 # interactive 30x30x16848 tile configuration with MITgcm checkpoint66h
56 cd ~/llc_1080
57 cvs co MITgcm_contrib/llc_hires/llc_1080
58 cvs co -r checkpoint66h MITgcm_code
59 qsub -I -q devel -l select=602:ncpus=28:model=bro,walltime=2:00:00 -m abe
60 module purge
61 module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
62 cd ~/llc_1080/MITgcm
63 mkdir build run
64 lfs setstripe -c -1 run
65 cd build
66 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_30x30 SIZE.h
67 ../tools/genmake2 -of \
68 ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
69 -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
70 make depend
71 make -j 56
72 cd ../run
73 ln -sf ../build/mitgcmuv .
74 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
75 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
76 ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
77 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
78 mv data.exch2_30x30x16848 data.exch2
79 mpiexec -n 16848 ./mitgcmuv &
80
81 ==============
82 # interactive 36x36x11700 tile configuration with MITgcm checkpoint66h
83 cd ~/llc_1080
84 cvs co MITgcm_contrib/llc_hires/llc_1080
85 cvs co -r checkpoint66h MITgcm_code
86 qsub -I -q devel -l select=418:ncpus=28:model=bro,walltime=2:00:00 -m abe
87 module purge
88 module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
89 cd ~/llc_1080/MITgcm
90 mkdir build run
91 lfs setstripe -c -1 run
92 cd build
93 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_36x36 SIZE.h
94 ../tools/genmake2 -of \
95 ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
96 -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
97 make depend
98 make -j 56
99 cd ../run
100 ln -sf ../build/mitgcmuv .
101 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
102 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
103 ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
104 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
105 mv data.exch2_36x36x11700 data.exch2
106 mpiexec -n 11700 ./mitgcmuv &
107
108 ==============
109 # interactive 40x40x9477 tile configuration with MITgcm checkpoint66h
110 cd ~/llc_1080
111 cvs co MITgcm_contrib/llc_hires/llc_1080
112 cvs co -r checkpoint66h MITgcm_code
113 qsub -I -q devel -l select=339:ncpus=28:model=bro,walltime=2:00:00 -m abe
114 module purge
115 module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
116 cd ~/llc_1080/MITgcm
117 mkdir build run
118 lfs setstripe -c -1 run
119 cd build
120 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_40x40 SIZE.h
121 ../tools/genmake2 -of \
122 ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
123 -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
124 make depend
125 make -j 56
126 cd ../run
127 ln -sf ../build/mitgcmuv .
128 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
129 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
130 ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
131 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
132 mv data.exch2_40x40x9477 data.exch2
133 mpiexec -n 9477 ./mitgcmuv &
134
135 ==============
136 # interactive 45x45x7488 tile configuration with MITgcm checkpoint66h
137 cd ~/llc_1080
138 cvs co MITgcm_contrib/llc_hires/llc_1080
139 cvs co -r checkpoint66h MITgcm_code
140 qsub -I -q devel -l select=268:ncpus=28:model=bro,walltime=2:00:00 -m abe
141 module purge
142 module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
143 cd ~/llc_1080/MITgcm
144 mkdir build run
145 lfs setstripe -c -1 run
146 cd build
147 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_45x45 SIZE.h
148 ../tools/genmake2 -of \
149 ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
150 -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
151 make depend
152 make -j 56
153 cd ../run
154 ln -sf ../build/mitgcmuv .
155 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
156 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
157 ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
158 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
159 mv data.exch2_45x45x7488 data.exch2
160 mpiexec -n 7488 ./mitgcmuv &
161
162 ==============
163 # interactive 54x54x5200 tile configuration with MITgcm checkpoint66h
164 cd ~/llc_1080
165 cvs co MITgcm_contrib/llc_hires/llc_1080
166 cvs co -r checkpoint66h MITgcm_code
167 qsub -I -q devel -l select=268:ncpus=28:model=bro,walltime=2:00:00 -m abe
168 module purge
169 module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
170 cd ~/llc_1080/MITgcm
171 mkdir build run
172 lfs setstripe -c -1 run
173 cd build
174 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_54x54 SIZE.h
175 ../tools/genmake2 -of \
176 ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
177 -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
178 make depend
179 make -j 56
180 cd ../run
181 ln -sf ../build/mitgcmuv .
182 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
183 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
184 ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
185 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
186 mv data.exch2_54x54x5200 data.exch2
187 mpiexec -n 5200 ./mitgcmuv &
188
189 ==============
190 # Interactive 90x90x1342 tile configuration with MITgcm checkpoint66h
191 cd ~/llc_1080
192 cvs co MITgcm_contrib/llc_hires/llc_1080
193 cvs co -r checkpoint66h MITgcm_code
194 qsub -I -q long -l select=48:ncpus=28:model=bro,walltime=120:00:00 -m abe
195 module purge
196 module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
197 cd ~/llc_1080/MITgcm
198 mkdir build run
199 lfs setstripe -c -1 run
200 cd build
201 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h
202 ../tools/genmake2 -of \
203 ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
204 -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
205 make depend
206 make -j 56
207 cd ../run
208 ln -sf ../build/mitgcmuv .
209 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
210 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
211 ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
212 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
213 mv data.exch2_90x90x1342 data.exch2
214 mpiexec -n 1342 ./mitgcmuv &
215
216 ==============
217 # interactive 90x90x1342 tile configuration from scratch
218 cd ~/llc_1080
219 cvs co MITgcm_contrib/llc_hires/llc_1080
220 cvs co -r checkpoint64p MITgcm_code
221 qsub -I -q long -l select=48:ncpus=28:model=bro,walltime=120:00:00 -m abe
222 module purge
223 module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
224 cd ~/llc_1080/MITgcm
225 mkdir build run
226 lfs setstripe -c -1 run
227 cd build
228 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h
229 ../tools/genmake2 -of \
230 ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
231 -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
232 make depend
233 make -j 16
234 cd ../run
235 ln -sf ../build/mitgcmuv .
236 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
237 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
238 ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
239 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
240 mv data.exch2_90x90x1342 data.exch2
241 mpiexec -n 1342 ./mitgcmuv &
242
243 =====================
244 For interactive session, Ivy Bridge nodes:
245 qsub -I -q debug -l select=48:ncpus=28:model=bro,walltime=02:00:00 -m abe -M email
246 qsub -I -q devel -l select=170:ncpus=20:model=ivy,walltime=02:00:00 -m abe -M email
247 qsub -I -q long -l select=170:ncpus=20:model=ivy,walltime=120:00:00 -m abe -M email
248 qsub -I -q long -l select=170:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M email
249
250 ==============
251 # 60x60x2872 tile configuration
252
253 cd ~/llc_1080
254 cvs co -r checkpoint64p MITgcm_code
255 cvs co MITgcm_contrib/llc_hires/llc_1080
256 cd MITgcm
257 module purge
258 module load comp-intel/2012.0.032 mpi-sgi/mpt.2.08r7 netcdf/4.0
259 mkdir build run
260 lfs setstripe -c -1 run
261 cd build
262 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_60x60x2872 SIZE.h
263 ../tools/genmake2 -of \
264 ../../MITgcm_contrib/llc_hires/llc_1080/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
265 '../../MITgcm_contrib/llc_hires/llc_1080/code ../../MITgcm_contrib/llc_hires/llc_1080/code-async'
266 make depend
267 make -j 16
268 cd ~/llc_1080/MITgcm/run
269 ln -sf ../build/mitgcmuv .
270 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
271 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
272 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
273 mv data.exch2_60x60x2872 data.exch2
274 export MPI_NUM_MEMORY_REGIONS=256
275 mpiexec -n 3400 ./mitgcmuv
276
277 ==============
278 # 90x90x1342 tile configuration
279
280 cd ~/llc_1080
281 cvs co MITgcm_code
282 cvs co MITgcm_contrib/llc_hires/llc_1080
283 cd MITgcm
284 module purge
285 module load comp-intel/2011.2 mpi-sgi/mpt.2.06r6 netcdf/4.0
286 mkdir build run
287 lfs setstripe -c -1 run
288 cd build
289 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h
290 ../tools/genmake2 -of \
291 ../../MITgcm_contrib/llc_hires/llc_1080/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
292 '../../MITgcm_contrib/llc_hires/llc_1080/code ../../MITgcm_contrib/llc_hires/llc_1080/code-async'
293 make depend
294 make -j 16
295 cd ../run
296 ln -sf ../build/mitgcmuv .
297 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
298 #ln -sf /nobackup/dmenemen/forcing/era_interim/EIG_*_2* .
299 #ln -sf /nobackup/dmenemen/forcing/era_interim_corrected/EIG_dlw_sub5p_2* .
300 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
301 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
302 mv data.exch2_90x90x1342 data.exch2
303 export MPI_NUM_MEMORY_REGIONS=256
304 mpiexec -n 1600 ./mitgcmuv
305
306 ==============
307 # 90x90x1342 tile test configuration
308
309 cd ~/llc_1080_test
310 cvs co MITgcm_code
311 cvs co MITgcm_contrib/llc_hires/llc_1080
312 cd MITgcm
313 module purge
314 module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
315 mkdir build run
316 lfs setstripe -c -1 run
317 cd build
318 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h
319 ../tools/genmake2 -of \
320 ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
321 -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
322 make depend
323 make -j 16
324 cd ../run
325 ln -sf ../build/mitgcmuv .
326 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
327 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
328 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
329 mv data.exch2_90x90x1342 data.exch2
330 mpiexec -n 1342 ./mitgcmuv
331
332
333 ==============
334
335 look at output
336
337 for ts=[50760]
338 fld=quikread_llc(['Eta.' myint2str(ts,10) '.data'],1080);
339 clf,quikplot_llc(fld),caxis([-2.5 2]),thincolorbar
340 title(ts)
341 pause(.1)
342 end
343
344 ts=50760;
345 for fld={'S''T','U','V','W','HEFF'}
346 tmp=quikread_llc([fld{1} '.' myint2str(ts,10) '.data'],1080);
347 clf,quikplot_llc(tmp),thincolorbar
348 title(fld{1})
349 pause
350 end
351
352 ==============
353
354 to determine empty tiles:
355 grep Empty STDOUT.*
356
357 ==============
358
359 memory requirements:
360 nPx sNx sNy nSx node0 total rank0 rank1
361 702 60 60 6 17,642,924kb 770,168,568kb
362 468 60 60 9 22,083,004kb 643,301,784kb
363 468 180 180 1 24,719,776kb 720,415,552kb 1678MB 1563MB
364 351 60 60 12 26,660,020kb 582,815,544kb
365 351 120 120 3 25,110,324kb 548,819,772kb 1711MB 1586MB
366 351 120 120 3 6,543,780kb 56,729,484kb - with globalFiles - run out of time
367 325 54 54 16 29,029,744kb 587,831,344kb
368 325 216 216 1 29,249,232kb 426,554,484kb - node ran out of memory and crashed
369 234 90 90 8 30,590,952kb 431,252,200kb - node ran out of memory and crashed
370 234 180 180 2 27,682,360kb 307,181,152kb - node ran out of memory and crashed
371
372 =============
373 factor(3240) = 2 2 2 3 3 3 3 5
374 factor(1080) = 2 2 2 3 3 3 5
375
376 2 = 2
377 3 = 3
378 2*2 = 4
379 5 = 5
380 2*3 = 6
381 2*2*2 = 8
382 3*3 = 9
383 2*5 = 10
384 2*2*3 = 12
385 3*5 = 15
386 2*3*3 = 18
387 2*2*5 = 20
388 2*2*2*3 = 24
389 3*3*3 = 27
390 2*3*5 = 30
391 2*2*3*3 = 36 * 30
392 2*2*2*5 = 40 * 27
393 3*3*5 = 45 * 24
394 2*3*3*3 = 54 * 20
395 2*2*3*5 = 60 * 18
396 2*2*2*3*3 = 72 * 15
397 2*3*3*5 = 90 * 12
398 2*2*3*3*3 = 108 * 10
399 2*2*2*3*5 = 120 * 9
400 3*3*3*5 = 135 * 8
401 2*2*3*3*5 = 180 * 6
402 2*2*2*3*3*3 = 216 * 5
403 2*3*3*3*5 = 270 * 4
404 2*2*2*3*3*5 = 360 * 3
405 2*2*3*3*3*5 = 540 * 2

  ViewVC Help
Powered by ViewVC 1.1.22