/[MITgcm]/MITgcm_contrib/llc_hires/llc_1080/readme.txt
ViewVC logotype

Contents of /MITgcm_contrib/llc_hires/llc_1080/readme.txt

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph


Revision 1.15 - (show annotations) (download)
Wed Feb 27 20:48:33 2019 UTC (6 years, 5 months ago) by dimitri
Branch: MAIN
Changes since 1.14: +60 -6 lines
File MIME type: text/plain
fixing data.seaice change of units

1 ==============
2 # Interactive 90x90x1342 tile configuration with latest MITgcm, no asyncio
3 cd ~/llc_1080
4 cvs co MITgcm_contrib/llc_hires/llc_1080
5 git clone https://github.com/MITgcm/MITgcm.git
6 qsub -I -q long -l select=48:ncpus=28:model=bro,walltime=120:00:00 -m abe
7 module purge
8 module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
9 cd ~/llc_1080/MITgcm
10 mkdir build run
11 lfs setstripe -c -1 run
12 cd build
13 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h
14 ../tools/genmake2 -of \
15 ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
16 -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
17 make depend
18 make -j 56
19 cd ../run_test_noasync
20 cp ../build/mitgcmuv .
21 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
22 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
23 ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
24 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
25 mv data.exch2_90x90x1342 data.exch2
26 mpiexec -n 1342 ./mitgcmuv
27
28 ==============
29 # Interactive 90x90x1342 tile configuration with latest MITgcm, with asyncio
30 cd ~/llc_1080
31 cvs co MITgcm_contrib/llc_hires/llc_1080
32 git clone https://github.com/MITgcm/MITgcm.git
33 qsub -I -q long -l select=52:ncpus=28:model=bro,walltime=120:00:00 -m abe
34 module purge
35 module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
36 cd ~/llc_1080/MITgcm
37 mkdir build run
38 lfs setstripe -c -1 run
39 cd build
40 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h
41 ../tools/genmake2 -of \
42 ../../MITgcm_contrib/llc_hires/llc_1080/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
43 '../../MITgcm_contrib/llc_hires/llc_1080/code ../../MITgcm_contrib/llc_hires/llc_1080/code-async'
44 make depend
45 make -j 56
46 cd ../run_test_async
47 cp ../build/mitgcmuv .
48 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
49 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
50 ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
51 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
52 mv data.exch2_90x90x1342 data.exch2
53 mpiexec -n 1442 ./mitgcmuv
54
55 ==============
56 # interactive 30x30x16848 tile configuration with MITgcm checkpoint66h
57 cd ~/llc_1080
58 cvs co MITgcm_contrib/llc_hires/llc_1080
59 cvs co -r checkpoint66h MITgcm_code
60 qsub -I -q devel -l select=602:ncpus=28:model=bro,walltime=2:00:00 -m abe
61 module purge
62 module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
63 cd ~/llc_1080/MITgcm
64 mkdir build run
65 lfs setstripe -c -1 run
66 cd build
67 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_30x30 SIZE.h
68 ../tools/genmake2 -of \
69 ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
70 -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
71 make depend
72 make -j 56
73 cd ../run
74 ln -sf ../build/mitgcmuv .
75 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
76 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
77 ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
78 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
79 mv data.exch2_30x30x16848 data.exch2
80 mpiexec -n 16848 ./mitgcmuv &
81
82 ==============
83 # interactive 36x36x11700 tile configuration with MITgcm checkpoint66h
84 cd ~/llc_1080
85 cvs co MITgcm_contrib/llc_hires/llc_1080
86 cvs co -r checkpoint66h MITgcm_code
87 qsub -I -q devel -l select=418:ncpus=28:model=bro,walltime=2:00:00 -m abe
88 module purge
89 module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
90 cd ~/llc_1080/MITgcm
91 mkdir build run
92 lfs setstripe -c -1 run
93 cd build
94 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_36x36 SIZE.h
95 ../tools/genmake2 -of \
96 ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
97 -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
98 make depend
99 make -j 56
100 cd ../run
101 ln -sf ../build/mitgcmuv .
102 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
103 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
104 ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
105 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
106 mv data.exch2_36x36x11700 data.exch2
107 mpiexec -n 11700 ./mitgcmuv &
108
109 ==============
110 # interactive 40x40x9477 tile configuration with MITgcm checkpoint66h
111 cd ~/llc_1080
112 cvs co MITgcm_contrib/llc_hires/llc_1080
113 cvs co -r checkpoint66h MITgcm_code
114 qsub -I -q devel -l select=339:ncpus=28:model=bro,walltime=2:00:00 -m abe
115 module purge
116 module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
117 cd ~/llc_1080/MITgcm
118 mkdir build run
119 lfs setstripe -c -1 run
120 cd build
121 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_40x40 SIZE.h
122 ../tools/genmake2 -of \
123 ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
124 -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
125 make depend
126 make -j 56
127 cd ../run
128 ln -sf ../build/mitgcmuv .
129 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
130 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
131 ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
132 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
133 mv data.exch2_40x40x9477 data.exch2
134 mpiexec -n 9477 ./mitgcmuv &
135
136 ==============
137 # interactive 45x45x7488 tile configuration with MITgcm checkpoint66h
138 cd ~/llc_1080
139 cvs co MITgcm_contrib/llc_hires/llc_1080
140 cvs co -r checkpoint66h MITgcm_code
141 qsub -I -q devel -l select=268:ncpus=28:model=bro,walltime=2:00:00 -m abe
142 module purge
143 module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
144 cd ~/llc_1080/MITgcm
145 mkdir build run
146 lfs setstripe -c -1 run
147 cd build
148 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_45x45 SIZE.h
149 ../tools/genmake2 -of \
150 ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
151 -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
152 make depend
153 make -j 56
154 cd ../run
155 ln -sf ../build/mitgcmuv .
156 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
157 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
158 ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
159 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
160 mv data.exch2_45x45x7488 data.exch2
161 mpiexec -n 7488 ./mitgcmuv &
162
163 ==============
164 # interactive 54x54x5200 tile configuration with MITgcm checkpoint66h
165 cd ~/llc_1080
166 cvs co MITgcm_contrib/llc_hires/llc_1080
167 cvs co -r checkpoint66h MITgcm_code
168 qsub -I -q devel -l select=268:ncpus=28:model=bro,walltime=2:00:00 -m abe
169 module purge
170 module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
171 cd ~/llc_1080/MITgcm
172 mkdir build run
173 lfs setstripe -c -1 run
174 cd build
175 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_54x54 SIZE.h
176 ../tools/genmake2 -of \
177 ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
178 -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
179 make depend
180 make -j 56
181 cd ../run
182 ln -sf ../build/mitgcmuv .
183 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
184 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
185 ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
186 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
187 mv data.exch2_54x54x5200 data.exch2
188 mpiexec -n 5200 ./mitgcmuv &
189
190 ==============
191 # Interactive 90x90x1342 tile configuration with MITgcm checkpoint66h
192 cd ~/llc_1080
193 cvs co MITgcm_contrib/llc_hires/llc_1080
194 cvs co -r checkpoint66h MITgcm_code
195 qsub -I -q long -l select=48:ncpus=28:model=bro,walltime=120:00:00 -m abe
196 module purge
197 module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
198 cd ~/llc_1080/MITgcm
199 mkdir build run
200 lfs setstripe -c -1 run
201 cd build
202 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h
203 ../tools/genmake2 -of \
204 ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
205 -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
206 make depend
207 make -j 56
208 cd ../run
209 ln -sf ../build/mitgcmuv .
210 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
211 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
212 ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
213 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
214 mv data.exch2_90x90x1342 data.exch2
215 mpiexec -n 1342 ./mitgcmuv &
216
217 ==============
218 # interactive 90x90x1342 tile configuration from scratch
219 cd ~/llc_1080
220 cvs co MITgcm_contrib/llc_hires/llc_1080
221 cvs co -r checkpoint64p MITgcm_code
222 qsub -I -q long -l select=48:ncpus=28:model=bro,walltime=120:00:00 -m abe
223 module purge
224 module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
225 cd ~/llc_1080/MITgcm
226 mkdir build run
227 lfs setstripe -c -1 run
228 cd build
229 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h
230 ../tools/genmake2 -of \
231 ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
232 -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
233 make depend
234 make -j 16
235 cd ../run
236 ln -sf ../build/mitgcmuv .
237 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
238 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
239 ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
240 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
241 mv data.exch2_90x90x1342 data.exch2
242 mpiexec -n 1342 ./mitgcmuv &
243
244 =====================
245 For interactive session, Ivy Bridge nodes:
246 qsub -I -q debug -l select=48:ncpus=28:model=bro,walltime=02:00:00 -m abe -M email
247 qsub -I -q devel -l select=170:ncpus=20:model=ivy,walltime=02:00:00 -m abe -M email
248 qsub -I -q long -l select=170:ncpus=20:model=ivy,walltime=120:00:00 -m abe -M email
249 qsub -I -q long -l select=170:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M email
250
251 ==============
252 # 60x60x2872 tile configuration
253
254 cd ~/llc_1080
255 cvs co -r checkpoint64p MITgcm_code
256 cvs co MITgcm_contrib/llc_hires/llc_1080
257 cd MITgcm
258 module purge
259 module load comp-intel/2012.0.032 mpi-sgi/mpt.2.08r7 netcdf/4.0
260 mkdir build run
261 lfs setstripe -c -1 run
262 cd build
263 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_60x60x2872 SIZE.h
264 ../tools/genmake2 -of \
265 ../../MITgcm_contrib/llc_hires/llc_1080/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
266 '../../MITgcm_contrib/llc_hires/llc_1080/code ../../MITgcm_contrib/llc_hires/llc_1080/code-async'
267 make depend
268 make -j 16
269 cd ~/llc_1080/MITgcm/run
270 ln -sf ../build/mitgcmuv .
271 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
272 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
273 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
274 mv data.exch2_60x60x2872 data.exch2
275 export MPI_NUM_MEMORY_REGIONS=256
276 mpiexec -n 3400 ./mitgcmuv
277
278 ==============
279 # 90x90x1342 tile configuration
280
281 cd ~/llc_1080
282 cvs co MITgcm_code
283 cvs co MITgcm_contrib/llc_hires/llc_1080
284 cd MITgcm
285 module purge
286 module load comp-intel/2011.2 mpi-sgi/mpt.2.06r6 netcdf/4.0
287 mkdir build run
288 lfs setstripe -c -1 run
289 cd build
290 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h
291 ../tools/genmake2 -of \
292 ../../MITgcm_contrib/llc_hires/llc_1080/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
293 '../../MITgcm_contrib/llc_hires/llc_1080/code ../../MITgcm_contrib/llc_hires/llc_1080/code-async'
294 make depend
295 make -j 16
296 cd ../run
297 ln -sf ../build/mitgcmuv .
298 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
299 #ln -sf /nobackup/dmenemen/forcing/era_interim/EIG_*_2* .
300 #ln -sf /nobackup/dmenemen/forcing/era_interim_corrected/EIG_dlw_sub5p_2* .
301 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
302 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
303 mv data.exch2_90x90x1342 data.exch2
304 export MPI_NUM_MEMORY_REGIONS=256
305 mpiexec -n 1600 ./mitgcmuv
306
307 ==============
308 # 90x90x1342 tile test configuration
309
310 cd ~/llc_1080_test
311 cvs co MITgcm_code
312 cvs co MITgcm_contrib/llc_hires/llc_1080
313 cd MITgcm
314 module purge
315 module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
316 mkdir build run
317 lfs setstripe -c -1 run
318 cd build
319 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h
320 ../tools/genmake2 -of \
321 ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
322 -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
323 make depend
324 make -j 16
325 cd ../run
326 ln -sf ../build/mitgcmuv .
327 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
328 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
329 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
330 mv data.exch2_90x90x1342 data.exch2
331 mpiexec -n 1342 ./mitgcmuv
332
333
334 ==============
335
336 look at output
337
338 for ts=[50760]
339 fld=quikread_llc(['Eta.' myint2str(ts,10) '.data'],1080);
340 clf,quikplot_llc(fld),caxis([-2.5 2]),thincolorbar
341 title(ts)
342 pause(.1)
343 end
344
345 ts=50760;
346 for fld={'S''T','U','V','W','HEFF'}
347 tmp=quikread_llc([fld{1} '.' myint2str(ts,10) '.data'],1080);
348 clf,quikplot_llc(tmp),thincolorbar
349 title(fld{1})
350 pause
351 end
352
353 ==============
354
355 to determine empty tiles:
356 grep Empty STDOUT.*
357
358 ==============
359
360 memory requirements:
361 nPx sNx sNy nSx node0 total rank0 rank1
362 702 60 60 6 17,642,924kb 770,168,568kb
363 468 60 60 9 22,083,004kb 643,301,784kb
364 468 180 180 1 24,719,776kb 720,415,552kb 1678MB 1563MB
365 351 60 60 12 26,660,020kb 582,815,544kb
366 351 120 120 3 25,110,324kb 548,819,772kb 1711MB 1586MB
367 351 120 120 3 6,543,780kb 56,729,484kb - with globalFiles - run out of time
368 325 54 54 16 29,029,744kb 587,831,344kb
369 325 216 216 1 29,249,232kb 426,554,484kb - node ran out of memory and crashed
370 234 90 90 8 30,590,952kb 431,252,200kb - node ran out of memory and crashed
371 234 180 180 2 27,682,360kb 307,181,152kb - node ran out of memory and crashed
372
373 =============
374 factor(3240) = 2 2 2 3 3 3 3 5
375 factor(1080) = 2 2 2 3 3 3 5
376
377 2 = 2
378 3 = 3
379 2*2 = 4
380 5 = 5
381 2*3 = 6
382 2*2*2 = 8
383 3*3 = 9
384 2*5 = 10
385 2*2*3 = 12
386 3*5 = 15
387 2*3*3 = 18
388 2*2*5 = 20
389 2*2*2*3 = 24
390 3*3*3 = 27
391 2*3*5 = 30
392 2*2*3*3 = 36 * 30
393 2*2*2*5 = 40 * 27
394 3*3*5 = 45 * 24
395 2*3*3*3 = 54 * 20
396 2*2*3*5 = 60 * 18
397 2*2*2*3*3 = 72 * 15
398 2*3*3*5 = 90 * 12
399 2*2*3*3*3 = 108 * 10
400 2*2*2*3*5 = 120 * 9
401 3*3*3*5 = 135 * 8
402 2*2*3*3*5 = 180 * 6
403 2*2*2*3*3*3 = 216 * 5
404 2*3*3*3*5 = 270 * 4
405 2*2*2*3*3*5 = 360 * 3
406 2*2*3*3*3*5 = 540 * 2

  ViewVC Help
Powered by ViewVC 1.1.22