1 |
============== |
2 |
# Interactive 90x90x1342 tile configuration with latest MITgcm, no asyncio |
3 |
cd ~/llc_1080 |
4 |
cvs co MITgcm_contrib/llc_hires/llc_1080 |
5 |
git clone https://github.com/MITgcm/MITgcm.git |
6 |
qsub -I -q long -l select=48:ncpus=28:model=bro,walltime=120:00:00 -m abe |
7 |
module purge |
8 |
module load comp-intel/2016.2.181 mpi-sgi/mpt.2.14r19 hdf4/4.2.12 hdf5/1.8.18_mpt netcdf/4.4.1.1_mpt |
9 |
cd ~/llc_1080/MITgcm |
10 |
mkdir build run |
11 |
lfs setstripe -c -1 run |
12 |
cd build |
13 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h |
14 |
../tools/genmake2 -of \ |
15 |
../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \ |
16 |
-mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code |
17 |
make depend |
18 |
make -j 56 |
19 |
cd ../run_test_noasync |
20 |
cp ../build/mitgcmuv . |
21 |
ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* . |
22 |
ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* . |
23 |
ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* . |
24 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* . |
25 |
mv data.exch2_90x90x1342 data.exch2 |
26 |
mpiexec -n 1342 ./mitgcmuv |
27 |
|
28 |
============== |
29 |
# Interactive 90x90x1342 tile configuration with latest MITgcm, with asyncio |
30 |
cd ~/llc_1080 |
31 |
cvs co MITgcm_contrib/llc_hires/llc_1080 |
32 |
git clone https://github.com/MITgcm/MITgcm.git |
33 |
qsub -I -q long -l select=52:ncpus=28:model=bro,walltime=120:00:00 -m abe |
34 |
module purge |
35 |
module load comp-intel/2016.2.181 mpi-sgi/mpt.2.14r19 hdf4/4.2.12 hdf5/1.8.18_mpt netcdf/4.4.1.1_mpt |
36 |
cd ~/llc_1080/MITgcm |
37 |
mkdir build run |
38 |
lfs setstripe -c -1 run |
39 |
cd build |
40 |
../tools/genmake2 -of \ |
41 |
../../MITgcm_contrib/llc_hires/llc_1080/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \ |
42 |
'../../MITgcm_contrib/llc_hires/llc_1080/code ../../MITgcm_contrib/llc_hires/llc_1080/code-async' |
43 |
make depend |
44 |
make -j 56 |
45 |
cd ../run_test_async |
46 |
cp ../build/mitgcmuv . |
47 |
ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* . |
48 |
ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* . |
49 |
ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* . |
50 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* . |
51 |
mv data.exch2_90x90x1342 data.exch2 |
52 |
mpiexec -n 1442 ./mitgcmuv |
53 |
|
54 |
============== |
55 |
# interactive 30x30x16848 tile configuration with MITgcm checkpoint66h |
56 |
cd ~/llc_1080 |
57 |
cvs co MITgcm_contrib/llc_hires/llc_1080 |
58 |
cvs co -r checkpoint66h MITgcm_code |
59 |
qsub -I -q devel -l select=602:ncpus=28:model=bro,walltime=2:00:00 -m abe |
60 |
module purge |
61 |
module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20 |
62 |
cd ~/llc_1080/MITgcm |
63 |
mkdir build run |
64 |
lfs setstripe -c -1 run |
65 |
cd build |
66 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_30x30 SIZE.h |
67 |
../tools/genmake2 -of \ |
68 |
../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \ |
69 |
-mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code |
70 |
make depend |
71 |
make -j 56 |
72 |
cd ../run |
73 |
ln -sf ../build/mitgcmuv . |
74 |
ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* . |
75 |
ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* . |
76 |
ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* . |
77 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* . |
78 |
mv data.exch2_30x30x16848 data.exch2 |
79 |
mpiexec -n 16848 ./mitgcmuv & |
80 |
|
81 |
============== |
82 |
# interactive 36x36x11700 tile configuration with MITgcm checkpoint66h |
83 |
cd ~/llc_1080 |
84 |
cvs co MITgcm_contrib/llc_hires/llc_1080 |
85 |
cvs co -r checkpoint66h MITgcm_code |
86 |
qsub -I -q devel -l select=418:ncpus=28:model=bro,walltime=2:00:00 -m abe |
87 |
module purge |
88 |
module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20 |
89 |
cd ~/llc_1080/MITgcm |
90 |
mkdir build run |
91 |
lfs setstripe -c -1 run |
92 |
cd build |
93 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_36x36 SIZE.h |
94 |
../tools/genmake2 -of \ |
95 |
../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \ |
96 |
-mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code |
97 |
make depend |
98 |
make -j 56 |
99 |
cd ../run |
100 |
ln -sf ../build/mitgcmuv . |
101 |
ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* . |
102 |
ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* . |
103 |
ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* . |
104 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* . |
105 |
mv data.exch2_36x36x11700 data.exch2 |
106 |
mpiexec -n 11700 ./mitgcmuv & |
107 |
|
108 |
============== |
109 |
# interactive 40x40x9477 tile configuration with MITgcm checkpoint66h |
110 |
cd ~/llc_1080 |
111 |
cvs co MITgcm_contrib/llc_hires/llc_1080 |
112 |
cvs co -r checkpoint66h MITgcm_code |
113 |
qsub -I -q devel -l select=339:ncpus=28:model=bro,walltime=2:00:00 -m abe |
114 |
module purge |
115 |
module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20 |
116 |
cd ~/llc_1080/MITgcm |
117 |
mkdir build run |
118 |
lfs setstripe -c -1 run |
119 |
cd build |
120 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_40x40 SIZE.h |
121 |
../tools/genmake2 -of \ |
122 |
../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \ |
123 |
-mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code |
124 |
make depend |
125 |
make -j 56 |
126 |
cd ../run |
127 |
ln -sf ../build/mitgcmuv . |
128 |
ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* . |
129 |
ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* . |
130 |
ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* . |
131 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* . |
132 |
mv data.exch2_40x40x9477 data.exch2 |
133 |
mpiexec -n 9477 ./mitgcmuv & |
134 |
|
135 |
============== |
136 |
# interactive 45x45x7488 tile configuration with MITgcm checkpoint66h |
137 |
cd ~/llc_1080 |
138 |
cvs co MITgcm_contrib/llc_hires/llc_1080 |
139 |
cvs co -r checkpoint66h MITgcm_code |
140 |
qsub -I -q devel -l select=268:ncpus=28:model=bro,walltime=2:00:00 -m abe |
141 |
module purge |
142 |
module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20 |
143 |
cd ~/llc_1080/MITgcm |
144 |
mkdir build run |
145 |
lfs setstripe -c -1 run |
146 |
cd build |
147 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_45x45 SIZE.h |
148 |
../tools/genmake2 -of \ |
149 |
../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \ |
150 |
-mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code |
151 |
make depend |
152 |
make -j 56 |
153 |
cd ../run |
154 |
ln -sf ../build/mitgcmuv . |
155 |
ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* . |
156 |
ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* . |
157 |
ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* . |
158 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* . |
159 |
mv data.exch2_45x45x7488 data.exch2 |
160 |
mpiexec -n 7488 ./mitgcmuv & |
161 |
|
162 |
============== |
163 |
# interactive 54x54x5200 tile configuration with MITgcm checkpoint66h |
164 |
cd ~/llc_1080 |
165 |
cvs co MITgcm_contrib/llc_hires/llc_1080 |
166 |
cvs co -r checkpoint66h MITgcm_code |
167 |
qsub -I -q devel -l select=268:ncpus=28:model=bro,walltime=2:00:00 -m abe |
168 |
module purge |
169 |
module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20 |
170 |
cd ~/llc_1080/MITgcm |
171 |
mkdir build run |
172 |
lfs setstripe -c -1 run |
173 |
cd build |
174 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_54x54 SIZE.h |
175 |
../tools/genmake2 -of \ |
176 |
../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \ |
177 |
-mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code |
178 |
make depend |
179 |
make -j 56 |
180 |
cd ../run |
181 |
ln -sf ../build/mitgcmuv . |
182 |
ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* . |
183 |
ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* . |
184 |
ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* . |
185 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* . |
186 |
mv data.exch2_54x54x5200 data.exch2 |
187 |
mpiexec -n 5200 ./mitgcmuv & |
188 |
|
189 |
============== |
190 |
# Interactive 90x90x1342 tile configuration with MITgcm checkpoint66h |
191 |
cd ~/llc_1080 |
192 |
cvs co MITgcm_contrib/llc_hires/llc_1080 |
193 |
cvs co -r checkpoint66h MITgcm_code |
194 |
qsub -I -q long -l select=48:ncpus=28:model=bro,walltime=120:00:00 -m abe |
195 |
module purge |
196 |
module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20 |
197 |
cd ~/llc_1080/MITgcm |
198 |
mkdir build run |
199 |
lfs setstripe -c -1 run |
200 |
cd build |
201 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h |
202 |
../tools/genmake2 -of \ |
203 |
../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \ |
204 |
-mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code |
205 |
make depend |
206 |
make -j 56 |
207 |
cd ../run |
208 |
ln -sf ../build/mitgcmuv . |
209 |
ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* . |
210 |
ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* . |
211 |
ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* . |
212 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* . |
213 |
mv data.exch2_90x90x1342 data.exch2 |
214 |
mpiexec -n 1342 ./mitgcmuv & |
215 |
|
216 |
============== |
217 |
# interactive 90x90x1342 tile configuration from scratch |
218 |
cd ~/llc_1080 |
219 |
cvs co MITgcm_contrib/llc_hires/llc_1080 |
220 |
cvs co -r checkpoint64p MITgcm_code |
221 |
qsub -I -q long -l select=48:ncpus=28:model=bro,walltime=120:00:00 -m abe |
222 |
module purge |
223 |
module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20 |
224 |
cd ~/llc_1080/MITgcm |
225 |
mkdir build run |
226 |
lfs setstripe -c -1 run |
227 |
cd build |
228 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h |
229 |
../tools/genmake2 -of \ |
230 |
../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \ |
231 |
-mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code |
232 |
make depend |
233 |
make -j 16 |
234 |
cd ../run |
235 |
ln -sf ../build/mitgcmuv . |
236 |
ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* . |
237 |
ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* . |
238 |
ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* . |
239 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* . |
240 |
mv data.exch2_90x90x1342 data.exch2 |
241 |
mpiexec -n 1342 ./mitgcmuv & |
242 |
|
243 |
===================== |
244 |
For interactive session, Ivy Bridge nodes: |
245 |
qsub -I -q debug -l select=48:ncpus=28:model=bro,walltime=02:00:00 -m abe -M email |
246 |
qsub -I -q devel -l select=170:ncpus=20:model=ivy,walltime=02:00:00 -m abe -M email |
247 |
qsub -I -q long -l select=170:ncpus=20:model=ivy,walltime=120:00:00 -m abe -M email |
248 |
qsub -I -q long -l select=170:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M email |
249 |
|
250 |
============== |
251 |
# 60x60x2872 tile configuration |
252 |
|
253 |
cd ~/llc_1080 |
254 |
cvs co -r checkpoint64p MITgcm_code |
255 |
cvs co MITgcm_contrib/llc_hires/llc_1080 |
256 |
cd MITgcm |
257 |
module purge |
258 |
module load comp-intel/2012.0.032 mpi-sgi/mpt.2.08r7 netcdf/4.0 |
259 |
mkdir build run |
260 |
lfs setstripe -c -1 run |
261 |
cd build |
262 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_60x60x2872 SIZE.h |
263 |
../tools/genmake2 -of \ |
264 |
../../MITgcm_contrib/llc_hires/llc_1080/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \ |
265 |
'../../MITgcm_contrib/llc_hires/llc_1080/code ../../MITgcm_contrib/llc_hires/llc_1080/code-async' |
266 |
make depend |
267 |
make -j 16 |
268 |
cd ~/llc_1080/MITgcm/run |
269 |
ln -sf ../build/mitgcmuv . |
270 |
ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* . |
271 |
ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* . |
272 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* . |
273 |
mv data.exch2_60x60x2872 data.exch2 |
274 |
export MPI_NUM_MEMORY_REGIONS=256 |
275 |
mpiexec -n 3400 ./mitgcmuv |
276 |
|
277 |
============== |
278 |
# 90x90x1342 tile configuration |
279 |
|
280 |
cd ~/llc_1080 |
281 |
cvs co MITgcm_code |
282 |
cvs co MITgcm_contrib/llc_hires/llc_1080 |
283 |
cd MITgcm |
284 |
module purge |
285 |
module load comp-intel/2011.2 mpi-sgi/mpt.2.06r6 netcdf/4.0 |
286 |
mkdir build run |
287 |
lfs setstripe -c -1 run |
288 |
cd build |
289 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h |
290 |
../tools/genmake2 -of \ |
291 |
../../MITgcm_contrib/llc_hires/llc_1080/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \ |
292 |
'../../MITgcm_contrib/llc_hires/llc_1080/code ../../MITgcm_contrib/llc_hires/llc_1080/code-async' |
293 |
make depend |
294 |
make -j 16 |
295 |
cd ../run |
296 |
ln -sf ../build/mitgcmuv . |
297 |
ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* . |
298 |
#ln -sf /nobackup/dmenemen/forcing/era_interim/EIG_*_2* . |
299 |
#ln -sf /nobackup/dmenemen/forcing/era_interim_corrected/EIG_dlw_sub5p_2* . |
300 |
ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* . |
301 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* . |
302 |
mv data.exch2_90x90x1342 data.exch2 |
303 |
export MPI_NUM_MEMORY_REGIONS=256 |
304 |
mpiexec -n 1600 ./mitgcmuv |
305 |
|
306 |
============== |
307 |
# 90x90x1342 tile test configuration |
308 |
|
309 |
cd ~/llc_1080_test |
310 |
cvs co MITgcm_code |
311 |
cvs co MITgcm_contrib/llc_hires/llc_1080 |
312 |
cd MITgcm |
313 |
module purge |
314 |
module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20 |
315 |
mkdir build run |
316 |
lfs setstripe -c -1 run |
317 |
cd build |
318 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h |
319 |
../tools/genmake2 -of \ |
320 |
../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \ |
321 |
-mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code |
322 |
make depend |
323 |
make -j 16 |
324 |
cd ../run |
325 |
ln -sf ../build/mitgcmuv . |
326 |
ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* . |
327 |
ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* . |
328 |
cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* . |
329 |
mv data.exch2_90x90x1342 data.exch2 |
330 |
mpiexec -n 1342 ./mitgcmuv |
331 |
|
332 |
|
333 |
============== |
334 |
|
335 |
look at output |
336 |
|
337 |
for ts=[50760] |
338 |
fld=quikread_llc(['Eta.' myint2str(ts,10) '.data'],1080); |
339 |
clf,quikplot_llc(fld),caxis([-2.5 2]),thincolorbar |
340 |
title(ts) |
341 |
pause(.1) |
342 |
end |
343 |
|
344 |
ts=50760; |
345 |
for fld={'S''T','U','V','W','HEFF'} |
346 |
tmp=quikread_llc([fld{1} '.' myint2str(ts,10) '.data'],1080); |
347 |
clf,quikplot_llc(tmp),thincolorbar |
348 |
title(fld{1}) |
349 |
pause |
350 |
end |
351 |
|
352 |
============== |
353 |
|
354 |
to determine empty tiles: |
355 |
grep Empty STDOUT.* |
356 |
|
357 |
============== |
358 |
|
359 |
memory requirements: |
360 |
nPx sNx sNy nSx node0 total rank0 rank1 |
361 |
702 60 60 6 17,642,924kb 770,168,568kb |
362 |
468 60 60 9 22,083,004kb 643,301,784kb |
363 |
468 180 180 1 24,719,776kb 720,415,552kb 1678MB 1563MB |
364 |
351 60 60 12 26,660,020kb 582,815,544kb |
365 |
351 120 120 3 25,110,324kb 548,819,772kb 1711MB 1586MB |
366 |
351 120 120 3 6,543,780kb 56,729,484kb - with globalFiles - run out of time |
367 |
325 54 54 16 29,029,744kb 587,831,344kb |
368 |
325 216 216 1 29,249,232kb 426,554,484kb - node ran out of memory and crashed |
369 |
234 90 90 8 30,590,952kb 431,252,200kb - node ran out of memory and crashed |
370 |
234 180 180 2 27,682,360kb 307,181,152kb - node ran out of memory and crashed |
371 |
|
372 |
============= |
373 |
factor(3240) = 2 2 2 3 3 3 3 5 |
374 |
factor(1080) = 2 2 2 3 3 3 5 |
375 |
|
376 |
2 = 2 |
377 |
3 = 3 |
378 |
2*2 = 4 |
379 |
5 = 5 |
380 |
2*3 = 6 |
381 |
2*2*2 = 8 |
382 |
3*3 = 9 |
383 |
2*5 = 10 |
384 |
2*2*3 = 12 |
385 |
3*5 = 15 |
386 |
2*3*3 = 18 |
387 |
2*2*5 = 20 |
388 |
2*2*2*3 = 24 |
389 |
3*3*3 = 27 |
390 |
2*3*5 = 30 |
391 |
2*2*3*3 = 36 * 30 |
392 |
2*2*2*5 = 40 * 27 |
393 |
3*3*5 = 45 * 24 |
394 |
2*3*3*3 = 54 * 20 |
395 |
2*2*3*5 = 60 * 18 |
396 |
2*2*2*3*3 = 72 * 15 |
397 |
2*3*3*5 = 90 * 12 |
398 |
2*2*3*3*3 = 108 * 10 |
399 |
2*2*2*3*5 = 120 * 9 |
400 |
3*3*3*5 = 135 * 8 |
401 |
2*2*3*3*5 = 180 * 6 |
402 |
2*2*2*3*3*3 = 216 * 5 |
403 |
2*3*3*3*5 = 270 * 4 |
404 |
2*2*2*3*3*5 = 360 * 3 |
405 |
2*2*3*3*3*5 = 540 * 2 |