/[MITgcm]/MITgcm_contrib/llc_hires/llc_1080/readme.txt
ViewVC logotype

Contents of /MITgcm_contrib/llc_hires/llc_1080/readme.txt

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph

Revision 1.12 - (show annotations) (download)
Wed Jun 7 00:32:51 2017 UTC (8 years, 1 month ago) by dimitri
Branch: MAIN
Changes since 1.11: +27 -0 lines
File MIME type: text/plain 
Updated to checkpoint66h. Similar but not identical to checkpoint64p.

For checkpoint66h:
bash-3.2$ grep advcfl_W STDOUT.0000 | head
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   8.0765112988021E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   4.6646790127962E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   4.2944741238975E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   4.0442152223351E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   5.3384062112729E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   3.9344950973214E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   3.0813966489876E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   3.2877764643888E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   3.8223244494440E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   3.6377593176494E-01

For checkpoint64p:
bash-3.2$ grep advcfl_W STDOUT.0000 | head
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   8.0765112988021E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   4.6646790128502E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   4.2944740988061E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   4.0440425691133E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   5.3384765401778E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   3.9344949519849E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   3.0813964719293E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   3.2877763699241E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   3.8221353737459E-01
(PID.TID 0000.0001) %MON advcfl_W_hf_max              =   3.6377680028628E-01
1 ==============
2 # interactive 90x90x1342 tile configuration with newer MITgcm code
3 cd ~/llc_1080
4 cvs co MITgcm_contrib/llc_hires/llc_1080
5 cvs co -r checkpoint66h MITgcm_code
6 qsub -I -q long -l select=48:ncpus=28:model=bro,walltime=120:00:00 -m abe
7 module purge
8 module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
9 cd ~/llc_1080/MITgcm
10 mkdir build run
11 lfs setstripe -c -1 run
12 cd build
13 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h
14 ../tools/genmake2 -of \
15 ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
16 -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
17 make depend
18 make -j 56
19 cd ../run
20 ln -sf ../build/mitgcmuv .
21 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
22 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
23 ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
24 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
25 mv data.exch2_90x90x1342 data.exch2
26 mpiexec -n 1342 ./mitgcmuv &
27
28 ==============
29 # interactive 90x90x1342 tile configuration from scratch
30 cd ~/llc_1080
31 cvs co MITgcm_contrib/llc_hires/llc_1080
32 cvs co -r checkpoint64p MITgcm_code
33 qsub -I -q long -l select=48:ncpus=28:model=bro,walltime=120:00:00 -m abe
34 module purge
35 module load comp-intel/2016.2.181 mpi-sgi/mpt.2.15r20
36 cd ~/llc_1080/MITgcm
37 mkdir build run
38 lfs setstripe -c -1 run
39 cd build
40 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h
41 ../tools/genmake2 -of \
42 ../../MITgcm_contrib/llc_hires/llc_1080/code/linux_amd64_ifort+mpi_ice_nas \
43 -mpi -mods ../../MITgcm_contrib/llc_hires/llc_1080/code
44 make depend
45 make -j 16
46 cd ../run
47 ln -sf ../build/mitgcmuv .
48 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
49 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
50 ln -sf ~dmenemen/llc_1080/MITgcm/run_2011/pick*354240* .
51 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
52 mv data.exch2_90x90x1342 data.exch2
53 mpiexec -n 1342 ./mitgcmuv &
54
55 =====================
56 For interactive session, Ivy Bridge nodes:
57 qsub -I -q devel -l select=170:ncpus=20:model=ivy,walltime=02:00:00 -m abe -M email
58 qsub -I -q long -l select=170:ncpus=20:model=ivy,walltime=120:00:00 -m abe -M email
59 qsub -I -q long -l select=170:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M email
60
61 ==============
62 # 60x60x2872 tile configuration
63
64 cd ~/llc_1080
65 cvs co -r checkpoint64p MITgcm_code
66 cvs co MITgcm_contrib/llc_hires/llc_1080
67 cd MITgcm
68 module purge
69 module load comp-intel/2012.0.032 mpi-sgi/mpt.2.08r7 netcdf/4.0
70 mkdir build run
71 lfs setstripe -c -1 run
72 cd build
73 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_60x60x2872 SIZE.h
74 ../tools/genmake2 -of \
75 ../../MITgcm_contrib/llc_hires/llc_1080/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
76 '../../MITgcm_contrib/llc_hires/llc_1080/code ../../MITgcm_contrib/llc_hires/llc_1080/code-async'
77 make depend
78 make -j 16
79 cd ~/llc_1080/MITgcm/run
80 ln -sf ../build/mitgcmuv .
81 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
82 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
83 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
84 mv data.exch2_60x60x2872 data.exch2
85 export MPI_NUM_MEMORY_REGIONS=256
86 mpiexec -n 3400 ./mitgcmuv
87
88 ==============
89 # 90x90x1342 tile configuration
90
91 cd ~/llc_1080
92 cvs co MITgcm_code
93 cvs co MITgcm_contrib/llc_hires/llc_1080
94 cd MITgcm
95 module purge
96 module load comp-intel/2011.2 mpi-sgi/mpt.2.06r6 netcdf/4.0
97 mkdir build run
98 lfs setstripe -c -1 run
99 cd build
100 cp ../../MITgcm_contrib/llc_hires/llc_1080/code/SIZE.h_90x90x1342 SIZE.h
101 ../tools/genmake2 -of \
102 ../../MITgcm_contrib/llc_hires/llc_1080/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
103 '../../MITgcm_contrib/llc_hires/llc_1080/code ../../MITgcm_contrib/llc_hires/llc_1080/code-async'
104 make depend
105 make -j 16
106 cd ../run
107 ln -sf ../build/mitgcmuv .
108 ln -sf /nobackup/dmenemen/tarballs/llc_1080/run_template/* .
109 #ln -sf /nobackup/dmenemen/forcing/era_interim/EIG_*_2* .
110 #ln -sf /nobackup/dmenemen/forcing/era_interim_corrected/EIG_dlw_sub5p_2* .
111 ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
112 cp ../../MITgcm_contrib/llc_hires/llc_1080/input/* .
113 mv data.exch2_90x90x1342 data.exch2
114 export MPI_NUM_MEMORY_REGIONS=256
115 mpiexec -n 1600 ./mitgcmuv
116
117 ==============
118
119 look at output
120
121 for ts=[50760]
122 fld=quikread_llc(['Eta.' myint2str(ts,10) '.data'],1080);
123 clf,quikplot_llc(fld),caxis([-2.5 2]),thincolorbar
124 title(ts)
125 pause(.1)
126 end
127
128 ts=50760;
129 for fld={'S''T','U','V','W','HEFF'}
130 tmp=quikread_llc([fld{1} '.' myint2str(ts,10) '.data'],1080);
131 clf,quikplot_llc(tmp),thincolorbar
132 title(fld{1})
133 pause
134 end
135
136 ==============
137
138 to determine empty tiles:
139 grep Empty STDOUT.*
140
141 ==============
142
143 memory requirements:
144 nPx sNx sNy nSx node0 total rank0 rank1
145 702 60 60 6 17,642,924kb 770,168,568kb
146 468 60 60 9 22,083,004kb 643,301,784kb
147 468 180 180 1 24,719,776kb 720,415,552kb 1678MB 1563MB
148 351 60 60 12 26,660,020kb 582,815,544kb
149 351 120 120 3 25,110,324kb 548,819,772kb 1711MB 1586MB
150 351 120 120 3 6,543,780kb 56,729,484kb - with globalFiles - run out of time
151 325 54 54 16 29,029,744kb 587,831,344kb
152 325 216 216 1 29,249,232kb 426,554,484kb - node ran out of memory and crashed
153 234 90 90 8 30,590,952kb 431,252,200kb - node ran out of memory and crashed
154 234 180 180 2 27,682,360kb 307,181,152kb - node ran out of memory and crashed
155
156 =============
157 factor(3240) = 2 2 2 3 3 3 3 5
158 factor(1080) = 2 2 2 3 3 3 5
159
160 2 = 2
161 3 = 3
162 2*2 = 4
163 5 = 5
164 2*3 = 6
165 2*2*2 = 8
166 3*3 = 9
167 2*5 = 10
168 2*2*3 = 12
169 3*5 = 15
170 2*3*3 = 18
171 2*2*5 = 20
172 2*2*2*3 = 24
173 3*3*3 = 27
174 2*3*5 = 30
175 2*2*3*3 = 36 * 30
176 2*2*2*5 = 40 * 27
177 3*3*5 = 45 * 24
178 2*3*3*3 = 54 * 20
179 2*2*3*5 = 60 * 18
180 2*2*2*3*3 = 72 * 15
181 2*3*3*5 = 90 * 12
182 2*2*3*3*3 = 108 * 10
183 2*2*2*3*5 = 120 * 9
184 3*3*3*5 = 135 * 8
185 2*2*3*3*5 = 180 * 6
186 2*2*2*3*3*3 = 216 * 5
187 2*3*3*3*5 = 270 * 4
188 2*2*2*3*3*5 = 360 * 3
189 2*2*3*3*3*5 = 540 * 2
  ViewVC Help
Powered by ViewVC 1.1.22