/[MITgcm]/MITgcm_contrib/llc_hires/llc_2160/readme.txt
ViewVC logotype

Diff of /MITgcm_contrib/llc_hires/llc_2160/readme.txt

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph | View Patch Patch

revision 1.1 by dimitri, Fri Sep 20 12:38:02 2013 UTC revision 1.15 by dimitri, Fri Nov 2 14:57:07 2018 UTC
# Line 1  Line 1 
1  For interactive session, Ivy Bridge nodes:  For interactive session, Ivy Bridge nodes:
2  qsub -I -q devel -l select=103:ncpus=20:model=ivy,walltime=02:00:00  qsub -I -q devel -l select=300:ncpus=20:model=ivy,walltime=02:00:00 -m abe
3  qsub -I -q long -l select=103:ncpus=20:model=ivy,walltime=120:00:00  qsub -I -q normal -l select=103:ncpus=20:model=ivy,walltime=8:00:00 -m abe
4  qsub -I -q long -l select=103:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00  qsub -I -q long -l select=300:ncpus=20:model=ivy,walltime=120:00:00 -m abe
5  qsub -I -q normal -l select=103:ncpus=20:model=ivy,walltime=8:00:00  qsub -I -q long -l select=103:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe
6    
7  For batch submission:  ##########################
8  qsub -q devel -l select=103:ncpus=20:model=ivy,walltime=02:00:00 runscript  # 72x72_7666 configuration
9  qsub qsub_llc2160.csh  qsub -I -q long -l select=400:ncpus=20:model=ivy,walltime=120:00:00 -m abe
10    module purge
11    module load comp-intel/2016.2.181 mpi-hpe/mpt.2.17r13 hdf4/4.2.12 hdf5/1.8.18_mpt netcdf/4.4.1.1_mpt
12    cd ~/llc_2160
13    git clone git@github.com:MITgcm/MITgcm.git
14    cvs co MITgcm_contrib/llc_hires/llc_2160
15    cd ~/llc_2160/MITgcm
16    mkdir build run
17    cd ~/llc_2160/MITgcm/build
18    cp ../../MITgcm_contrib/llc_hires/llc_2160/code/SIZE.h_72x72_7666 SIZE.h
19    ../tools/genmake2 -of \
20     ../../MITgcm_contrib/llc_hires/llc_2160/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
21     '../../MITgcm_contrib/llc_hires/llc_2160/code ../../MITgcm_contrib/llc_hires/llc_2160/code-async'
22    make depend
23    make -j 16
24    
25    cd ~/llc_2160/MITgcm/run
26    cp ../build/mitgcmuv mitgcmuv_72x72_7666
27    ln -sf /nobackup/dmenemen/tarballs/llc_2160/run_template/* .
28    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
29    cp ../../MITgcm_contrib/llc_hires/llc_2160/input/* .
30    cp data.exch2_72x72x7666 data.exch2
31    emacs data
32    
33    mpiexec -n 8000 ./mitgcmuv_72x72_7666
34    
35    tail -f STDOUT.00000 | grep advcfl_W
36    
37    ###################################################################
38    # 90x90_5004 configuration with newer code on electra special queue
39    qsub -I -q electra -l select=200:ncpus=28:model=bro_ele:aoe=sles12 -l walltime=10:00:00
40    module purge
41    module load comp-intel/2016.2.181 mpi-hpe/mpt.2.17r13 hdf4/4.2.12 hdf5/1.8.18_mpt netcdf/4.4.1.1_mpt
42    cd ~/llc_2160
43    git clone git@github.com:MITgcm/MITgcm.git
44    cvs co MITgcm_contrib/llc_hires/llc_2160
45    cd ~/llc_2160/MITgcm
46    mkdir build run
47    cd ~/llc_2160/MITgcm/build
48    cp ../../MITgcm_contrib/llc_hires/llc_2160/code/SIZE.h_90x90_5004 SIZE.h
49    ../tools/genmake2 -of \
50     ../../MITgcm_contrib/llc_hires/llc_2160/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
51     '../../MITgcm_contrib/llc_hires/llc_2160/code ../../MITgcm_contrib/llc_hires/llc_2160/code-async'
52    make depend
53    make -j 16
54    
55    cd ~/llc_2160/MITgcm/run
56    cp ../build/mitgcmuv mitgcmuv_90x90_5004
57    ln -sf /nobackup/dmenemen/tarballs/llc_2160/run_template/* .
58    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
59    cp ../../MITgcm_contrib/llc_hires/llc_2160/input/* .
60    cp ../../MITgcm_contrib/llc_hires/llc_2160/input/data.exch2_90x90x5004 data.exch2
61    emacs data
62    
63    mpiexec -n 5500 ./mitgcmuv_90x90_5004
64    
65    tail -f STDOUT.00000 | grep advcfl_W
66    
67    #############################
68    # 60x60x10882 configuration
69    
70    qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe
71    module purge
72    module load comp-intel/2012.0.032 mpi-sgi/mpt.2.10r6 netcdf/4.0
73    cd ~/llc_2160
74    cvs co -r checkpoint64t MITgcm_code
75    cvs co MITgcm_contrib/llc_hires/llc_2160
76    cd MITgcm
77    mkdir build run
78    cd build
79    cp ../../MITgcm_contrib/llc_hires/llc_2160/code/SIZE.h_60x60_10882 SIZE.h
80    cp ../../MITgcm_contrib/llc_hires/llc_2160/code-async/readtile_mpiio.c .
81    emacs readtile_mpiio.c
82        tileSizeX = 60;
83        tileSizeY = 60;
84    ../tools/genmake2 -of \
85     ../../MITgcm_contrib/llc_hires/llc_2160/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
86     '../../MITgcm_contrib/llc_hires/llc_2160/code ../../MITgcm_contrib/llc_hires/llc_2160/code-async'
87    make depend
88    make -j 16
89    
90    cd ~/llc_2160/MITgcm/run
91    cp ../build/mitgcmuv mitgcmuv_60x60x10882
92    ln -sf /nobackup/dmenemen/tarballs/llc_2160/run_template/* .
93    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
94    cp ../../MITgcm_contrib/llc_hires/llc_2160/input/* .
95    cp ../../MITgcm_contrib/llc_hires/llc_2160/input/data.exch2_60x60x10882 data.exch2
96    emacs data
97    
98    export MPI_BUFS_PER_PROC=1024
99    export MPI_REQUEST_MAX=65536
100    export MPI_GROUP_MAX=1024
101    export MPI_NUM_MEMORY_REGIONS=8
102    export MPI_UNBUFFERED_STDIO=1
103    export MPI_MEMMAP_OFF=1
104    export MPI_UD_TIMEOUT=100
105    mpiexec -n 12000 ./mitgcmuv_60x60x10882
106    
107    tail -f STDOUT.00000 | grep advcfl_W
108    
109    ################################################
110    # 144x144x2047 configuration for grid generation
111    
112    qsub -I -q devel -l select=103:ncpus=20:model=ivy,walltime=02:00:00 -m abe -M YOUR_EMAIL_HERE
113    module purge
114    module load comp-intel/2012.0.032 mpi-sgi/mpt.2.10r6 netcdf/4.0
115    cd ~/llc_2160
116    cvs co MITgcm_code
117    cvs co MITgcm_contrib/llc_hires/llc_2160
118    cd MITgcm
119    mkdir build run_grid
120    lfs setstripe -c -1 run_grid
121    cd build
122    cp ../../MITgcm_contrib/llc_hires/llc_2160/code/SIZE.h_144x144x2047 SIZE.h
123    ../tools/genmake2 -of \
124     ../../MITgcm_contrib/llc_hires/llc_2160/code-async/linux_amd64_ifort+mpi_ice_nas \
125     -mpi  -mods ../../MITgcm_contrib/llc_hires/llc_2160/code
126    make depend
127    make -j 16
128    
129    cd ~/llc_2160/MITgcm/run_grid
130    cp ../build/mitgcmuv mitgcmuv_144x144x2047
131    ln -sf /nobackup/dmenemen/tarballs/llc_2160/run_template/* .
132    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
133    cp ../../MITgcm_contrib/llc_hires/llc_2160/input/* .
134    cp ../../MITgcm_contrib/llc_hires/llc_2160/input/data.exch2_144x144x2047 data.exch2
135    
136    emacs data
137     debuglevel=3,
138     useSingleCPUio=.TRUE.,
139     endtime=0.,
140     deltaT = 1.,
141    
142    mpiexec -n 2047 ./mitgcmuv_144x144x2047
143    
144  ==============  ==============
145    
146    cd ~/llc_2160
147  cvs co MITgcm_code  cvs co MITgcm_code
148    cvs co MITgcm_contrib/llc_hires/llc_2160
149  cd MITgcm  cd MITgcm
150  module purge  module purge
151  module load comp-intel/2011.7.256 mpi-sgi/mpt.2.08r7 netcdf/4.0  module load comp-intel/2012.0.032 mpi-sgi/mpt.2.08r7 netcdf/4.0
152  mkdir build run  mkdir build run
153  lfs setstripe -c -1 /nobackupp5/dmenemen/llc_2160/MITgcm/run  lfs setstripe -c -1 run
154  cd build  cd build
155  ../tools/genmake2 -of ~/tarballs/llc_2160/code/linux_amd64_ifort+mpi_ice_nas -mpi -mods ~/tarballs/llc_2160/code  cp ../../MITgcm_contrib/llc_hires/llc_2160/code/SIZE.h_90x90_5004 SIZE.h
156    ../tools/genmake2 -of \
157     ../../MITgcm_contrib/llc_hires/llc_2160/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
158     '../../MITgcm_contrib/llc_hires/llc_2160/code ../../MITgcm_contrib/llc_hires/llc_2160/code-async'
159  make depend  make depend
160  make -j 16  make -j 16
161  cd ../run  cd ~/llc_2160/MITgcm/run
162  ln -sf ../build/mitgcmuv .  ln -sf ../build/mitgcmuv .
163  ln -sf /nobackup/dmenemen/tarballs/llc_2160/run_template/* .  ln -sf /nobackup/dmenemen/tarballs/llc_2160/run_template/* .
164  ln -sf /nobackup/dmenemen/forcing/era_interim/EIG_*_2* .  ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
165  ln -sf /nobackup/dmenemen/forcing/era_interim_corrected/EIG_dlw_sub5p_2* .  cp ../../MITgcm_contrib/llc_hires/llc_2160/input/* .
166  cp /nobackup/dmenemen/tarballs/llc_2160/input/* .  mv data.exch2_90x90x5004 data.exch2
167  mpiexec -n 2047 ./mitgcmuv  export MPI_NUM_MEMORY_REGIONS=256
168    mpiexec -n 6000 ./mitgcmuv
169    
170  ==============  ==============
171    
# Line 42  end Line 181  end
181  ==============  ==============
182    
183  to determine empty tiles:  to determine empty tiles:
184  grep Empty STDOUT.*  grep Empty STDOUT.* > empty.txt
185    
186    ==============
187    
188    # generate 30x30 blank tiles
189    qsub -I -q normal -l select=339:ncpus=28:model=bro,walltime=2:00:00 -m abe
190    module purge
191    module load comp-intel/2016.2.181 mpi-sgi/mpt.2.14r19 hdf4/4.2.12 hdf5/1.8.18_mpt netcdf/4.4.1.1_mpt
192    cd ~/llc_2160/MITgcm
193    mkdir run_30x30
194    lfs setstripe -c -1 run_30x30
195    cd build
196    rm *
197    cp ../../MITgcm_contrib/llc_hires/llc_2160/code-async/readtile_mpiio.c .
198    emacs readtile_mpiio.c
199        tileSizeX = 30;
200        tileSizeY = 30;
201    cp ../../MITgcm_contrib/llc_hires/llc_2160/code/SIZE.h_60x60_10882 SIZE.h
202    emacs SIZE.h
203         &           sNx =  30,
204         &           sNy =  30,
205         &           nSx =   8,
206         &           nPx = 8424,
207         &           Nr  =  2 )
208    cp ../../MITgcm_contrib/llc_hires/llc_2160/code-async/eeboot_minimal.F .
209    emacs eeboot_minimal.F
210    C         standardMessageUnit=errorMessageUnit
211             WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)
212             OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')
213    ../tools/genmake2 -of \
214     ../../MITgcm_contrib/llc_hires/llc_2160/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
215     '../../MITgcm_contrib/llc_hires/llc_2160/code ../../MITgcm_contrib/llc_hires/llc_2160/code-async'
216    make depend
217    make -j 16
218    cd ~/llc_2160/MITgcm/run_30x30
219    cp ../build/mitgcmuv .
220    ln -sf /nobackup/dmenemen/tarballs/llc_2160/run_template/* .
221    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
222    cp ../../MITgcm_contrib/llc_hires/llc_2160/input/* .
223    mv data.exch2_144x144x2047 data.exch2
224    emacs data.exch2
225    # remove the blank tile list
226    emacs data
227     tRef =  18.89, 18.89,
228     sRef =  34.84, 34.84,
229     endtime=45.,
230     delR =   1.00,    1.14,
231    # hydrogThetaFile='THETA_llc1080_14jan2011_2160x28080x90_r4',
232    # hydrogSaltFile ='SALT_llc1080_14jan2011_2160x28080x90_r4',
233    # uVelInitFile   ='UVEL_llc1080_14jan2011_2160x28080x90_r4',
234    # vVelInitFile   ='VVEL_llc1080_14jan2011_2160x28080x90_r4',
235    # pSurfInitFile  ='ETAN_llc1080_14jan2011_2160x28080_r4',
236    mpiexec -n 9492 ./mitgcmuv
237    
238  ==============  ==============
239    
Legend:
Removed from v.1.1  
changed lines
  Added in v.1.15
  ViewVC Help
Powered by ViewVC 1.1.22