/[MITgcm]/MITgcm_contrib/llc_hires/llc_2160/readme.txt
ViewVC logotype

Diff of /MITgcm_contrib/llc_hires/llc_2160/readme.txt

Parent Directory Parent Directory | Revision Log Revision Log | View Revision Graph Revision Graph | View Patch Patch

revision 1.3 by dimitri, Tue Oct 29 09:40:37 2013 UTC revision 1.14 by dimitri, Fri Nov 2 14:00:07 2018 UTC
# Line 1  Line 1 
1  For interactive session, Ivy Bridge nodes:  For interactive session, Ivy Bridge nodes:
2  qsub -I -q devel -l select=103:ncpus=20:model=ivy,walltime=02:00:00  qsub -I -q devel -l select=300:ncpus=20:model=ivy,walltime=02:00:00 -m abe -M YOUR_EMAIL_HERE
3  qsub -I -q long -l select=103:ncpus=20:model=ivy,walltime=120:00:00  qsub -I -q normal -l select=103:ncpus=20:model=ivy,walltime=8:00:00 -m abe -M YOUR_EMAIL_HERE
4  qsub -I -q long -l select=103:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00  qsub -I -q long -l select=300:ncpus=20:model=ivy,walltime=120:00:00 -m abe -M YOUR_EMAIL_HERE
5  qsub -I -q normal -l select=103:ncpus=20:model=ivy,walltime=8:00:00  qsub -I -q long -l select=103:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M YOUR_EMAIL_HERE
6  qsub -q ecco -I -W group_list=g26209 -l select=400:ncpus=20:aoe=sles11,walltime=100:00:00 -m abe -M menemenlis@jpl.nasa.gov  
7    ##########################################
8  For batch submission:  # 90x90_5004 configuration with newer code
9  qsub -q devel -l select=103:ncpus=20:model=ivy,walltime=02:00:00 runscript  qsub -I -q electra -l select=200:ncpus=28:model=bro_ele:aoe=sles12 -l walltime=10:00:00
10  qsub qsub_llc2160.csh  module purge
11    module load comp-intel/2016.2.181 mpi-sgi/mpt.2.14r19 hdf4/4.2.12 hdf5/1.8.18_mpt netcdf/4.4.1.1_mpt
12    module load comp-intel/2016.2.181 mpi-hpe/mpt.2.17r13 hdf4/4.2.12 hdf5/1.8.18_mpt netcdf/4.4.1.1_mpt
13    
14    mpt2.17r13
15    stripe to 16 or 32
16    7840
17    12040
18    
19    cd ~/llc_2160
20    git clone git@github.com:MITgcm/MITgcm.git
21    cvs co MITgcm_contrib/llc_hires/llc_2160
22    cd ~/llc_2160/MITgcm
23    mkdir build run
24    cd ~/llc_2160/MITgcm/build
25    cp ../../MITgcm_contrib/llc_hires/llc_2160/code/SIZE.h_90x90_5004 SIZE.h
26    ../tools/genmake2 -of \
27     ../../MITgcm_contrib/llc_hires/llc_2160/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
28     '../../MITgcm_contrib/llc_hires/llc_2160/code ../../MITgcm_contrib/llc_hires/llc_2160/code-async'
29    make depend
30    make -j 16
31    
32    cd ~/llc_2160/MITgcm/run
33    cp ../build/mitgcmuv mitgcmuv_90x90_5004
34    ln -sf /nobackup/dmenemen/tarballs/llc_2160/run_template/* .
35    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
36    cp ../../MITgcm_contrib/llc_hires/llc_2160/input/* .
37    cp ../../MITgcm_contrib/llc_hires/llc_2160/input/data.exch2_90x90x5004 data.exch2
38    emacs data
39    
40    mpiexec -n 5500 ./mitgcmuv_90x90_5004
41    
42    tail -f STDOUT.00000 | grep advcfl_W
43    
44    #############################
45    # 60x60x10882 configuration
46    
47    qsub -I -q long -l select=600:ncpus=20:model=ivy,min_walltime=30:00,max_walltime=120:00:00 -m abe -M YOUR_EMAIL_HERE
48    module purge
49    module load comp-intel/2012.0.032 mpi-sgi/mpt.2.10r6 netcdf/4.0
50    cd ~/llc_2160
51    cvs co -r checkpoint64t MITgcm_code
52    cvs co MITgcm_contrib/llc_hires/llc_2160
53    cd MITgcm
54    mkdir build run
55    cd build
56    cp ../../MITgcm_contrib/llc_hires/llc_2160/code/SIZE.h_60x60_10882 SIZE.h
57    cp ../../MITgcm_contrib/llc_hires/llc_2160/code-async/readtile_mpiio.c .
58    emacs readtile_mpiio.c
59        tileSizeX = 60;
60        tileSizeY = 60;
61    ../tools/genmake2 -of \
62     ../../MITgcm_contrib/llc_hires/llc_2160/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
63     '../../MITgcm_contrib/llc_hires/llc_2160/code ../../MITgcm_contrib/llc_hires/llc_2160/code-async'
64    make depend
65    make -j 16
66    
67    cd ~/llc_2160/MITgcm/run
68    cp ../build/mitgcmuv mitgcmuv_60x60x10882
69    ln -sf /nobackup/dmenemen/tarballs/llc_2160/run_template/* .
70    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
71    cp ../../MITgcm_contrib/llc_hires/llc_2160/input/* .
72    cp ../../MITgcm_contrib/llc_hires/llc_2160/input/data.exch2_60x60x10882 data.exch2
73    emacs data
74    
75    export MPI_BUFS_PER_PROC=1024
76    export MPI_REQUEST_MAX=65536
77    export MPI_GROUP_MAX=1024
78    export MPI_NUM_MEMORY_REGIONS=8
79    export MPI_UNBUFFERED_STDIO=1
80    export MPI_MEMMAP_OFF=1
81    export MPI_UD_TIMEOUT=100
82    mpiexec -n 12000 ./mitgcmuv_60x60x10882
83    
84    tail -f STDOUT.00000 | grep advcfl_W
85    
86    ################################################
87    # 144x144x2047 configuration for grid generation
88    
89    qsub -I -q devel -l select=103:ncpus=20:model=ivy,walltime=02:00:00 -m abe -M YOUR_EMAIL_HERE
90    module purge
91    module load comp-intel/2012.0.032 mpi-sgi/mpt.2.10r6 netcdf/4.0
92    cd ~/llc_2160
93    cvs co MITgcm_code
94    cvs co MITgcm_contrib/llc_hires/llc_2160
95    cd MITgcm
96    mkdir build run_grid
97    lfs setstripe -c -1 run_grid
98    cd build
99    cp ../../MITgcm_contrib/llc_hires/llc_2160/code/SIZE.h_144x144x2047 SIZE.h
100    ../tools/genmake2 -of \
101     ../../MITgcm_contrib/llc_hires/llc_2160/code-async/linux_amd64_ifort+mpi_ice_nas \
102     -mpi  -mods ../../MITgcm_contrib/llc_hires/llc_2160/code
103    make depend
104    make -j 16
105    
106    cd ~/llc_2160/MITgcm/run_grid
107    cp ../build/mitgcmuv mitgcmuv_144x144x2047
108    ln -sf /nobackup/dmenemen/tarballs/llc_2160/run_template/* .
109    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
110    cp ../../MITgcm_contrib/llc_hires/llc_2160/input/* .
111    cp ../../MITgcm_contrib/llc_hires/llc_2160/input/data.exch2_144x144x2047 data.exch2
112    
113    emacs data
114     debuglevel=3,
115     useSingleCPUio=.TRUE.,
116     endtime=0.,
117     deltaT = 1.,
118    
119    mpiexec -n 2047 ./mitgcmuv_144x144x2047
120    
121  ==============  ==============
122    
# Line 16  cvs co MITgcm_code Line 125  cvs co MITgcm_code
125  cvs co MITgcm_contrib/llc_hires/llc_2160  cvs co MITgcm_contrib/llc_hires/llc_2160
126  cd MITgcm  cd MITgcm
127  module purge  module purge
128  module load comp-intel/2012.0.032 mpi-sgi/mpt.2.06rp16  netcdf/4.0  module load comp-intel/2012.0.032 mpi-sgi/mpt.2.08r7 netcdf/4.0
129  mkdir build run  mkdir build run
130  lfs setstripe -c -1 run  lfs setstripe -c -1 run
131  cd build  cd build
# Line 26  cp ../../MITgcm_contrib/llc_hires/llc_21 Line 135  cp ../../MITgcm_contrib/llc_hires/llc_21
135   '../../MITgcm_contrib/llc_hires/llc_2160/code ../../MITgcm_contrib/llc_hires/llc_2160/code-async'   '../../MITgcm_contrib/llc_hires/llc_2160/code ../../MITgcm_contrib/llc_hires/llc_2160/code-async'
136  make depend  make depend
137  make -j 16  make -j 16
138  cd ../run  cd ~/llc_2160/MITgcm/run
139  ln -sf ../build/mitgcmuv .  ln -sf ../build/mitgcmuv .
140  ln -sf /nobackup/dmenemen/tarballs/llc_2160/run_template/* .  ln -sf /nobackup/dmenemen/tarballs/llc_2160/run_template/* .
141  ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .  ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
142  cp ../../MITgcm_contrib/llc_hires/llc_2160/input/* .  cp ../../MITgcm_contrib/llc_hires/llc_2160/input/* .
143  mv data.exch2_90x90x5004 data.exch2  mv data.exch2_90x90x5004 data.exch2
144  mpiexec -n 8000 ./mitgcmuv  export MPI_NUM_MEMORY_REGIONS=256
145    mpiexec -n 6000 ./mitgcmuv
146    
147  ==============  ==============
148    
# Line 48  end Line 158  end
158  ==============  ==============
159    
160  to determine empty tiles:  to determine empty tiles:
161  grep Empty STDOUT.*  grep Empty STDOUT.* > empty.txt
162    
163    ==============
164    
165    # generate 30x30 blank tiles
166    qsub -I -q normal -l select=339:ncpus=28:model=bro,walltime=2:00:00 -m abe
167    module purge
168    module load comp-intel/2016.2.181 mpi-sgi/mpt.2.14r19 hdf4/4.2.12 hdf5/1.8.18_mpt netcdf/4.4.1.1_mpt
169    cd ~/llc_2160/MITgcm
170    mkdir run_30x30
171    lfs setstripe -c -1 run_30x30
172    cd build
173    rm *
174    cp ../../MITgcm_contrib/llc_hires/llc_2160/code-async/readtile_mpiio.c .
175    emacs readtile_mpiio.c
176        tileSizeX = 30;
177        tileSizeY = 30;
178    cp ../../MITgcm_contrib/llc_hires/llc_2160/code/SIZE.h_60x60_10882 SIZE.h
179    emacs SIZE.h
180         &           sNx =  30,
181         &           sNy =  30,
182         &           nSx =   8,
183         &           nPx = 8424,
184         &           Nr  =  2 )
185    cp ../../MITgcm_contrib/llc_hires/llc_2160/code-async/eeboot_minimal.F .
186    emacs eeboot_minimal.F
187    C         standardMessageUnit=errorMessageUnit
188             WRITE(fNam,'(A,A)') 'STDOUT.', myProcessStr(1:5)
189             OPEN(standardMessageUnit,FILE=fNam,STATUS='unknown')
190    ../tools/genmake2 -of \
191     ../../MITgcm_contrib/llc_hires/llc_2160/code-async/linux_amd64_ifort+mpi_ice_nas -mpi -mods \
192     '../../MITgcm_contrib/llc_hires/llc_2160/code ../../MITgcm_contrib/llc_hires/llc_2160/code-async'
193    make depend
194    make -j 16
195    cd ~/llc_2160/MITgcm/run_30x30
196    cp ../build/mitgcmuv .
197    ln -sf /nobackup/dmenemen/tarballs/llc_2160/run_template/* .
198    ln -sf /nobackup/dmenemen/forcing/ECMWF_operational/* .
199    cp ../../MITgcm_contrib/llc_hires/llc_2160/input/* .
200    mv data.exch2_144x144x2047 data.exch2
201    emacs data.exch2
202    # remove the blank tile list
203    emacs data
204     tRef =  18.89, 18.89,
205     sRef =  34.84, 34.84,
206     endtime=45.,
207     delR =   1.00,    1.14,
208    # hydrogThetaFile='THETA_llc1080_14jan2011_2160x28080x90_r4',
209    # hydrogSaltFile ='SALT_llc1080_14jan2011_2160x28080x90_r4',
210    # uVelInitFile   ='UVEL_llc1080_14jan2011_2160x28080x90_r4',
211    # vVelInitFile   ='VVEL_llc1080_14jan2011_2160x28080x90_r4',
212    # pSurfInitFile  ='ETAN_llc1080_14jan2011_2160x28080_r4',
213    mpiexec -n 9492 ./mitgcmuv
214    
215  ==============  ==============
216    
Legend:
Removed from v.1.3  
changed lines
  Added in v.1.14
  ViewVC Help
Powered by ViewVC 1.1.22