Instructions for setting up a cube-sphere integration with sea-ice.

=================
UPDATES

December 5, 2006: 216-CPU code for cube46
Changed diagnostics to accomodate JMCs latest and some
changes to code-mods/CPP_EEOPTIONS.h and W2_OPTIONS.h
to speed up code.

November 7, 2006: 216-CPU code for cube43
Bug fixes in growth.F routine and new set of diagnostics

October 15, 2006: 216-CPU code for cube40

September 28, 2006: 216-CPU code for cube38

September 4, 2006
Modified 32*32*6*15 experiments to use grid_cs32.* files from
 verification/tutorial_held_suarez_cs/input instead of
 tile* files from verification/global_ocean.cs32x15/input

August 8, 2006: 216-CPU code and input used for cube37
using "June 10, 2006" MITgcm code.

for online T/S profile diagnostics all that is needed is
  profiles_init_fixed
  profiles_inloop
see pkg/ecco/ecco_cost_init_fixed.F
    pkg/ecco/cost_averagesfields.F

##########################################################################
##########################################################################
# Get forcing files, etc.
# Example below is for cube46 on /nobackup2a/menemenl

cd /nobackup2a/menemenl
mkdir cube46
cd cube46
ln -sf ../run_template .
ln -sf ../ncep_rgau .


##########################################################################
##########################################################################
# getting the code from anonymous CVS server

  bash or sh shell:
    $ export CVSROOT=':pserver:cvsanon@mitgcm.org:/u/gcmpack'
    $ cvs login 
      ( enter the CVS password: "cvsanon" )

  tcsh or csh shell:
    $ setenv CVSROOT ':pserver:cvsanon@mitgcm.org:/u/gcmpack'
    $ cvs login 
      ( enter the CVS password: "cvsanon" )

cvs co MITgcm_contrib/high_res_cube/README_ice
cvs co MITgcm_contrib/high_res_cube/code-mods
cvs co MITgcm_contrib/high_res_cube/input
cvs co MITgcm_contrib/high_res_cube/results
cvs co MITgcm_code
cvs co MITgcm/verification/global_ocean.cs32x15
cvs co MITgcm/verification/tutorial_held_suarez_cs/input


##########################################################################
##########################################################################
# 510*510*6*50, 216-cpu cube sphere on altix with ifort and pkg/diagnostics

cd MITgcm/verification/global_ocean.cs32x15/code
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/* .
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/s216t_85x85/* .
\cp SIZE.h_216 SIZE.h
\cp packages.conf_hr packages.conf

cd ../build
\rm *
../../../tools/genmake2 -mods=../code -of ../code/linux_ia64_ifort+mpi_altix_nas
make depend
make -j

cd ..
mkdir run
cd run
\rm *
\cp ../input/* .
\cp ../../../../MITgcm_contrib/high_res_cube/input/* .
ln -sf ../../../../run_template/* .
\rm pickup* eedata.mth data.pkg data.seaice data.exf_clim data.exf data.mnc data pr*
\rm bathy_Hmin50.bin *192_94* *cs32* *.m lev* tren* job_c* job_altix150 job_altix92_54t
ln -sf ../../../../run_template/pickup.0000000216.cube38 pickup.0000000216
ln -sf  ../../../../run_template/pickup_seaice.0000000216.cube38 pickup_seaice.0000000216
\cp ../build/mitgcmuv .
qsub job_altix


##########################################################################
##########################################################################
# 32*32*6*15, 1-cpu cube sphere

cd MITgcm/verification/global_ocean.cs32x15/code
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/* .
\cp ../../../utils/exch2/code-mods/s12t_16x32/* .
cd ../build
\rm *
../../../tools/genmake2 -mods=../code
make depend
make
cd ..
mkdir run
cd run
\rm *
\cp ../../tutorial_held_suarez_cs/input/grid* .
\cp ../input/* .
\cp ../../../../MITgcm_contrib/high_res_cube/input/* .
../build/mitgcmuv >& output.txt

comparison output is in:
../../../../MITgcm_contrib/high_res_cube/results/output.txt

to use matlab to look at the output
matlab

for fld={'SIuice','SIvice','SIheff','SIarea','oceQsw', ...
         'surForcT','oceTAUX','oceTAUY','surForcS'}
    tmp=permute(readbin([fld{1} '.0000000072.data'],[32,6,32]),[1 3 2]);
    cx=[min(tmp(:)) max(tmp(:))]; clf, plot_cube, title(fld{1}), pause
end

heff=permute(readbin(['SIheff.0000000072.data'],[32,6,32]),[1 3 2]);
mask=0*heff; mask(find(heff>.001))=1;
uice=mask.*permute(readbin(['SIuice.0000000072.data'],[32,6,32]),[1 3 2]);
vice=mask.*permute(readbin(['SIvice.0000000072.data'],[32,6,32]),[1 3 2]);
clf
subplot(221), mypcolor(uice(:,:,3)'); colorbar, title('uice, tile 3')
subplot(222), mypcolor(uice(:,:,6)'); colorbar, title('uice, tile 6')
subplot(223), mypcolor(vice(:,:,3)'); colorbar, title('vice, tile 3')
subplot(224), mypcolor(vice(:,:,6)'); colorbar, title('vice, tile 6')


##########################################################################
##########################################################################
# 32*32*6*15, 17-tile, 1-cpu cube sphere experiment

cd MITgcm/verification/global_ocean.cs32x15/code
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/* .
\cp ../code_alt/code.176t_8x4/* .
cd ../build
\rm *
../../../tools/genmake2 -mods=../code
make depend
make
cd ..
mkdir run
cd run
\rm *
\cp ../../tutorial_held_suarez_cs/input/grid* .
\cp ../input/* .
\cp ../../../../MITgcm_contrib/high_res_cube/input/* .
../build/mitgcmuv >& output.txt


##########################################################################
##########################################################################
# 32*32*6*15, 2-cpu cube sphere on the altices with ifort

cd MITgcm/verification/global_ocean.cs32x15/code
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/* .
\cp ../../../utils/exch2/code-mods/s12t_16x32/* .
\cp SIZE.h_mpi SIZE.h

cd ../build
\rm *
../../../tools/genmake2 -mods=../code -of ../../../tools/build_options/linux_ia64_ifort+mpi_altix_nas
make depend
make -j

cd ..
mkdir run
cd run
\rm *
\cp ../../tutorial_held_suarez_cs/input/grid* .
\cp ../input/* .
\cp ../../../../MITgcm_contrib/high_res_cube/input/* .
mpirun -np 2 ../build/mitgcmuv


##########################################################################
##########################################################################
# 510*510*6*50, 54-tile, 54-cpu cube sphere on the altices with efc
# starting with pickup file from end of 1992
# requires input files in sudirectories run_template and ncep_rgau

cd MITgcm/verification/global_ocean.cs32x15/code
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/* .
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/s54t_170x170/* .
\cp packages.conf_hr packages.conf

cd ../build
\rm *
../../../tools/genmake2 -mods=../code -of ../../../tools/build_options/linux_ia64_efc+mpi_altix
make depend
make -j

cd ..
mkdir run
cd run
\rm *
\cp ../input/* .
\cp ../../../../MITgcm_contrib/high_res_cube/input/* .
qsub job_altix92_54t


##########################################################################
##########################################################################
# 510*510*6*50, 1500-tile, 150-cpu, excludes land
# uses s1500t_17x51 configuration

cd MITgcm/verification/global_ocean.cs32x15/code
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/* .
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/s1500t_17x51/* .
\cp SIZE.h_150 SIZE.h
\cp packages.conf_hr packages.conf

cd ../build
\rm *
../../../tools/genmake2 -mods=../code -of ../../../tools/build_options/linux_ia64_ifort+mpi_altix_nas
make depend
make -j

cd ..
mkdir run
cd run
\rm *
\cp ../input/* .
\cp ../../../../MITgcm_contrib/high_res_cube/input/* .
ln -sf ../../../../run_template/* .
\rm pickup*
ln -sf ../../../../run_template/pickup.0000316440.cube19 pickup.0000000216
ln -sf  ../../../../run_template/pickup_seaice.0000316440.cube19 pickup_seaice.0000000216
\cp ../build/mitgcmuv mitgcmuv150
qsub job_altix150


##########################################################################
##########################################################################
# 510*510*6*50, 1500-tile, 375-cpu, excludes land
# uses s1500t_17x51 configuration
# example for running on c17-c20 using arrayd

cd MITgcm/verification/global_ocean.cs32x15/code
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/* .
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/s1500t_17x51/* .
\cp SIZE.h_375 SIZE.h
\cp packages.conf_hr packages.conf

cd ../build
\rm *
../../../tools/genmake2 -mods=../code -of ../../../tools/build_options/linux_ia64_ifort+mpi_altix_nas
make depend
make -j

cd ..
mkdir run
cd run
\rm *
\cp ../input/* .
\cp ../../../../MITgcm_contrib/high_res_cube/input/* .
ln -sf ../../../../run_template/* .
\rm pickup*
ln -sf ../../../../run_template/pickup.0000316440.cube19 pickup.0000000216
ln -sf  ../../../../run_template/pickup_seaice.0000316440.cube19 pickup_seaice.0000000216
\cp ../build/mitgcmuv .
job_c20_375cpu < /dev/null >&! output.run &


##########################################################################
##########################################################################
# 510*510*6*50, 1500-tile, 375-cpu, excludes land
# uses s1500t_17x51 configuration
# example for running on c17-c20 using arrayd

cd MITgcm/verification/global_ocean.cs32x15/code
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/* .
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/s1500t_17x51/* .
\cp SIZE.h_375 SIZE.h
\cp packages.conf_hr packages.conf

cd ../build
\rm *
../../../tools/genmake2 -mods=../code -of ../../../tools/build_options/linux_ia64_ifort+mpi_altix_nas
make depend
make -j

cd ..
mkdir run
cd run
\rm *
\cp ../input/* .
\cp ../../../../MITgcm_contrib/high_res_cube/input/* .
ln -sf ../../../../run_template/* .
\rm pickup*
ln -sf ../../../../run_template/pickup.0000316440.cube19 pickup.0000000216
ln -sf  ../../../../run_template/pickup_seaice.0000316440.cube19 pickup_seaice.0000000216
\cp ../build/mitgcmuv .
job_c20_375cpu < /dev/null >&! output.run &


##########################################################################
##########################################################################
# 510*510*6*50, 1500-tile, 500-cpu, excludes land
# uses s1500t_17x51 configuration
# example for running on c17-c20 using arrayd

cd MITgcm/verification/global_ocean.cs32x15/code
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/* .
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/s1500t_17x51/* .
\cp SIZE.h_500 SIZE.h
\cp packages.conf_hr packages.conf

cd ../build
\rm *
../../../tools/genmake2 -mods=../code -of ../../../tools/build_options/linux_ia64_ifort+mpi_altix_nas
make depend
make -j

cd ..
mkdir run
cd run
\rm *
\cp ../input/* .
\cp ../../../../MITgcm_contrib/high_res_cube/input/* .
ln -sf ../../../../run_template/* .
\rm pickup*
ln -sf ../../../../run_template/pickup.0000316440.cube19 pickup.0000000216
ln -sf  ../../../../run_template/pickup_seaice.0000316440.cube19 pickup_seaice.0000000216
\cp ../build/mitgcmuv .
job_c19_500cpu < /dev/null >&! output.run &


##########################################################################
##########################################################################
# 510*510*6*50, 216-cpu cube sphere on cosmos

cd MITgcm/verification/global_ocean.cs32x15/code
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/* .
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/s216t_85x85/* .
\cp SIZE.h_216 SIZE.h
\cp packages.conf_hr packages.conf

module unload latest_intel80
module unload mpich-gm-intel80
module load latest_intel81
module load mpich-gm-intel81

cd ../build
\rm *
../../../tools/genmake2 -mods=../code -of ../../../tools/build_options/linux_ia32_ifort+mpi_cosmos
make depend
make -j

cd ..
mkdir run
cd run
\rm *
\cp ../input/* .
\cp ../../../../MITgcm_contrib/high_res_cube/input/* .
ln -sf ../../../../run_template/* .
\rm pickup*
ln -sf ../../../../run_template/pickup.0000316440.cube19 pickup.0000000216
ln -sf  ../../../../run_template/pickup_seaice.0000316440.cube19 pickup_seaice.0000000216
\cp ../build/mitgcmuv .
bsub < job_cosmos


##########################################################################
##########################################################################
# 510*510*6*50, 216-tile, 54-cpu cube sphere on the altices with efc
# requires input files in sudirectories run_template and ncep_rgau
# presently residing under orion:/tmp1/dmenem/cube

cd MITgcm/verification/global_ocean.cs32x15/code
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/* .
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/s216t_85x85/* .
\cp SIZE.h_54 SIZE.h
\cp packages.conf_hr packages.conf

cd ../build
\rm *
../../../tools/genmake2 -mods=../code -of ../../../tools/build_options/linux_ia64_efc+mpi_altix
make depend
make -j

cd ..
mkdir run
cd run
\rm *
\cp ../input/* .
\cp ../../../../MITgcm_contrib/high_res_cube/input/* .
qsub job_orion


##########################################################################
##########################################################################
# 510*510*6*50, 205-tile, 41-cpu cube sphere on the altices with efc
# requires input files in sudirectories run_template and ncep_rgau
# presently residing under orion:/tmp1/dmenem/cube

cd MITgcm/verification/global_ocean.cs32x15/code
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/* .
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/s205t_85x85/* .
\cp SIZE.h_41 SIZE.h
\cp packages.conf_hr packages.conf

cd ../build
\rm *
../../../tools/genmake2 -mods=../code -of ../../../tools/build_options/linux_ia64_efc+mpi_altix
make depend
make -j

cd ..
mkdir run
cd run
\rm *
\cp ../input/* .
\cp ../../../../MITgcm_contrib/high_res_cube/input/* .
qsub job_orion41


##########################################################################
##########################################################################
# 510*510*6*50, 1500-tile, 50-cpu cube sphere on the altices with efc
# requires input files in sudirectories run_template and ncep_rgau
# presently residing under orion:/tmp1/dmenem/cube

cd MITgcm/verification/global_ocean.cs32x15/code
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/* .
\cp ../../../../MITgcm_contrib/high_res_cube/code-mods/s1500t_17x51/* .
\cp SIZE.h_50 SIZE.h
\cp packages.conf_hr packages.conf

cd ../build
\rm *
../../../tools/genmake2 -mods=../code -of ../../../tools/build_options/linux_ia64_efc+mpi_altix
make depend
make -j

cd ..
mkdir run
cd run
\rm *
\cp ../input/* .
\cp ../../../../MITgcm_contrib/high_res_cube/input/* .
qsub job_orion