--- MITgcm/tools/build_options/linux_ia32_ifort+mpi_aces	2009/06/15 02:21:47	1.7
+++ MITgcm/tools/build_options/linux_ia32_ifort+mpi_aces	2009/06/26 16:06:08	1.8
@@ -1,33 +1,68 @@
 #!/bin/bash
 #
-#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/tools/build_options/Attic/linux_ia32_ifort+mpi_aces,v 1.7 2009/06/15 02:21:47 jmc Exp $
+#  $Header: /home/ubuntu/mnt/e9_copy/MITgcm/tools/build_options/Attic/linux_ia32_ifort+mpi_aces,v 1.8 2009/06/26 16:06:08 jmc Exp $
 #
-#  These are the build options used with the Intel 8.x compiler for the
-#  daily testing on ACES (which are initiated by cron jobs on the "ao"
-#  head node).
+#  build options used with the Intel compiler, version 8 and 9,
+#  for the daily testing on ACES (which are initiated by cron jobs 
+#  on the "ao" head node).
 #
-#  Please note that the appropriate module commands:
+#  Needs the appropriate module commands,
+#  and DON'T FORGET to set environment variable MPI_INC_DIR to the include
+#  directory of the selected MPI implementation
 #
+#-- using default intel (v8.1) and default mpich/intel:
 #    module add mpich/intel
+#    module add netcdf/3.6.1/icc
+#   e.g.(sh,bash): 
+#    export MPI_INC_DIR='/usr/local/pkg/mpich/mpich-intel/include/'
+#   (and run using mpirun -machinefile my_list_of_nodes)
+#   
+#-- using intel v9.0 and mpich2-intel, in this order:
+#    module add ifc/9.0.021 icc/9.0.021 intel/9.0
+#    module add mpich2/1.0.3/intel
+#    module add netcdf/3.6.1/icc
+#    module add mpiexec             (<-- to run with mpiexec)
+#   e.g.(sh,bash): 
+#    export MPI_INC_DIR='/usr/local/pkg/mpich2/mpich2-1.0.3/intel/include/'
+#   (and run using mpiexec)
 #
-#  will add /usr/local/pkg/ifc/ifc-8.0.034/lib to LD_LIBRARY_PATH
-#  which is needed by any executables generated with this optfile.
-#  The PBS -V option for exporting environment variables <== does not work
-#  may also be helpful if you encounter missing-library problems.
-#  Or, alternatively, one may choose to link with one of the intel
-#  "-static" or "-i-static" flags.
+#-- Multi-Threading with OpenMP:
+#  -several problems with earlier version of ifort (including some version 8)
+#  -with version 9 and more recent one:
+#   1) compile with genmake2 -omp option ;
+#   2) needs to set environment variable OMP_NUM_THREADS, and generally, 
+#      needs also to increase the thread stack-size:
+#     (sh, bash) > export OMP_NUM_THREADS=2  
+#                > export KMP_STACKSIZE=400m
+#     (csh,tcsh) > setenv OMP_NUM_THREADS 2
+#                > setenv KMP_STACKSIZE 400m
+#  NOTE: set KMP_STACKSIZE in .bashrc/.profile/.cshrc/.tcshrc is the easiest 
+#  way (I found) to set it for all proc; not an issue (?) for OMP_NUM_THREADS
 
+# Notes: the PBS -V option for exporting environment variables does not work
+#        => need to set all env var on compute nodes
 
 FC='mpif77'
 CC='mpicc'
 F90C='mpif90 -fixed -c '
 LINK='mpif77'
-INCLUDES='-I/usr/local/pkg/ifc/ifc-8.1.018/include -I/usr/local/pkg/mpich/mpich-intel/include/'
-LIBS='-L/usr/local/pkg/ifc/ifc-8.1.018/lib'
+
+#-- for NetCDF: 
+INCLUDES="-I$NETCDF_INCDIR"
+INCLUDEDIRS=$NETCDF_INCDIR
+LIBS="-L$NETCDF_LIBDIR"
+
+#- for MPI:
+INCLUDES="$INCLUDES -I$MPI_INC_DIR"
+INCLUDEDIRS="$INCLUDEDIRS $MPI_INC_DIR"
+MPIINCLUDEDIR="$MPI_INC_DIR"
+MPI_HEADER_FILES='mpif.h mpiof.h'
+MPI_HEADER_FILES_INC='./mpi_headers/mpif.h ./mpi_headers/mpiof.h'
 
 DEFINES='-DALLOW_USE_MPI -DALWAYS_USE_MPI -DWORDLENGTH=4'
 CPP='cpp  -traditional -P'
 EXTENDED_SRC_FLAG='-132'
+OMPFLAG='-openmp'
 
 NOOPTFLAGS='-O0'
 NOOPTFILES='mds_byteswapr8.F mds_byteswapr4.F mds_byteswapi4.F'
@@ -50,3 +85,4 @@
 
 NOOPTFILES=$NOOPTFILES' mitcplr_char2real.F mitcplr_real2char.F'
 NOOPTFILES=$NOOPTFILES' mitcplr_char2int.F mitcplr_int2char.F mds_byteswap.F'
+