--- MITgcm/tools/genmake2	2013/08/22 01:10:53	1.260
+++ MITgcm/tools/genmake2	2015/02/19 20:48:43	1.269
@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 #
-# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/tools/genmake2,v 1.260 2013/08/22 01:10:53 jmc Exp $
+# $Header: /home/ubuntu/mnt/e9_copy/MITgcm/tools/genmake2,v 1.269 2015/02/19 20:48:43 jmc Exp $
 # $Name:  $
 #
 # Makefile generator for MITgcm UV codes
@@ -19,9 +19,17 @@
     -help | --help | -h | --h
 	  Print this help message and exit.
 
+    -oad | --oad
+	  Generate a Makefile for an OpenAD built
+
+    -nocat4ad | -dog4ad | -ncad | -dad
+	  do not concatenate (cat) source code sent to TAF
+	  resulting in compilation of multiple files
+
     -adoptfile NAME | --adoptfile NAME | -adof NAME | --adof NAME
       -adoptfile=NAME | --adoptfile=NAME | -adof=NAME | --adof=NAME
 	  Use "NAME" as the adoptfile.  By default, the file at
+	  "tools/adjoint_options/adjoint_oad" (for OpenAD built) or
 	  "tools/adjoint_options/adjoint_default" will be used.
 
     -nooptfile | --nooptfile
@@ -1079,9 +1087,7 @@
 }
 
 check_lapack_libs()  {
-    if test ! "x$SKIP_LAPACK_CHECK" = x ; then
-	return
-    fi
+    if test "x$CHECK_FOR_LAPACK" = xf ; then return ; fi
     echo >> $LOGFILE
     echo "running: check_lapack_libs()" >> $LOGFILE
     cat <<EOF > genmake_tla.F
@@ -1277,8 +1283,8 @@
 DEVEL=
 HAVE_TEST_L=
 
-# comment this line out to enable lapack test
-SKIP_LAPACK_CHECK=t
+# set this to "t" to enable lapack test
+CHECK_FOR_LAPACK=f
 
 # DEFINES checked by test compilation or command-line
 HAVE_SYSTEM=
@@ -1325,7 +1331,9 @@
 FS90=
 
 AUTODIFF_PKG_USED=f
+OPENAD=
 AD_OPTFILE=
+CAT_SRC_FOR_TAF=1
 TAF=
 AD_TAF_FLAGS=
 FTL_TAF_FLAGS=
@@ -1355,7 +1363,7 @@
 gm_s8="HAVE_NETCDF HAVE_SYSTEM HAVE_FDATE HAVE_ETIME HAVE_LAPACK HAVE_FLUSH"
 
 #  The following are all related to adjoint/tangent-linear stuff
-gm_s10="AUTODIFF_PKG_USED AD_OPTFILE TAMC TAF AD_TAMC_FLAGS AD_TAF_FLAGS"
+gm_s10="AUTODIFF_PKG_USED AD_OPTFILE OPENAD TAMC TAF AD_TAMC_FLAGS AD_TAF_FLAGS"
 gm_s11="FTL_TAMC_FLAGS FTL_TAF_FLAGS SVD_TAMC_FLAGS SVD_TAF_FLAGS"
 gm_s12="TAF_EXTRA TAMC_EXTRA DIVA MPIINCLUDEDIR MPI_HEADER_FILES"
 
@@ -1407,6 +1415,10 @@
 	-optfile=* | --optfile=* | -of=* | --of=*)
 	    OPTFILE=$ac_optarg ;;
 
+	-oad | --oad)
+	    OPENAD="true" ; ALWAYS_USE_F90=1 ;;
+	-nocat4ad | -dog4ad | -ncad | -dad)
+	    CAT_SRC_FOR_TAF=0 ;;
 	-adoptfile | --adoptfile | -adof | --adof)
 	    ac_prev=AD_OPTFILE ;;
 	-adoptfile=* | --adoptfile=* | -adof=* | --adof=*)
@@ -1576,17 +1588,17 @@
 
 done
 
-TMP=./genmk_$$
+#TMP=./genmk_$$
 #- try to put temporary files in system-local /tmp dir
-#TMP=/tmp/genmk_$$
-#touch $TMP ; retVal=$?
-#if [ $retVal -eq 0 ] ; then
-#  if test ! -r $TMP ; then TMP=./genmk_$$ ; fi
-#else
-#  TMP=./genmk_$$
-#fi
-#rm -f $TMP
-##echo "  temp files: $TMP.*"
+TMP=/tmp/genmk_$$
+touch $TMP ; retVal=$?
+if [ $retVal -eq 0 ] ; then
+  if test ! -r $TMP ; then TMP=./genmk_$$ ; fi
+else
+  TMP=./genmk_$$
+fi
+rm -f $TMP
+#echo "  temp files: $TMP.*"
 
 printf "  getting local config information:  "
 if test -f $gm_local ; then
@@ -1680,10 +1692,12 @@
 
 echo "  getting AD_OPTFILE information:"
 if test "x${AD_OPTFILE}" = x ; then
-    if test "x$MITGCM_AD_OF" = x ; then
+    if test "x$MITGCM_AD_OF" != x ; then
+	AD_OPTFILE=$MITGCM_AD_OF
+    elif test "x$OPENAD" = x ; then
 	AD_OPTFILE=$ROOTDIR/tools/adjoint_options/adjoint_default
     else
-	AD_OPTFILE=$MITGCM_AD_OF
+	AD_OPTFILE=$ROOTDIR/tools/adjoint_options/adjoint_oad
     fi
 fi
 if test "x${AD_OPTFILE}" != xNONE ; then
@@ -2356,24 +2370,20 @@
     fi
 fi
 
-#  Check for package RADTRANS: if LAPACK is not available,
-#  then issue a warning that the direct radtrans solver is not available.
-if test "x$HAVE_LAPACK" != xt ; then
-    echo " $PACKAGES " | grep ' radtrans ' > /dev/null 2>&1
+#  Make sure the openad package is enabled if using OpenAD
+if test "x$OPENAD" != x ; then
+    echo " $PACKAGES " | grep ' openad ' > /dev/null 2>&1
     RETVAL=$?
-    if test "x$RETVAL" = x0 ; then
+    if test "x$RETVAL" != x0 ; then
 	cat <<EOF
-*********************************************************************
-WARNING: the "radtrans" package was enabled but tests failed to
-  compile LAPACK applications.  This means that the direct radtrans
-  solver is not available and compilation will fail if it is enabled.
-  If you want to use the direct solver, please check that:
 
-  1) LAPACK is correctly installed for this compiler and
-  2) the LIBS variable (within the "optfile") specifies the correct
-     LAPACK library to link against.
 *********************************************************************
+ERROR: when generating an adjoint with OpenAD, the openad package
+  must be enabled.  Please add it to packages.conf.
+*********************************************************************
+
 EOF
+        exit 1
     fi
 fi
 
@@ -2628,7 +2638,7 @@
     OAD_CB2M_FILES="/dev/null"
     OADTOOLS="$TOOLSDIR/OAD_support"
     echo "  looking for dontCompile file:  "
-    for i in "." $MODS ; do
+    for i in "." $MODS $OADTOOLS ; do
 	if test -r $i"/dontCompile" ; then
 	    OAD_DONT_COMPILE=$i"/dontCompile"
 	    echo "     found $OAD_DONT_COMPILE"
@@ -2636,7 +2646,7 @@
 	fi
     done
     echo "  looking for dontTransform file:  "
-    for i in "." $MODS ; do
+    for i in "." $MODS $OADTOOLS ; do
 	if test -r $i"/dontTransform" ; then
 	    OAD_DONT_TRANSFORM=$i"/dontTransform"
 	    echo "     found $OAD_DONT_TRANSFORM"
@@ -2644,7 +2654,7 @@
 	fi
     done
     echo "  looking for keepOriginal file:  "
-    for i in "." $MODS ; do
+    for i in "." $MODS $OADTOOLS ; do
 	if test -r $i"/keepOriginal" ; then
 	    OAD_KEEP_ORIGINAL=$i"/keepOriginal"
 	    echo "     found $OAD_KEEP_ORIGINAL"
@@ -2652,7 +2662,7 @@
 	fi
     done
     echo "  looking for cb2mFiles:  "
-    for i in "." $MODS ; do
+    for i in "." $MODS $OADTOOLS ; do
 	if test -r $i"/cb2mFiles" ; then
 	    OAD_CB2M_FILES=$i"/cb2mFiles"
 	    echo "     found $OAD_CB2M_FILES"
@@ -2968,10 +2978,6 @@
 
 small_f: \$(F77_PP_SRC_FILES) \$(F90_PP_SRC_FILES)
 
-output.txt: \$(EXECUTABLE)
-	@printf 'running ... '
-	@\$(EXECUTABLE) > \$@
-
 # remove most of the files that "make" generates
 clean:
 	-rm -rf *.p *.$FS90 *.mod ${RMFILES} work.{pc,pcl} *.template
@@ -3123,6 +3129,24 @@
 echo >> $MAKEFILE
 rm -f $TMP.adSrcFiles
 
+if test $CAT_SRC_FOR_TAF = 0 ; then
+cat >>$MAKEFILE <<EOF
+
+# ... Utilities ...
+remove_comments:
+	sed -i.bak -f \$(TOOLSDIR)/remove_comments_sed \$(AD_FILES)
+	@-rm -f \$(AD_FILES:.$FS=.$FS.bak)
+adj_sed:
+	sed -i.bak -f \$(TOOLSDIR)/adjoint_sed \$(AD_FILES:.$FS=_ad.$FS)
+	@-rm -f \$(AD_FILES:.$FS=_ad.$FS.bak)
+ftl_sed:
+	sed -i.bak -f \$(TOOLSDIR)/adjoint_sed \$(AD_FILES:.$FS=_tl.$FS)
+	@-rm -f \$(AD_FILES:.$FS=_tl.$FS.bak)
+adobjfiles: \$(AD_FILES:.$FS=_ad.o)
+ftlobjfiles: \$(AD_FILES:.$FS=_tl.o)
+EOF
+fi
+
 cat >>$MAKEFILE <<EOF
 
 # ... AD ...
@@ -3145,13 +3169,18 @@
 	@\$(MAKE) -f \$(MAKEFILE) \$(FLOWFILES)
 	cat \$(FLOWFILES) \$(AD_FILES) | sed -f \$(TOOLSDIR)/remove_comments_sed > ad_input_code.$FS
 
-ad_taf_output.$FS: ad_input_code.$FS
-	@-rm -f ad_input_code_ad.$FS ; echo ''
+adtafonly:
 	\$(TAF) \$(AD_TAF_FLAGS) \$(TAF_EXTRA) ad_input_code.$FS
 	ls -l ad_input_code_ad.$FS
 	cat ad_input_code_ad.$FS | sed -f \$(TOOLSDIR)/adjoint_sed > ad_taf_output.$FS
 
-adtafonly:
+EOF
+
+if test $CAT_SRC_FOR_TAF = 1 ; then
+cat >>$MAKEFILE <<EOF
+# ... send 1 file to TAF ...
+ad_taf_output.$FS: ad_input_code.$FS
+	@-rm -f ad_input_code_ad.$FS ; echo ''
 	\$(TAF) \$(AD_TAF_FLAGS) \$(TAF_EXTRA) ad_input_code.$FS
 	ls -l ad_input_code_ad.$FS
 	cat ad_input_code_ad.$FS | sed -f \$(TOOLSDIR)/adjoint_sed > ad_taf_output.$FS
@@ -3159,20 +3188,47 @@
 \$(EXE_AD): \$(SPECIAL_FILES) \$(H_SRC_FILES) ad_taf_output.o \$(NON_AD_F77_SRC_FILES:.F=.o) \$(F90_SRC_FILES:.F90=.o) \$(C_SRC_FILES:.c=.o) \$(EMBEDDED_FILES)
 	\$(LINK) -o \${EXE_AD} \$(FFLAGS) \$(FOPTIM) ad_taf_output.o \$(NON_AD_F77_SRC_FILES:.F=.o) \$(F90_SRC_FILES:.F90=.o) \$(C_SRC_FILES:.c=.o) \$(LIBS)
 
-ad_tamc_output.$FS: ad_input_code.$FS
-	\$(TAMC) \$(AD_TAMC_FLAGS) \$(TAMC_EXTRA) ad_input_code.$FS
-	cat ad_input_code_ad.$FS | sed -f \$(TOOLSDIR)/adjoint_sed > ad_tamc_output.$FS
+adobj: ad_taf_output.o \$(NON_AD_F77_SRC_FILES:.F=.o) \$(F90_SRC_FILES:.F90=.o) \$(C_SRC_FILES:.c=.o)
 
-ad_tamc: ad_tamc_output.o \$(OBJFILES)
-	\$(LINK) -o ${EXE_AD} \$(FFLAGS) \$(FOPTIM) \$(OBJFILES) ad_tamc_output.o \$(LIBS)
+EOF
 
+else
+cat >>$MAKEFILE <<EOF
+# ... send multiple files to TAF ...
+ad_taf_output.$FS: \$(AD_FLOW_FILES) \$(AD_FILES)
+	@$BASH\$(TOOLSDIR)/convert_cpp_cmd2defines "Adjoint version" -bAD_CONFIG_H -DALLOW_ADJOINT_RUN -UALLOW_TANGENTLINEAR_RUN > ad_config.template
+	cmp ad_config.template AD_CONFIG.h || cat ad_config.template > AD_CONFIG.h
+	@-rm -f ad_config.template
+	@\$(MAKE) -f \$(MAKEFILE) \$(F77_PP_SRC_FILES)
+	@\$(MAKE) -f \$(MAKEFILE) \$(FLOWFILES)
+	@-rm -f \$(AD_FILES:.$FS=_ad.$FS) \$(AD_FILES:.$FS=_ad.o); echo ''
+	\$(MAKE) -f \$(MAKEFILE) remove_comments
+	\$(TAF) \$(AD_TAF_FLAGS) \$(TAF_EXTRA) \$(FLOWFILES) \$(AD_FILES)
+	\$(MAKE) -f \$(MAKEFILE) adj_sed
+	\$(MAKE) -f \$(MAKEFILE) adobjfiles
+	cat \$(AD_FILES:.$FS=_ad.$FS) > ad_taf_output.$FS
+
+\$(EXE_AD): \$(SPECIAL_FILES) \$(H_SRC_FILES) ad_taf_output.$FS \$(NON_AD_F77_SRC_FILES:.F=.o) \$(F90_SRC_FILES:.F90=.o) \$(C_SRC_FILES:.c=.o) \$(EMBEDDED_FILES)
+	\$(LINK) -o \${EXE_AD} \$(FFLAGS) \$(FOPTIM) \$(AD_FILES:.$FS=_ad.o) \$(NON_AD_F77_SRC_FILES:.F=.o) \$(F90_SRC_FILES:.F90=.o) \$(C_SRC_FILES:.c=.o) \$(LIBS)
+
+adobj: \$(AD_FILES:.$FS=_ad.o) \$(NON_AD_F77_SRC_FILES:.F=.o) \$(F90_SRC_FILES:.F90=.o) \$(C_SRC_FILES:.c=.o)
+
+EOF
+fi
+
+cat >>$MAKEFILE <<EOF
 adonlyfwd:
 	patch < \$(TOOLSDIR)/ad_taf_output.f.onlyfwd.diff
 
 adtrick:
 	patch < \$(TOOLSDIR)/ad_taf_output.f.adtrick.diff
 
-adobj: ad_taf_output.o \$(NON_AD_F77_SRC_FILES:.F=.o) \$(F90_SRC_FILES:.F90=.o) \$(C_SRC_FILES:.c=.o)
+ad_tamc_output.$FS: ad_input_code.$FS
+	\$(TAMC) \$(AD_TAMC_FLAGS) \$(TAMC_EXTRA) ad_input_code.$FS
+	cat ad_input_code_ad.$FS | sed -f \$(TOOLSDIR)/adjoint_sed > ad_tamc_output.$FS
+
+ad_tamc: ad_tamc_output.o \$(OBJFILES)
+	\$(LINK) -o ${EXE_AD} \$(FFLAGS) \$(FOPTIM) \$(OBJFILES) ad_tamc_output.o \$(LIBS)
 
 # ... FTL ...
 ftlall: ftl_exe_target
@@ -3194,13 +3250,18 @@
 	@\$(MAKE) -f \$(MAKEFILE) \$(AD_FLOW_FILES)
 	cat \$(AD_FLOW_FILES) \$(AD_FILES) | sed -f \$(TOOLSDIR)/remove_comments_sed > ftl_input_code.$FS
 
-ftl_taf_output.$FS: ftl_input_code.$FS
-	@-rm -f ftl_input_code_tl.$FS ; echo ''
+ftltafonly:
 	\$(TAF) \$(FTL_TAF_FLAGS) \$(TAF_EXTRA) ftl_input_code.$FS
 	ls -l ftl_input_code_tl.$FS
 	cat ftl_input_code_tl.$FS | sed -f \$(TOOLSDIR)/adjoint_sed > ftl_taf_output.$FS
 
-ftltafonly:
+EOF
+
+if test $CAT_SRC_FOR_TAF = 1 ; then
+cat >>$MAKEFILE <<EOF
+# ... send 1 file to TAF ...
+ftl_taf_output.$FS: ftl_input_code.$FS
+	@-rm -f ftl_input_code_tl.$FS ; echo ''
 	\$(TAF) \$(FTL_TAF_FLAGS) \$(TAF_EXTRA) ftl_input_code.$FS
 	ls -l ftl_input_code_tl.$FS
 	cat ftl_input_code_tl.$FS | sed -f \$(TOOLSDIR)/adjoint_sed > ftl_taf_output.$FS
@@ -3208,6 +3269,31 @@
 \$(EXE_FTL): \$(SPECIAL_FILES) \$(H_SRC_FILES) ftl_taf_output.o \$(NON_AD_F77_SRC_FILES:.F=.o) \$(F90_SRC_FILES:.F90=.o) \$(C_SRC_FILES:.c=.o) \$(EMBEDDED_FILES)
 	\$(LINK) -o \${EXE_FTL} \$(FFLAGS) \$(FOPTIM) ftl_taf_output.o \$(NON_AD_F77_SRC_FILES:.F=.o) \$(F90_SRC_FILES:.F90=.o) \$(C_SRC_FILES:.c=.o) \$(LIBS)
 
+EOF
+
+else
+cat >>$MAKEFILE <<EOF
+# ... send multiple files to TAF ...
+ftl_taf_output.$FS: \$(AD_FLOW_FILES) \$(AD_FILES)
+	@$BASH\$(TOOLSDIR)/convert_cpp_cmd2defines "TangLin version" -bAD_CONFIG_H -UALLOW_ADJOINT_RUN -DALLOW_TANGENTLINEAR_RUN > ad_config.template
+	cmp ad_config.template AD_CONFIG.h || cat ad_config.template > AD_CONFIG.h
+	@-rm -f ad_config.template
+	@\$(MAKE) -f \$(MAKEFILE) \$(F77_PP_SRC_FILES)
+	@\$(MAKE) -f \$(MAKEFILE) \$(AD_FLOW_FILES)
+	@-rm -f \$(AD_FILES:.$FS=_tl.$FS) \$(AD_FILES:.$FS=_tl.o); echo ''
+	\$(MAKE) -f \$(MAKEFILE) remove_comments
+	\$(TAF) \$(FTL_TAF_FLAGS) \$(TAF_EXTRA) \$(AD_FLOW_FILES) \$(AD_FILES)
+	\$(MAKE) -f \$(MAKEFILE) ftl_sed
+	\$(MAKE) -f \$(MAKEFILE) ftlobjfiles
+	cat \$(AD_FILES:.$FS=_tl.$FS) > ftl_taf_output.$FS
+
+\$(EXE_FTL): \$(SPECIAL_FILES) \$(H_SRC_FILES) ftl_taf_output.$FS \$(NON_AD_F77_SRC_FILES:.F=.o) \$(F90_SRC_FILES:.F90=.o) \$(C_SRC_FILES:.c=.o) \$(EMBEDDED_FILES)
+	\$(LINK) -o \${EXE_FTL} \$(FFLAGS) \$(FOPTIM) \$(AD_FILES:.$FS=_tl.o) \$(NON_AD_F77_SRC_FILES:.F=.o) \$(F90_SRC_FILES:.F90=.o) \$(C_SRC_FILES:.c=.o) \$(LIBS)
+
+EOF
+fi
+
+cat >>$MAKEFILE <<EOF
 ftl_tamc_output.$FS: ftl_input_code.$FS
 	\$(TAMC) \$(FTL_TAMC_FLAGS) \$(TAMC_EXTRA) ftl_input_code.$FS
 	cat ftl_input_code_ftl.$FS | sed -f \$(TOOLSDIR)/adjoint_sed > ftl_tamc_output.$FS
@@ -3250,6 +3336,7 @@
 # ============ begin OpenAD specific section ==============
 
 cat >>$MAKEFILE <<EOF
+# ... OpenAD ...
 
 # all the source files linked from the various locations:
 ALL_LINKED_FILES= \
@@ -3303,9 +3390,16 @@
 
 cat >>$MAKEFILE <<EOF
 
-adAll: \$(EXE_AD)
+adAll: ad_exe_target
 .PHONY: adAll
 
+ad_exe_target:
+	@echo Update AD_CONFIG.h and make \$(EXE_AD)
+	@$BASH\$(TOOLSDIR)/convert_cpp_cmd2defines "Adjoint version" -bAD_CONFIG_H -DALLOW_ADJOINT_RUN -UALLOW_TANGENTLINEAR_RUN > ad_config.template
+	@cmp ad_config.template AD_CONFIG.h || cat ad_config.template > AD_CONFIG.h
+	@-rm -f ad_config.template
+	\$(MAKE) -f \$(MAKEFILE) \$(EXE_AD)
+
 CB2M_F90_PP_SRC_FILES=\$(addsuffix _mod.f$FS90, \$(CB2M_F90_SRC_NAMES))
 
 .PRECIOUS: \$(CB2M_F90_PP_SRC_FILES) \$(NON_AD_F77_SRC_FILES:.F=_cb2m.f$FS90)
@@ -3363,13 +3457,6 @@
 small_f: \$(CB2M_F90_PP_SRC_FILES)
 .PHONY: small_f
 
-ad_output.txt: \$(EXE_AD)
-	@printf 'linking data files ... '
-	\$(LN) -f ../input_ad/data* ../input_ad/eedata .
-	\$(LN) -f ../../global_ocean.90x40x15/input/*.bin .
-	@printf 'running ... '
-	@./\$(EXE_AD) > \$@
-
 ad_input_code.f$FS90:  \$(CB2M_AD_FILES)
 	cat \$^ > \$@