mitgcm.org/front_content/make_summary

#! /usr/bin/env bash

# $Header: /u/gcmpack/mitgcm.org/front_content/make_summary,v 1.73 2012/09/04 20:12:20 jmc Exp $
#
#  The purpose of this script is to create HTML summaries of the
#  directories produced by the "parse_emails" script.


usage()
{
    echo
    echo "Usage:  $0 [OPTIONS]"
    echo
    echo "where possible OPTIONS are:"
    echo "  (-help|-h)           print usage"
    echo "  (-date |-d )PERIOD   run for PERIOD=\"YYYY_MM\""
    echo "                         [def=\"$PERIOD\"]"
    echo
    exit 1
}

export LC_ALL="en_US.UTF-8"
CURR_PER=`date +%Y`"_"`date +%m`
# defaults
PERIOD=$CURR_PER

#  Parse options
ac_prev=
for ac_option ; do

    # If the previous option needs an argument, assign it.
    if test -n "$ac_prev"; then
	eval "$ac_prev=\$ac_option"
	ac_prev=
	continue
    fi

    ac_optarg=`expr "x$ac_option" : 'x[^=]*=\(.*\)'`

    case $ac_option in

	-help | --help | -h | --h)
	    usage ;;

	-date | --date | -d | --d)
	    ac_prev=PERIOD ;;
	--date=* | -date=*)
	    PERIOD=$ac_optarg ;;

	*)
	    echo "Error: don't understand argument \"$ac_option\""
	    usage
	    ;;

     esac

done

#INDIR="/net/orwell/export/export-9/mitgcm-testing/results/$PERIOD"
#OUTDIR="/home/jmc/mitgcm/test_web/summary"
INDIR="/u/u0/httpd/html/testing/results/$PERIOD"
OUTDIR="/u/u0/httpd/html/testing/summary"

OUTFILE=$OUTDIR"/output_"$PERIOD".html"
res_url="http://mitgcm.org/testing/"

#  Create the links in $OUTFILE :
echo "Creating the \"latest\" file for each machine: "
the_date=`date`

sed "s/_PERIOD/$PERIOD/" summary_head > $OUTFILE
cat <<EOF >>$OUTFILE
    <table align="center" cellpadding="0" cellspacing="0" border="0" width="95%">
<tr bgcolor="#00cccc">
  <td height="0"> <b>Nickname</b> </td>
  <td> <b>OPTFILE Name</b> </td>
  <td> <b>Type</b> </td>
  <td> <b>Date</b> </td>
  <td> <b>Summary</b> </td>
  <td> <b>Ratio</b> </td>
</tr>

EOF

color="#bbffdd"
ncolor="#bbddff"

MACHINES="faulks meander aces- acesgrid baudelaire dickens danton beagle harbor"
MACHINES="$MACHINES pleiades iblade rays solasrv sx8"
MACHINES="$MACHINES trane dodongo dokdo stomp bigred"

( cd $INDIR ; ls -1 -t */summary.txt | sed 's/\/summary.txt//' ) > ./dir_all

MALL=`cat ./dir_all | sed -e 's|_| |g' | awk '{print $2}' | sort | uniq`
for madd in $MALL ; do
    present=0
    for m in $MACHINES ; do
	echo $madd | grep $m > /dev/null 2>&1
	RETVAL=$?
	test $RETVAL = 0  &&  present=1
	continue
    done
    test $present = 0  &&  MACHINES="$MACHINES $madd"
done
#MACHINES="baudelaire"

for mname in $MACHINES ; do

    echo "  $mname"
    sname=`echo $mname | sed 's/-$//'`

    dir_list=`grep $mname ./dir_all`
    echo -n "" > ./mlist

    for i in $dir_list ; do

	dir=$INDIR"/"$i
	OPTFILE=
	if test -r $dir/summary.txt ; then
	    comm=`grep 'OPTFILE=' $dir/summary.txt`
	    eval $comm
	    OPTFILE=${OPTFILE##*/}
	fi
	if test "x$OPTFILE" = x -a -r "$dir/genmake_state" ; then
	    comm=`grep 'OPTFILE=' $dir/genmake_state 2>/dev/null`
	    eval $comm
	    OPTFILE=${OPTFILE##*/}
	fi
	if test "x$OPTFILE" = x ; then
	    comm=`grep '^# OPTFILE=' $dir/*/Makefile* 2>/dev/null | head -1`
	    comm=${comm##*#}
	    eval $comm
	    OPTFILE=${OPTFILE##*/}
	fi
	if test "x$OPTFILE" = x ; then
	    OPTFILE="not_explicitly_specified"
	fi

	# EXTRA = non-standard list of experiment
	ADJOINT=
	TANGLIN=
	RESTART=0
	EXTRA=0
	FAST=0
	DVLP=0
	MPI=0
	MTH=0
	UR4=0
	if test -r $dir/summary.txt ; then
	    comm=`grep 'ADJOINT=true' $dir/summary.txt 2>/dev/null`
	    eval $comm
	    comm=`grep 'TANGLIN=true' $dir/summary.txt 2>/dev/null`
	    eval $comm
	    RESTART=`grep -c 'test 2+2=4 summary' $dir/summary.txt`
	    comm=`grep '^run: .*testreport.* ' $dir/summary.txt`
	    EXTRA=`echo "$comm" | grep -c " -*-tdir\>"`
	    if test "x$EXTRA" = x0 ; then
	       EXTRA=`echo "$comm" | grep -c " -*-t\>"`
	    fi
	    FAST=`echo "$comm" | grep -c " -*-fast\>"`
	    if test "x$FAST" = x0 ; then
	       FAST=`echo "$comm" | grep -c " '*-noieee'*"`
	    fi
	    DVLP=`echo "$comm" | grep -c " -*-devel\>"`
	    MPI=`echo "$comm" | grep -c " -*-mpi\>"`
	    if test "x$MPI" = x0 ; then
	       MPI=`echo "$comm" | grep -c " -*-MPI\>"`
	    fi
	    MTH=`echo "$comm" | grep -c " -*-mth\>"`
	    UR4=`echo "$comm" | grep -c " -*-use_r4\>"`
	    if test "x$UR4" = x0 ; then
	       UR4=`echo "$comm" | grep -c " -*-ur4\>"`
	    fi
	fi
	if test "x$ADJOINT" = xtrue ; then
	    kind="adjoint" ; order=0
	elif test "x$TANGLIN" = xtrue ; then
	    kind="tanglin" ; order=1
	elif test "x$RESTART" = x0 ; then
	    kind="forward" ; order=2
	else
	    kind="restart" ; order=3
	fi
	if test "x$EXTRA" != x0 ; then
	    order=`expr $order + 4`
	fi
	order=`printf '%3.3i' $order`
	if test "x$UR4" = x1 ; then
	    OPTFILE="${OPTFILE}.use_r4"
	fi
	if test "x$MPI" = x1 ; then
	    yy=`echo $OPTFILE | grep -c '+mpi'`
	    if test $yy = 0 ; then OPTFILE="${OPTFILE}+mpi" ; fi
	fi
	if test "x$MTH" = x1 ; then
	    yy=`echo $OPTFILE | grep -c '+mth$'`
	    if test $yy = 0 ; then OPTFILE="${OPTFILE}+mth" ; fi
	fi
	if test "x$FAST" = x1 ; then
	    OPTFILE="${OPTFILE}.fast"
	fi
	if test "x$DVLP" = x1 ; then
	    OPTFILE="${OPTFILE}.dvlp"
	fi

	t_pass="--"
	t_tot="--"
	if test -r $dir/summary.txt ; then
	    grep '^[YN] [YN] [YN] [YN]' $dir/summary.txt > ./all_tests 2>/dev/null
	    t_tot=`cat ./all_tests | wc -l | sed -e 's| ||g'`
	    t_pass=`grep '^Y Y Y Y' ./all_tests | grep 'pass ' | wc -l | sed -e 's| ||g'`
	fi
	rm -f ./all_tests
	# echo "${dir##*/} : $t_pass out of $t_tot"

	tokens=`echo $i | sed -e 's|_| |g'`
	echo "" > ./ms_tmp
	for tok in $tokens ; do
	    echo $tok >> ./ms_tmp
	done
	DAY=`cat ./ms_tmp | awk '(length($1)==8 && substr($1,0,2)=="20")'`
	rm -f ./ms_tmp

	echo "$OPTFILE$order $DAY $OPTFILE $kind $i $t_pass:$t_tot" >> ./mlist

    done

    # helpful for debugging
    # cat ./mlist

    #  Do we have any data?  If so, create the latest pointer.
    num=`wc -l ./mlist | awk '{print $1}'`
    if test $num -gt 0 ; then

	# swap colors
	ctmp=$color
	color=$ncolor
	ncolor=$ctmp

	keys=`cat ./mlist | cut -d " " -f 1 | sort | uniq`

	for key in $keys ; do
	    tline=`grep "^$key " ./mlist | head -1`
	    ratio=`echo $tline | cut -d " " -f 6`
	    ldir=`echo $tline | cut -d " " -f 5`
	    kind=`echo $tline | cut -d " " -f 4`
	    optf=`echo $tline | cut -d " " -f 3`
	    DAY=`echo $tline | cut -d " " -f 2`
	    URL="results/$PERIOD/$ldir"
	    cat <<EOF >>$OUTFILE
<tr bgcolor="$color">
  <td height="0"> $sname </td>
  <td> $optf </td>
  <td> $kind </td>
  <td> <a href="$res_url$URL">$DAY</a> </td>
  <td> <a href="$res_url$URL/summary.txt"> summary.txt </a> </td>
  <td> $ratio </td>
</tr>
EOF
	done
    fi

done

cat >> $OUTFILE << EOF
<tr bgcolor="#00cccc">
  <td height="0" colspan="6" align="center" >This table generated on: $the_date</td>
</tr>

    </table>

<p>Examples of the scripts used for these testing runs can be obtained from: <a
href="http://mitgcm.org/viewvc/MITgcm/MITgcm/tools/example_scripts/">
MITgcm/tools/example_scripts</a>.</p>


  </body>
</html>

EOF

rm -f ./dir_all ./mlist

#- put the file in place
chgrp gcmpack $OUTFILE
chmod 664 $OUTFILE
LATEST=$OUTDIR"/latest_"$PERIOD".html"
mv -f $OUTFILE $LATEST

if test "x$PERIOD" = "x$CURR_PER" ; then
    cp $LATEST ./testing.xml
    (
	cd $OUTDIR
	rm -f latest.html
	ln -s $LATEST latest.html
    )
fi
1	edhill	1.1	#! /usr/bin/env bash
2
3	jmc	1.74	# $Header: /u/gcmpack/mitgcm.org/front_content/make_summary,v 1.73 2012/09/04 20:12:20 jmc Exp $
4	edhill	1.1	#
5			# The purpose of this script is to create HTML summaries of the
6			# directories produced by the "parse_emails" script.
7
8
9			usage()
10			{
11			echo
12			echo "Usage: $0 [OPTIONS]"
13	jmc	1.67	echo
14	edhill	1.1	echo "where possible OPTIONS are:"
15			echo " (-help\|-h) print usage"
16			echo " (-date \|-d )PERIOD run for PERIOD=\"YYYY_MM\""
17			echo " [def=\"$PERIOD\"]"
18	jmc	1.67	echo
19	edhill	1.1	exit 1
20			}
21
22	jmc	1.61	export LC_ALL="en_US.UTF-8"
23	jmc	1.36	CURR_PER=`date +%Y`"_"`date +%m`
24	edhill	1.1	# defaults
25	jmc	1.36	PERIOD=$CURR_PER
26	edhill	1.1
27			# Parse options
28			ac_prev=
29			for ac_option ; do
30	jmc	1.35
31	edhill	1.1	# If the previous option needs an argument, assign it.
32			if test -n "$ac_prev"; then
33	jmc	1.74	eval "$ac_prev=\$ac_option"
34			ac_prev=
35			continue
36	edhill	1.1	fi
37	jmc	1.35
38	edhill	1.1	ac_optarg=`expr "x$ac_option" : 'x[^=]=\(.\)'`
39	jmc	1.35
40	edhill	1.1	case $ac_option in
41	jmc	1.71
42	edhill	1.1	-help \| --help \| -h \| --h)
43			usage ;;
44	jmc	1.35
45	jmc	1.71	-date \| --date \| -d \| --d)
46			ac_prev=PERIOD ;;
47			--date=* \| -date=*)
48			PERIOD=$ac_optarg ;;
49
50			*)
51	edhill	1.1	echo "Error: don't understand argument \"$ac_option\""
52			usage
53	jmc	1.71	;;
54	jmc	1.35
55	edhill	1.1	esac
56	jmc	1.35
57	edhill	1.1	done
58
59	jmc	1.35	#INDIR="/net/orwell/export/export-9/mitgcm-testing/results/$PERIOD"
60	jmc	1.44	#OUTDIR="/home/jmc/mitgcm/test_web/summary"
61	edhill	1.1	INDIR="/u/u0/httpd/html/testing/results/$PERIOD"
62			OUTDIR="/u/u0/httpd/html/testing/summary"
63	jmc	1.35
64	jmc	1.36	OUTFILE=$OUTDIR"/output_"$PERIOD".html"
65	jmc	1.35	res_url="http://mitgcm.org/testing/"
66	edhill	1.1
67	jmc	1.36	# Create the links in $OUTFILE :
68	edhill	1.1	echo "Creating the \"latest\" file for each machine: "
69	edhill	1.5	the_date=`date`
70	edhill	1.1
71	jmc	1.60	sed "s/_PERIOD/$PERIOD/" summary_head > $OUTFILE
72			cat <<EOF >>$OUTFILE
73	edhill	1.1	<table align="center" cellpadding="0" cellspacing="0" border="0" width="95%">
74			<tr bgcolor="#00cccc">
75	edhill	1.4	<td height="0"> <b>Nickname</b> </td>
76	edhill	1.1	<td> <b>OPTFILE Name</b> </td>
77	edhill	1.4	<td> <b>Type</b> </td>
78	edhill	1.5	<td> <b>Date</b> </td>
79	edhill	1.4	<td> <b>Summary</b> </td>
80	edhill	1.6	<td> <b>Ratio</b> </td>
81	edhill	1.1	</tr>
82
83			EOF
84
85			color="#bbffdd"
86	edhill	1.10	ncolor="#bbddff"
87	edhill	1.1
88	jmc	1.68	MACHINES="faulks meander aces- acesgrid baudelaire dickens danton beagle harbor"
89	jmc	1.64	MACHINES="$MACHINES pleiades iblade rays solasrv sx8"
90	jmc	1.65	MACHINES="$MACHINES trane dodongo dokdo stomp bigred"
91	edhill	1.1
92	jmc	1.34	( cd $INDIR ; ls -1 -t */summary.txt \| sed 's/\/summary.txt//' ) > ./dir_all
93	edhill	1.1
94	edhill	1.20	MALL=`cat ./dir_all \| sed -e 's\|_\| \|g' \| awk '{print $2}' \| sort \| uniq`
95			for madd in $MALL ; do
96			present=0
97			for m in $MACHINES ; do
98	jmc	1.71	echo $madd \| grep $m > /dev/null 2>&1
99			RETVAL=$?
100			test $RETVAL = 0 && present=1
101			continue
102	edhill	1.20	done
103			test $present = 0 && MACHINES="$MACHINES $madd"
104			done
105	jmc	1.70	#MACHINES="baudelaire"
106	edhill	1.20
107	edhill	1.1	for mname in $MACHINES ; do
108
109			echo " $mname"
110	jmc	1.68	sname=`echo $mname \| sed 's/-$//'`
111	edhill	1.1
112			dir_list=`grep $mname ./dir_all`
113			echo -n "" > ./mlist
114
115			for i in $dir_list ; do
116
117			dir=$INDIR"/"$i
118			OPTFILE=
119			if test -r $dir/summary.txt ; then
120			comm=`grep 'OPTFILE=' $dir/summary.txt`
121			eval $comm
122			OPTFILE=${OPTFILE##*/}
123			fi
124			if test "x$OPTFILE" = x -a -r "$dir/genmake_state" ; then
125	edhill	1.3	comm=`grep 'OPTFILE=' $dir/genmake_state 2>/dev/null`
126	edhill	1.1	eval $comm
127			OPTFILE=${OPTFILE##*/}
128			fi
129			if test "x$OPTFILE" = x ; then
130	edhill	1.21	comm=`grep '^# OPTFILE=' $dir//Makefile 2>/dev/null \| head -1`
131	edhill	1.3	comm=${comm##*#}
132	edhill	1.1	eval $comm
133			OPTFILE=${OPTFILE##*/}
134			fi
135			if test "x$OPTFILE" = x ; then
136			OPTFILE="not_explicitly_specified"
137			fi
138	edhill	1.4
139	jmc	1.73	# EXTRA = non-standard list of experiment
140	edhill	1.4	ADJOINT=
141	jmc	1.71	TANGLIN=
142	jmc	1.32	RESTART=0
143	jmc	1.73	EXTRA=0
144	jmc	1.66	FAST=0
145			DVLP=0
146	jmc	1.69	MPI=0
147	jmc	1.49	MTH=0
148	jmc	1.62	UR4=0
149	edhill	1.4	if test -r $dir/summary.txt ; then
150			comm=`grep 'ADJOINT=true' $dir/summary.txt 2>/dev/null`
151			eval $comm
152	jmc	1.71	comm=`grep 'TANGLIN=true' $dir/summary.txt 2>/dev/null`
153			eval $comm
154	jmc	1.32	RESTART=`grep -c 'test 2+2=4 summary' $dir/summary.txt`
155	jmc	1.73	comm=`grep '^run: .testreport. ' $dir/summary.txt`
156			EXTRA=`echo "$comm" \| grep -c " -*-tdir\>"`
157			if test "x$EXTRA" = x0 ; then
158	jmc	1.74	EXTRA=`echo "$comm" \| grep -c " -*-t\>"`
159	jmc	1.73	fi
160			FAST=`echo "$comm" \| grep -c " -*-fast\>"`
161	jmc	1.66	if test "x$FAST" = x0 ; then
162	jmc	1.74	FAST=`echo "$comm" \| grep -c " '-noieee'"`
163	jmc	1.66	fi
164	jmc	1.73	DVLP=`echo "$comm" \| grep -c " -*-devel\>"`
165			MPI=`echo "$comm" \| grep -c " -*-mpi\>"`
166	jmc	1.69	if test "x$MPI" = x0 ; then
167	jmc	1.73	MPI=`echo "$comm" \| grep -c " -*-MPI\>"`
168	jmc	1.69	fi
169	jmc	1.73	MTH=`echo "$comm" \| grep -c " -*-mth\>"`
170			UR4=`echo "$comm" \| grep -c " -*-use_r4\>"`
171	jmc	1.62	if test "x$UR4" = x0 ; then
172	jmc	1.73	UR4=`echo "$comm" \| grep -c " -*-ur4\>"`
173	jmc	1.62	fi
174	edhill	1.4	fi
175	jmc	1.71	if test "x$ADJOINT" = xtrue ; then
176	jmc	1.73	kind="adjoint" ; order=0
177	jmc	1.71	elif test "x$TANGLIN" = xtrue ; then
178	jmc	1.73	kind="tanglin" ; order=1
179	jmc	1.70	elif test "x$RESTART" = x0 ; then
180	jmc	1.73	kind="forward" ; order=2
181	edhill	1.4	else
182	jmc	1.73	kind="restart" ; order=3
183			fi
184			if test "x$EXTRA" != x0 ; then
185			order=`expr $order + 4`
186	edhill	1.4	fi
187	jmc	1.73	order=`printf '%3.3i' $order`
188	jmc	1.62	if test "x$UR4" = x1 ; then
189			OPTFILE="${OPTFILE}.use_r4"
190			fi
191	jmc	1.69	if test "x$MPI" = x1 ; then
192			yy=`echo $OPTFILE \| grep -c '+mpi'`
193			if test $yy = 0 ; then OPTFILE="${OPTFILE}+mpi" ; fi
194			fi
195	jmc	1.55	if test "x$MTH" = x1 ; then
196			yy=`echo $OPTFILE \| grep -c '+mth$'`
197			if test $yy = 0 ; then OPTFILE="${OPTFILE}+mth" ; fi
198			fi
199	jmc	1.66	if test "x$FAST" = x1 ; then
200			OPTFILE="${OPTFILE}.fast"
201			fi
202			if test "x$DVLP" = x1 ; then
203			OPTFILE="${OPTFILE}.dvlp"
204	jmc	1.42	fi
205	edhill	1.4
206	edhill	1.6	t_pass="--"
207			t_tot="--"
208			if test -r $dir/summary.txt ; then
209	jmc	1.30	grep '^[YN] [YN] [YN] [YN]' $dir/summary.txt > ./all_tests 2>/dev/null
210	edhill	1.6	t_tot=`cat ./all_tests \| wc -l \| sed -e 's\| \|\|g'`
211	jmc	1.38	t_pass=`grep '^Y Y Y Y' ./all_tests \| grep 'pass ' \| wc -l \| sed -e 's\| \|\|g'`
212	edhill	1.6	fi
213	jmc	1.30	rm -f ./all_tests
214	edhill	1.6	# echo "${dir##*/} : $t_pass out of $t_tot"
215
216	edhill	1.1	tokens=`echo $i \| sed -e 's\|_\| \|g'`
217			echo "" > ./ms_tmp
218			for tok in $tokens ; do
219			echo $tok >> ./ms_tmp
220			done
221	jmc	1.59	DAY=`cat ./ms_tmp \| awk '(length($1)==8 && substr($1,0,2)=="20")'`
222	edhill	1.1	rm -f ./ms_tmp
223	edhill	1.4
224	jmc	1.70	echo "$OPTFILE$order $DAY $OPTFILE $kind $i $t_pass:$t_tot" >> ./mlist
225	edhill	1.1
226			done
227
228			# helpful for debugging
229			# cat ./mlist
230
231			# Do we have any data? If so, create the latest pointer.
232			num=`wc -l ./mlist \| awk '{print $1}'`
233			if test $num -gt 0 ; then
234	edhill	1.10
235			# swap colors
236	jmc	1.71	ctmp=$color
237			color=$ncolor
238			ncolor=$ctmp
239	edhill	1.10
240	edhill	1.4	keys=`cat ./mlist \| cut -d " " -f 1 \| sort \| uniq`
241	edhill	1.1
242	edhill	1.4	for key in $keys ; do
243	edhill	1.17	tline=`grep "^$key " ./mlist \| head -1`
244	edhill	1.6	ratio=`echo $tline \| cut -d " " -f 6`
245	edhill	1.4	ldir=`echo $tline \| cut -d " " -f 5`
246			kind=`echo $tline \| cut -d " " -f 4`
247			optf=`echo $tline \| cut -d " " -f 3`
248			DAY=`echo $tline \| cut -d " " -f 2`
249	edhill	1.1	URL="results/$PERIOD/$ldir"
250	jmc	1.36	cat <<EOF >>$OUTFILE
251	edhill	1.1	<tr bgcolor="$color">
252	jmc	1.68	<td height="0"> $sname </td>
253	edhill	1.1	<td> $optf </td>
254	edhill	1.4	<td> $kind </td>
255			<td> <a href="$res_url$URL">$DAY</a> </td>
256			<td> <a href="$res_url$URL/summary.txt"> summary.txt </a> </td>
257	edhill	1.6	<td> $ratio </td>
258	edhill	1.1	</tr>
259			EOF
260			done
261			fi
262
263			done
264
265	jmc	1.36	cat >> $OUTFILE << EOF
266	edhill	1.5	<tr bgcolor="#00cccc">
267	edhill	1.6	<td height="0" colspan="6" align="center" >This table generated on: $the_date</td>
268			</tr>
269	edhill	1.1
270			</table>
271	edhill	1.6
272	edhill	1.26	<p>Examples of the scripts used for these testing runs can be obtained from: <a
273	jmc	1.60	href="http://mitgcm.org/viewvc/MITgcm/MITgcm/tools/example_scripts/">
274	edhill	1.26	MITgcm/tools/example_scripts</a>.</p>
275	edhill	1.6
276
277	edhill	1.1	</body>
278			</html>
279
280			EOF
281
282			rm -f ./dir_all ./mlist
283
284	jmc	1.36	#- put the file in place
285			chgrp gcmpack $OUTFILE
286			chmod 664 $OUTFILE
287			LATEST=$OUTDIR"/latest_"$PERIOD".html"
288			mv -f $OUTFILE $LATEST
289
290	edhill	1.1	if test "x$PERIOD" = "x$CURR_PER" ; then
291	edhill	1.2	cp $LATEST ./testing.xml
292	edhill	1.1	(
293			cd $OUTDIR
294			rm -f latest.html
295			ln -s $LATEST latest.html
296			)
297			fi