#!/bin/sh

# ----------------------------------------------------------------------------
#      Run VisIt regression tests from NERSC source repo on LLNL systems 
#
#  Programmer: Mark C. Miller 
#  Date:       May 16, 2007 
#
#  Modifications:
#    Mark C. Miller, Thu Aug 30 06:07:23 PDT 2007
#    Added logging of 'changed' message when script itself is changed. 
#
#    Mark C. Miller, Thu Aug 30 13:32:54 PDT 2007
#    Added testing of 'make distclean' target
#
#    Mark C. Miller, Thu Sep 13 10:20:08 PDT 2007
#    Added filtering of distclean for known special cases
#
#    Mark C. Miller, Fri Sep 14 08:53:32 PDT 2007
#    Fixed poor use of 'tr -d' in construction of 'files_to_status' input
#    to generating list of modifiers.
#
#    Mark C. Miller, Tue Nov  6 08:31:40 PST 2007
#    Fixed problem removing uncleaned files manually.
#   
#    Hank Childs, Mon Dec 17 17:05:53 PST 2007
#    Remove whole scripts from the skipList and replace them with the 
#    individually failing tests.
#
#    Hank Childs, Thu Dec 20 17:20:52 PST 2007
#    Added QueryGlobalId to the skip list.
#
#    Cyrus Harrison, Thu Dec 20 17:54:30 PST 2007
#    Removed QueryGlobalId from the skip list.
#
#    Hank Childs, Thu Dec 20 17:20:52 PST 2007
#    Added defer_expr to the skip list.
#
#    Hank Childs, Sat Feb  9 15:23:42 PST 2008
#    Rename components directory to avt.
#
#    Mark C. Miller, Thu May 15 13:34:29 PDT 2008
#    Added --enable-buildall configuration option to all configure invokations
#
#    Mark Miller, Tue Jun 17 23:18:51 PDT 2008
#    Added --with-google-perftools=no configuration option
#
#    Mark C. Miller, Wed Jul 23 22:14:11 PDT 2008
#    Modified to match repo dir structure
#
#    Mark C. Miller, Sun Aug  3 21:14:22 PDT 2008
#    Fixed to work in /groups/visit dir (new location of LLNL testing)
#
#    Mark C. Miller, Fri Aug  8 09:00:27 PDT 2008
#    Fixed path used to invoke nersc_username_to_email
#
#    Mark C. Miller, Thu Aug 21 11:05:01 PDT 2008
#    Added chgcar_11 to skipList
#
#    Jeremy Meredith, Fri Aug 22 14:10:45 EDT 2008
#    Removed chgcar_11 from skipList.
#
#    Mark C. Miller, Thu Oct 23 09:50:31 PDT 2008
#    Found and fixed another issue handling output from wc correctly.
#
#    Mark C. Miller, Fri Oct 31 17:00:45 PDT 2008
#    Added some itaps tests to the skip list
#
#    Mark C. Miller, Wed Feb  4 07:50:54 PST 2009
#    Added 'killed' to grep filter on summary file to produce summary email.
#
#    Mark C. Miller, Tue Mar 31 08:53:37 PDT 2009
#    Removed itaps tests from skip lists
#
#    Mark C. Miller, Thu May  7 14:51:53 PDT 2009
#    Added some itaps tests to skip lists.
#
#    Tom Fogal, Wed May 27 17:36:36 MDT 2009
#    Not getting these mails; changed my email to see if that helps.
#
#    Mark C. Miller, Mon Jun  8 14:09:34 PDT 2009
#    Change configure invokations to use ./configure CXXFLAGS=... instead of
#    env CXXFLAGS=... ./configure.
#
#    Mark C. Miller Mon Jun  8 21:22:04 PDT 2009
#    Added diff tolerances due to transition of baselines to Davinci.
#
#    Mark C. Miller, Mon Aug  3 12:51:27 PDT 2009
#    Changed order of operations so that make clean and make distclean
#    occur BEFORE svn update. That way, any dirs that might have been created
#    during a previous day's make are, in theory, removed and cannot collide
#    with what 'svn update' needs to do. Also, I removed several non-LLNL
#    developers from logrecipients and filtered emails so that only LLNL
#    developers get them.
#
#    Mark C. Miller, Tue Aug  4 11:58:13 PDT 2009
#    Fixed problem cd'ing to correct dirs resulting from above changes.
#
#    Mark C. Miller, Tue Aug 25 08:26:49 PDT 2009
#    Added support for truncating error details. Fixed filter for sending
#    emails to LLNL developers only.
#
#    Mark C. Miller, Wed Aug 26 10:51:47 PDT 2009
#    Moved '-x' from shell invokation in line 1 to -trace option to script.
#
#    Cyrus Harrison, Thu Aug 27 09:07:48 PDT 2009
#    Added '--enable-visitmodule' to the configure line, so we can detect
#    visitmodule build failures.
#
#    Brad Whitlock, Thu Dec 17 15:18:03 PST 2009
#    I changed some things to be compatible with cmake changes. This script
#    should really do an out of source build instead of updating a persistent
#    directory.
#
#    Mark C. Miller, Fri Jan 15 22:44:32 PST 2010
#    Temporarily set to run only serial tests.
#
#    Mark C. Miller, Fri Mar  5 16:21:43 PST 2010
#    Added VERBOSE=1 to make commands
#
#    Mark C. Miller, Mon Apr 12 17:22:47 PDT 2010
#    Re-enabled parallel tests.
#
#    Mark C. Miller, Mon Apr 12 22:40:12 PDT 2010
#    Replaced mpich path with path to 1.2.7p1 mpich.
#
#    Jeremy Meredith, Tue Aug  9 17:19:55 EDT 2011
#    Updated the subversion host/path.
#
# ----------------------------------------------------------------------------
dateTag=`date +%y%b%d`
svnHost=cori.nersc.gov
rootTestDir=/groups/visit/miller/nersc_repo_test/trunk
errorLogFile=$rootTestDir/testLogFile_$dateTag
modifiersFile=$rootTestDir/nersc_repo_modifiers_since_last_pass
modifiersEmails=
subjectLeader="LLNL-VisIt Testing:"
rm -f $rootTestDir/testLogFile_*
errorMsgCount=0;
touch $errorLogFile

# list of users who want email every night with the log file
logrecipients="miller86@llnl.gov"

eqzero() {
    if test "$1" = "0"; then
        echo 0
	return
    fi
    if test -z "$1"; then
        echo 0
	return
    fi
    echo 1
}

nezero() {
    if test "$1" != "0"; then
        echo 0
	return
    fi
    if test -n "$1"; then
        echo 0
	return
    fi
    echo 1
}

#
# Check current error status and, if set, log the given error
# checkAndHandleError args...
#     $1 <error-status-flag>
#     $2 <select warning/fatal/email behavior (other behaviors could be added)>
#     $3 <short-subject-text>
#     $4 <optional file or string containing details>
#
checkAndHandleError() {

    local errorStatus=$1
    local handlerBehavior=$2
    local shortSubject=$3
    local errorDetails=$4
    local truncateDetails=$5
    local timeOfError=`date '+%l:%M:%S %p'`

    # there is nothing to do if the error status flag is zero
    if test "$errorStatus" = "0"; then
        return
    fi
    errorMsgCount=`expr $errorMsgCount + 1`

    # put this error's information in the log file
    echo "********************************************************************************" >> $errorLogFile
    echo "Message #$errorMsgCount [$timeOfError]: $shortSubject" >> $errorLogFile
    if test -n "$errorDetails"; then
        echo "Details follow..." >> $errorLogFile
        if test -e $errorDetails; then
            if test -n "$truncateDetails"; then
                cat $errorDetails | tail -n $truncateDetails >> $errorLogFile
            else
                cat $errorDetails >> $errorLogFile
            fi
        else
            echo "$errorDetails" >> $errorLogFile 
        fi
    else
        echo "No details given." >> $errorLogFile
    fi
    echo "" >> $errorLogFile
    echo "" >> $errorLogFile
    echo "" >> $errorLogFile

    # if this is just a warning, return now
    if test -n "`echo $handlerBehavior | grep warn`"; then
        return
    fi

    # we get here either because the error is fatal or we're flushing the log
    # at the end of a run
    emailSubject=$shortSubject
    if test -n "`echo $handlerBehavior | grep flush`"; then
        if test `wc -l $errorLogFile | tr -s ' ' | sed -e 's/^ //' | cut -d' ' -f1` -eq 0; then
            return
	else
	    emailSubject="Message(s) from VisIt's Automatic Test Run"
	fi
    fi

    # put logfile contents in an email and send it
    rm -rf mailmsg
    cat > mailmsg << EOF
From: visit-developers@ornl.gov 
Subject: $subjectLeader $emailSubject
This email was generated from VisIt's Automatic Test Run

One or more messages were logged.

The list of users who have modified VisIt since the
test suite last succesfully PASSED or PASSED w/SKIPS
is...

EOF
    echo $modifiersEmails | tr ' ' '\n' >> mailmsg
    echo "" >> mailmsg

    cat $errorLogFile >> mailmsg

    # Make sure we don't wind up including a user from logrecipients
    # and modifiers twice.
    # Ensure this email distribution is sent ONLY to LLNL developers.
    emailList="`echo $logrecipients` `echo $modifiersEmails | tr ' ' '\n' | grep 'llnl.gov' | tr '\n' ' '`"
    emailList=`echo $emailList | tr ' ' '\n' | sort | uniq`

    cat mailmsg | /usr/sbin/sendmail $emailList
    rm -f mailmsg

    if test -n "`echo $handlerBehavior | grep fatal`"; then
        arg1Len=`expr length "$errorStatus"`
        if test $arg1Len -lt 3; then
            exit $errorStatus
        else
            exit 1
        fi
    fi
}

# set up the environment
PATH=/usr/gapps/visit/thirdparty_shared/mpich/1.2.7p1/linux-i686_gcc-3.2/bin:/data_vobs/VisIt/clearcase_bin:/usr/atria/bin:/usr/security/bin:/sbin:/usr/sbin:/usr/bsd:/usr/local/bin:/usr/bin:/bin:/etc:/usr/bin/X11:/usr/local/X11:/usr/etc:/usr/lib:/usr/atria/bin:/usr/ccs/bin:/usr/SUNWspro/bin:.
export PATH

optError=0
skipIfNoChanges=1
scriptWasChanged=0
for options
do
   case $1 in
      "")
         # handle empty argument
         ;;
      -force)
         skipIfNoChanges=0;
	 shift
	 ;;
      -changed)
         scriptWasChanged=1;
	 shift
	 ;;
      -help)
         optError=1
         shift
         ;;
      -trace)
         set -x
         shift
         ;;
      -*)
         echo "Unknown option $1"
         optError=1
         shift
         ;;
   esac
done

if test $optError -eq 1; then
    echo "Usage:  $0 <options>"
    echo "Available options:"
    echo "        -help             display this help message"
    echo "        -force            force a test even if no files have changed since last run."
    echo "        -changed          Used to tell this script to issue the 'changed' log message."
    echo "        -trace            set -x."
    exit 1
fi

# clean up ipc resources
/home/visit/clearcase_bin/cleanipcs 1> /dev/null 2>&1

# check connectivity to nersc
ping -c 1 $svnHost 1> /dev/null 2>&1
checkAndHandleError $? fatal "ping of $svnHost failed"
pingWorks=`ping -c 5 $svnHost | grep "0% packet loss"`
checkAndHandleError "`nezero $pingWorks`" fatal "dropped packets to $svnHost"

# check if this script itself has been changed 
checkAndHandleError $scriptWasChanged warning "regressiontest_nersc was changed"

# configure visit
cd $rootTestDir/src
rm -f config.log config.status config.cache config_log.txt make_log.txt
#find . -name .svn -prune -false -o -name '*.so' -o -name '*.o' -o -name '*.d' -exec rm {} \;
echo -e "LD_LIBRARY_PATH = $LD_LIBRARY_PATH" 1> config_log.txt 2>&1
echo -e "Forcing LD_LIBRARY_PATH to /usr/local/lib" 1>> config_log.txt 2>&1
LD_LIBRARY_PATH=/usr/local/lib
export LD_LIBRARY_PATH
./configure CXXFLAGS=-g MAKE=gmake --enable-parallel --enable-buildall 1>> config_log.txt 2>&1
checkAndHandleError $? warn "Pre-update configure failed" config_log.txt 

# test 'make clean' acts as expected
rm -f ./make_clean.out
cd ../data
gmake clean 1> ../src/make_clean.out 2>&1
cd ../src
gmake clean 1>> ./make_clean.out 2>&1
rm -f uncleaned_files_log.txt
find . -name .svn -prune -false -o -name '*.so' -o -name '*.o' -o -name '*_moc.[Ch]' -o -name 'moc_*.[Ch]' 1> uncleaned_files_log.txt
uncleanedFilesCount=`wc -l uncleaned_files_log.txt | tr -s ' ' | sed -e 's/^ //' | cut -d' ' -f1`
checkAndHandleError $uncleanedFilesCount warning "prior to update, make clean left files; removing manually" uncleaned_files_log.txt
if test -n "$uncleanedFilesCount" -a $uncleanedFilesCount -ne 0; then
    cat uncleaned_files_log.txt | tr -s ' ' | cut -d' ' -f2 | xargs -n 1 rm -f
fi

# Update our current 'checkout' of visit trunk from NERSC
cd $rootTestDir
rm -f nersc_repo_update_$dateTag
rm -f nersc_repo_modifiers_$dateTag
touch nersc_repo_update_$dateTag
touch nersc_repo_modifiers_$dateTag
for dir in src data test; do
    pushd $dir 1> /dev/null 2>&1
#    svn status -u -v >> $rootTestDir/svn_status.out 2> $rootTestDir/stderr.txt
#    checkAndHandleError $? warning "svn status on $dir failed" $rootTestDir/stderr.txt
#    rm -f $rootTestDir/stderr.txt
#    cat $rootTestDir/svn_status.out | grep '^ *\*' | tr -s ' ' | cut -d' ' -f5 >> $rootTestDir/nersc_repo_modifiers_$dateTag
#    rm -rf $rootTestDir/svn_status.out
#    svn update >> $rootTestDir/nersc_repo_update_$dateTag 2> $rootTestDir/stderr.txt
#    checkAndHandleError $? warning "svn update on $dir failed" $rootTestDir/stderr.txt
#    rm -f stderr.txt

    svn update >> $rootTestDir/nersc_repo_update_$dateTag 2> $rootTestDir/stderr.txt
    checkAndHandleError $? warning "svn update on $dir failed" $rootTestDir/stderr.txt
    rm -f stderr.txt
    cat $rootTestDir/nersc_repo_update_$dateTag | grep '^[ADUCG]' | grep -v revision | tr -s ' ' | cut -d' ' -f2 > files_to_status
    svn status -u -v >> $rootTestDir/nersc_repo_status_$dateTag 2> $rootTestDir/stderr.txt
    checkAndHandleError $? warning "svn status on $dir failed" $rootTestDir/stderr.txt
    rm -f $rootTestDir/stderr.txt
    grep -f files_to_status $rootTestDir/nersc_repo_status_$dateTag | tr -s ' ' | cut -d' ' -f4 > $rootTestDir/nersc_repo_modifiers_$dateTag
    rm -f files_to_status stderr.txt

    popd 1> /dev/null 2>&1
done

# build list of modifiers email addresses
cat $modifiersFile nersc_repo_modifiers_$dateTag | grep -v '^[0-9]*$' | sort | uniq > ${modifiersFile}_tmp
rm -f $modifiersFile
mv ${modifiersFile}_tmp $modifiersFile 
for u in `cat $modifiersFile`; do
    uemail=`$rootTestDir/src/svn_bin/nersc_username_to_email $u`
    modifiersEmails="$modifiersEmails $uemail" 
done

# See if any files actually changed
numChangedFiles=`grep '^[ADUCG]' nersc_repo_update_$dateTag | grep -v '^At revision' | wc -l | tr -s ' ' | cut -d' ' -f2`
if test $numChangedFiles -eq 0; then
    if test $skipIfNoChanges -eq 1; then
        checkAndHandleError 1 flush "No Changes Found. Skipping Tests"
        exit 0
    else
        checkAndHandleError 1 warning "Although no changes were found, testing forced"
    fi
else
    checkAndHandleError $numChangedFiles warning "$numChangedFiles files changed" nersc_repo_update_$dateTag
fi

# make sure we don't have conflicts
conflictFiles=`grep '^C' nersc_repo_update_$dateTag | tr -s ' ' | cut -d' ' -f2`
checkAndHandleError "`eqzero $conflictFiles`" fatal "conflicting files" "$conflictFiles"

# ok, now really go ahead and configure and build VisIt
cd src
./configure CXXFLAGS=-g MAKE=gmake --enable-parallel --enable-buildall 1>> config_log.txt 2>&1
checkAndHandleError $? fatal "Post-update configure failed" config_log.txt 
gmake VERBOSE=1 -j 4 1>> make_log.txt 2>&1
checkAndHandleError $? fatal "gmake VERBOSE=1 -j 4 failed in src" make_log.txt 1000

# build data
cd ../data
rm -f make_log.txt
gmake clean 1> /dev/null 2>&1
gmake VERBOSE=1 -j 4 test 1>> make_log.txt 2>&1
checkAndHandleError $? fatal "gmake VERBOSE=1 -j 4 failed in data" make_log.txt  1000
cd ../src

# set the datestamp used for all test modes
curdate=`date +%Y-%m-%d-%p%I%M`
theDay=`date +%A`

# set mode to run in
modes="nersc,serial nersc,parallel nersc,scalable,parallel"

# set list of tests/modes to skip
skipList_dueTo_clientServer="\
          nersc,scalable,parallel:tests/hybrid/locus.py \
          nersc,scalable,parallel:tests/rendering/saveformats.py \
          nersc,scalable,parallel:tests/queries/bestfitline.py"

skipList_dueTo_parallelRendering="\
          nersc,scalable,parallel:tests/databases/boxlib.py"

skipList_dueTo_temporaryBugs="\
          nersc,scalable,parallel:tests/databases/defer_expr.py \
          nersc,parallel:tests/databases/defer_expr.py \
          nersc,serial:tests/databases/defer_expr.py \
          nersc,scalable,parallel:tests/databases/netcdf.py:netcdf_1_00,netcdf_1_02 \
          nersc,scalable,parallel:tests/operators/transform.py:ops_transform05 \
          nersc,serial:tests/rendering/scalable.py:scalable_01,scalable_02,scalable_03,scalable_04,scalable_05 \
          nersc,parallel:tests/rendering/scalable.py:scalable_01,scalable_02,scalable_03,scalable_04,scalable_05 \
          nersc,scalable,parallel:tests/rendering/scalable.py:scalable_01,scalable_02,scalable_03,scalable_04,scalable_05 \
          nersc,serial:tests/databases/itaps.py:itaps_14,itaps_15 \
          nersc,parallel:tests/databases/itaps.py:itaps_14,itaps_15 \
          nersc,scalable,parallel:tests/databases/itaps.py:itaps_14,itaps_15"

skipList="$skipList_dueTo_clientServer $skipList_dueTo_parallelRendering $skipList_dueTo_temporaryBugs"

#
# Make directory for where to store core files
#
rm -rf ../test/cores
mkdir ../test/cores

#
# run the test(s)
error=0
cd ../test

rm -f ~miller/.visit/linux-intel/plugins/databases/*.so

for m in $modes; do

    # clean up ipc resources
    /home/visit/clearcase_bin/cleanipcs 1> /dev/null 2>&1

    if test "$skipList" = ""; then
        ./runtest -q -p -m "$m" -d "$curdate" -notrackmem -pixdiff 5 -avgdiff 5 -numdiff 0.0001
    else
        ./runtest -q -s "$skipList" -p -m "$m" -d "$curdate" -notrackmem -pixdiff 5 -avgdiff 5 -numdiff 0.0001
    fi
    curerror=$?
    theMode=`echo $m | tr ',' '_'`
    rm -f short_summary
    touch short_summary
    echo "Failed results should be obtainable from" >> short_summary
    echo "ftp://ftp.llnl.gov/outgoing/visit_results_${theMode}_${dateTag}.tar.gz" >> short_summary
    grep 'category\|failed\|killed' summary >> short_summary
    checkAndHandleError $curerror warning "test mode $m failed" short_summary 

    # try to use auto-login, anonymous ftp to
    # put failed results on ftp.llnl.gov
    failedPyFiles=`grep failed short_summary | cut -d':' -f1`
    if test -n "$failedPyFiles"; then
        mkdir html_$dateTag
        cp html/index.html html_$dateTag/.
        for f in $failedPyFiles; do
            fileKey=`basename $f .py`
	    cp html/*$fileKey* html_$dateTag/.
        done
	tar cf - html_$dateTag | gzip > visit_results_${theMode}_${dateTag}.tar.gz
	hasOrigNetrc=0
	if test -e .netrc; then
	    hasOrigNetrc=1
	    mv .netrc .netrc.orig
	fi
	touch .netrc
	chmod 600 .netrc
	echo "machine ftp.llnl.gov login anonymous password miller86@llnl.gov" >> ./.netrc
        rm -f ftp_commands.txt 
	echo "cd outgoing" >> ftp_commands.txt
	echo "bin" >> ftp_commands.txt
	echo "put visit_results_${theMode}_${dateTag}.tar.gz" >> ftp_commands.txt
	env HOME=. ftp ftp.llnl.gov < ftp_commands.txt
        checkAndHandleError $? "ftp transfer of results failed" sftp.log
	rm -f visit_results_${theMode}_${dateTag}.tar.gz
	rm -f .netrc
	rm -f ftp_commands.txt
	if test $hasOrigNetrc -eq 1; then
	    mv .netrc.orig .netrc
	fi
        rm -f sftp_batch sftp.log
        rm -rf html_$dateTag
    fi

    if test ! "$curerror" = "0"; then
        error=1
    else
        checkAndHandleError 1 warning "test mode $m succeeded"
    fi
done

#
# Reset last pass modifiers if everything passed
#
if test $error -eq 0; then
    rm -f $modifiersFile
    touch $modifiersFile
    if test "$skipList" = ""; then
        checkAndHandleError 1 flush "Complete Pass -- All tests Passed!!!"
    else
        checkAndHandleError 1 flush "Partial Pass - Some tests skipped"
    fi
    exit 0
else
    checkAndHandleError 1 flush "Tests failed"
    exit 1
fi
