#!/bin/sh

# ----------------------------------------------------------------------------
#      Run VisIt regression tests from NERSC source repo on LLNL systems 
#
#  Programmer: Mark C. Miller 
#  Date:       May 16, 2007 
#
#  Modifications:
#    Mark C. Miller, Thu Aug 30 06:07:23 PDT 2007
#    Added logging of 'changed' message when script itself is changed. 
#
#    Mark C. Miller, Thu Aug 30 13:32:54 PDT 2007
#    Added testing of 'make distclean' target
#
#    Mark C. Miller, Thu Sep 13 10:20:08 PDT 2007
#    Added filtering of distclean for known special cases
#
#    Mark C. Miller, Fri Sep 14 08:53:32 PDT 2007
#    Fixed poor use of 'tr -d' in construction of 'files_to_status' input
#    to generating list of modifiers.
#
#    Mark C. Miller, Tue Nov  6 08:31:40 PST 2007
#    Fixed problem removing uncleaned files manually.
#   
#    Hank Childs, Mon Dec 17 17:05:53 PST 2007
#    Remove whole scripts from the skipList and replace them with the 
#    individually failing tests.
#
#    Hank Childs, Thu Dec 20 17:20:52 PST 2007
#    Added QueryGlobalId to the skip list.
#
#    Cyrus Harrison, Thu Dec 20 17:54:30 PST 2007
#    Removed QueryGlobalId from the skip list.
#
#    Hank Childs, Thu Dec 20 17:20:52 PST 2007
#    Added defer_expr to the skip list.
#
#    Hank Childs, Sat Feb  9 15:23:42 PST 2008
#    Rename components directory to avt.
#
#    Mark C. Miller, Tue Mar  4 14:02:51 PST 2008
#    Increased time limit to 20 minutes. Often davinci is heavily loaded.
#
#    Mark C. Miller, Thu Mar  6 10:03:17 PST 2008
#    Forgot to 'post' the py file source code for failed tests. Added it.
#
#    Mark C. Miller, Thu May 15 13:34:29 PDT 2008
#    Added --enable-buildall configuration option to all configure invokations
#
#    Mark C. Miller, Thu Aug 21 11:05:01 PDT 2008
#    Added chgcar_11 to skipList
#
#    Mark C. Miller, Thu Sep 11 10:12:53 PDT 2008
#    Added extra chmod/chgrp for top-level date dir when 'posting' results.
#
#    Mark C. Miller, Thu Oct 23 09:27:46 PDT 2008
#    Fixed problem getting uncleaned file count correctly.
#    Fixed cd to data dir during clean
#
#    Mark C. Miller, Thu Oct 23 09:50:31 PDT 2008
#    Found and fixed another issue handling output from wc correctly.
#
#    Mark C. Miller, Fri Dec 12 08:14:28 PST 2008
#    Added code to check for common case of /project/projectdir being down.
#
#    Mark C. Miller, Fri Dec 19 10:38:15 PST 2008
#    Changed directory where data is posted to /project/projectdir/visit/www
#
#    Mark C. Miller, Thu Jan 22 11:49:50 PST 2009
#    Added visus to skip list, removed tabs.
#
#    Mark C. Miller, Thu Jan 22 16:44:32 PST 2009
#    Removed some silo tests from the skip list
#
#    Mark C. Miller, Wed Feb  4 07:50:54 PST 2009
#    Added 'killed' to grep filter on summary file to produce summary email.
#
#    Mark C. Miller, Thu Jun  4 12:28:52 PDT 2009
#    Changed refs to 'tif' files to 'png'
#
#    Mark C. Miller, Thu Jun  4 13:00:00 PDT 2009
#    Temporarily disable copying only failed results to public html dir.
#    Instead, copy everything. 
#
#    Mark C. Miller, Mon Jun  8 14:09:34 PDT 2009
#    Change configure invokations to use ./configure CXXFLAGS=... instead of
#    env CXXFLAGS=... ./configure.
#
#    Mark C. Miller Mon Jun  8 21:22:04 PDT 2009
#    Removed diff tolerances due to transition of baselines to Davinci.
#
#    Mark C. Miller, Wed Jun 17 14:57:12 PDT 2009
#    Changed URL in email to specify portal.nersc.gov
#
#    Mark C. Miller, Tue Jun 23 15:14:43 PDT 2009
#    Removed some items from the skip list.
#
#    Mark C. Miller, Tue Jun 23 18:28:53 PDT 2009
#    Increased processor count for makes from 4 to 6.
#
#    Mark C. Miller, Wed Jun 24 11:27:59 PDT 2009
#    Removed setting of PYTHONPATH prior to runtest to get PIL as VisIt's
#    python on Davinci now has PIL installed.
#
#    Mark C. Miller, Mon Jul  6 22:07:07 PDT 2009
#    Modified to create symlinks for results that are identical to baseline
#    instead of making copies.
#
#    Mark C. Miller, Tue Jul  7 09:37:05 PDT 2009
#    Fix logic triggering creation of a link to a baseline.
#
#    Mark C. Miller, Wed Jul  8 13:08:15 PDT 2009
#    Fix and improve logic copying diff, current, baseline to html dir.
#
#    Tom Fogal, Thu Jul 23 16:58:54 PDT 2009
#    Changed parallel SR to IceT parallel SR.
#    Updated Hank's email.
#
#    Mark C. Miller, Tue Jul 28 11:42:04 PDT 2009
#    Added compositing test to skip list. Adding some line_scan results to
#    skip list.
#
#    Mark C. Miller, Wed Jul 29 09:26:23 PDT 2009
#    Change to post not just failed results but all results.
#
#    Tom Fogal, Wed Jul 29 13:09:57 MDT 2009
#    Added Sean's email to the list of people who always get emails.
#
#    Mark C. Miller, Thu Jul 30 08:23:14 PDT 2009
#    Made it count fact that script was changed as +1 in numChangedFiles.
#
#    Mark C. Miller, Mon Aug  3 12:51:27 PDT 2009
#    Changed order of operations so that make clean and make distclean
#    occur BEFORE svn update. That way, any dirs that might have been created
#    during a previous day's make are, in theory, removed and cannot collide
#    with what 'svn update' needs to do.
#
#    Mark C. Miller, Tue Aug  4 11:58:13 PDT 2009
#    Fixed problem cd'ing to correct dirs resulting from above changes. Added
#    option (-test <email>) to facilitate testing of this script itself.
#
#    Mark C. Miller, Tue Aug  4 15:22:44 PDT 2009
#    Making so that for a test run (e.g. -test arg given), we don't attempt
#    to post any results.
#
#    Mark C. Miller, Mon Aug 10 15:33:15 PDT 2009
#    Added option to truncate error details in checkAndHandleError() func.    
#    Made calls to checkAndHandleError that include make_log.txt truncate to
#    last 1000 lines of output to make emails smaller. Changed path of
#    rootTestDir because the testing dir structure has been moved to
#    projectdir.
#
#    Mark C. Miller, Mon Aug 10 15:43:44 PDT 2009
#    Changed dir to which 'links' to current baselines are done to the
#    main test dir hierarchy instead of one in the www dir. Also, changed
#    from symbolic to hard links.
#
#    Mark C. Miller, Mon Aug 10 19:21:12 PDT 2009
#    Added usage for '-test' argument. Made '-test' imply '-force'.
#
#    Mark C. Miller, Wed Aug 26 10:52:56 PDT 2009
#    Moved '-x' option from shell invokation in line 1 to -trace option.
#
#    Cyrus Harrison, Thu Aug 27 09:07:48 PDT 2009
#    Added '--enable-visitmodule' to the configure line, so we can detect
#    visitmodule build failures.
#
#    Mark C. Miller, Fri Aug 28 10:33:20 PDT 2009
#    Removed now obsolete code to update baseline dir.
#
#    Mark C. Miller, Mon Sep 28 13:25:00 PDT 2009
#    Replaced 'cp -dr' (which was not handling hard links as I expected)
#    with 'mv' as command to 'post' results.
#
#    Tom Fogal, Sat Oct 31 20:42:12 MDT 2009
#    Improve error message when log file does not exist.
#
#    Mark C. Miller, Mon Nov  2 17:17:59 PST 2009
#    Added partial skips for arbpoly, tuvok and specular.
#
#    Brad Whitlock, Thu Dec 17 14:51:28 PST 2009
#    I removed some flags from configure that we did not support under cmake.
#    I also removed scripting to "make distclean" and check for leftover
#    files. CMake has no distclean so this should really be changed to an
#    out of source build that gets checked out every night, built, tested,
#    and then clobbered.
#
#    Mark C. Miller, Tue Jan 26 12:28:42 PST 2010
#    Eliminated cating of stdout/stderr of mkdir command that makes 'post'
#    directory in /project/projectdirs/visit/www to see why its failing.
#
#    Mark C. Miller, Fri Feb 19 08:46:29 PST 2010
#    Made pre-update configure failure a warning instead of fatal error.
#
#    Mark C. Miller, Fri Feb 26 11:15:51 PST 2010
#    Removed logic to skip on no changes as we test a different mode
#    each time. Added logic to alternate mode tested based on day of week.
#
#    Mark C. Miller, Fri Mar  5 16:21:43 PST 2010
#    Added VERBOSE=1 to make commands
#
#    Cyrus Harrison, Mon Mar 22 10:59:26 PDT 2010
#    Temporarily only run the 'serial' case.
#
#    Mark C. MIller, Sat Mar 27 08:49:54 PDT 2010
#    Removed above, temporary fix once serial started passing.
#
#    Mark C. Miller, Wed Mar 31 14:22:12 PDT 2010
#    Set pixel diff tolerances slightly above zero for scalable modes. This
#    is to help reduce need for mode-specific baselines.
#
#    Mark C. Miller, Thu Apr  1 14:20:50 PDT 2010
#    Make sure we can actually generate core files; added ulimit call.
#
#    Mark C. Miller, Thu Apr  1 14:26:36 PDT 2010
#    Moved ulimit call to runtest instead. That way, anyone running runtest
#    will get desired behavior.
#
#    Mark C. Miller, Fri Apr  2 17:37:22 PDT 2010
#    Removed some visus tests from skip list as those tests have been
#    eliminated. Removed some csg tests from skip list as those tests have
#    been improved. Temporarily removed some whole .py files from skip list.
#
#    Mark C. Miller, Mon Apr 12 18:27:02 PDT 2010
#    Added mpich installation to PATH
#
#    Eric Brugger, Mon Nov  2 17:17:59 PST 2009
#    I added some more partial skips for tuvok.
#
#    Eric Brugger, Tue Apr 27 08:23:23 PDT 2010
#    Corrected a bug where the large baseline image would always be the non
#    mode specific version regardless of the mode the test was being run in.
#
#    Mark C. Miller, Wed May 12 09:58:44 PDT 2010
#    Added logic to check for a previous instance of this script still
#    running.
#
#    Eric Brugger, Thu May 27 17:04:35 PDT 2010
#    Removed the partial skips for arbpoly.
#
#    Jeremy Meredith, Tue Aug  9 11:36:34 EDT 2011
#    Removed Sean from list of devs always getting results.
#
#    Jeremy Meredith, Tue Aug  9 17:19:55 EDT 2011
#    Updated the subversion host/path.
#
# ----------------------------------------------------------------------------
dateTag=`date +%y%b%d`
svnHost=edison.nersc.gov
rootTestDir=/project/projectdirs/visit/test/trunk
errorLogFile=$rootTestDir/testLogFile_$dateTag
modifiersFile=$rootTestDir/nersc_repo_modifiers_since_last_pass
modifiersEmails=
subjectLeader="DAVINCI-VisIt Testing:"
rm -f $rootTestDir/testLogFile_*
errorMsgCount=0;
touch $errorLogFile

# list of users who want email every night with the log file
logrecipients="miller86@llnl.gov hankchilds@gmail.com"

eqzero() {
    if test "$1" = "0"; then
        echo 0
        return
    fi
    if test -z "$1"; then
        echo 0
        return
    fi
    echo 1
}

nezero() {
    if test "$1" != "0"; then
        echo 0
        return
    fi
    if test -n "$1"; then
        echo 0
        return
    fi
    echo 1
}

#
# Check current error status and, if set, log the given error
# checkAndHandleError args...
#     $1 <error-status-flag>
#     $2 <select warning/fatal/email behavior (other behaviors could be added)>
#     $3 <short-subject-text>
#     $4 <optional file or string containing details>
#     $5 <number of lines to truncate details to>
#
checkAndHandleError() {

    local errorStatus=$1
    local handlerBehavior=$2
    local shortSubject=$3
    local errorDetails=$4
    local truncateDetails=$5
    local timeOfError=`date '+%l:%M:%S %p'`

    # there is nothing to do if the error status flag is zero
    if test "$errorStatus" = "0"; then
        return
    fi
    errorMsgCount=`expr $errorMsgCount + 1`

    # put this error's information in the log file
    echo "********************************************************************************" >> $errorLogFile
    echo "Message #$errorMsgCount [$timeOfError]: $shortSubject" >> $errorLogFile
    if test -n "$errorDetails"; then
        echo "Details follow..." >> $errorLogFile
        if test -e $errorDetails; then
            if test -n "$truncateDetails"; then
                cat $errorDetails | tail -n $truncateDetails >> $errorLogFile 
            else
                cat $errorDetails >> $errorLogFile 
            fi
        else
            echo "file '${errorDetails}' does not exist." >> $errorLogFile
        fi
    else
        echo "No details given." >> $errorLogFile
    fi
    echo "" >> $errorLogFile
    echo "" >> $errorLogFile
    echo "" >> $errorLogFile

    # if this is just a warning, return now
    if test -n "`echo $handlerBehavior | grep warn`"; then
        return
    fi

    # we get here either because the error is fatal or we're flushing the log
    # at the end of a run
    emailSubject=$shortSubject
    if test -n "`echo $handlerBehavior | grep flush`"; then
        if test `wc -l $errorLogFile | tr -s ' ' | sed -e 's/^ //' | cut -d' ' -f1` -eq 0; then
            return
        else
            emailSubject="Message(s) from VisIt's Automatic Test Run"
        fi
    fi

    # put logfile contents in an email and send it
    rm -rf mailmsg
    cat > mailmsg << EOF
From: visit-developers@ornl.gov 
Subject: $subjectLeader $emailSubject
This email was generated from VisIt's Automatic Test Run

One or more messages were logged.

The list of users who have modified VisIt since the
test suite last succesfully PASSED or PASSED w/SKIPS
is...

EOF
    echo $modifiersEmails | tr ' ' '\n' >> mailmsg
    echo "" >> mailmsg

    cat $errorLogFile >> mailmsg

    # make sure we don't wind up including a user from logrecipients
    # and modifiers twice
    emailList="`echo $logrecipients` `echo $modifiersEmails`"
    emailList=`echo $emailList | tr ' ' '\n' | sort | uniq`

    if test -n "$testEmail"; then
        cat mailmsg | /usr/sbin/sendmail $testEmail
    else
        cat mailmsg | /usr/sbin/sendmail $emailList
    fi
    rm -f mailmsg

    if test -n "`echo $handlerBehavior | grep fatal`"; then
        arg1Len=`expr length "$errorStatus"`
        if test $arg1Len -lt 3; then
            exit $errorStatus
        else
            exit 1
        fi
    fi
}

# set up the environment
PATH=/project/projectdirs/visit/visit_3rdparty/mpich/1.2.7p1/linux-ia64_gcc-4.1.2/bin:/usr/common/usg/bin:/usr/common/mss/bin:/usr/common/pbs/bin:/global/u1/m/miller86/bin:/usr/common/mss/bin:/usr/common/usg/bin:/usr/local/bin:/usr/bin/X11:/usr/bin:/bin:/usr/lib/java/jre/bin:.
export PATH

optError=0
skipIfNoChanges=0
scriptWasChanged=0
testEmail=""
for options
do
   case $1 in
      "")
         # handle empty argument
         ;;
      -force)
         skipIfNoChanges=0;
         shift
         ;;
      -changed)
         scriptWasChanged=1;
         shift
         ;;
      -trace)
         set -x
         shift
         ;;
      -help)
         optError=1
         shift
         ;;
      -test)
         testEmail=$2
         shift 2
         skipIfNoChanges=0; # treat a test like a -force
         ;;
      -*)
         echo "Unknown option $1"
         optError=1
         shift
         ;;
   esac
done

if test $optError -eq 1; then
    echo "Usage:  $0 <options>"
    echo "Available options:"
    echo "        -help             display this help message"
    echo "        -force            force a test even if no files have changed since last run."
    echo "        -changed          Used to tell this script to issue the 'changed' log message."
    echo "        -test <email-addr>Used run this script in 'test' mode and email results to"
    echo "                          specified address. Implies -force, too."
    echo "        -trace            set -x."
    exit 1
fi

# check to see if (due to slow system) a previous instance of this script
# is still running from a previous night.
prevRun=$(ps -f -U $USER | tr '\n' '@')
prevRun=$(echo $prevRun | tr '@' '\n' | grep $0 | grep -v $$)
if test -n "$prevRun"; then
    checkAndHandleError 1 fatal "A previous instance of $0 is running"
fi

# check that /projectdir is actually there. Sometimes it is down for maint.
if test ! -d /project/projectdirs/visit; then
    checkAndHandleError 1 fatal "/project/projectdirs/visit is down"
fi

# check connectivity to nersc
ping -c 1 $svnHost 1> /dev/null 2>&1
checkAndHandleError $? fatal "ping of $svnHost failed"
pingWorks=`ping -c 5 $svnHost | grep "0% packet loss"`
checkAndHandleError "`nezero $pingWorks`" fatal "dropped packets to $svnHost"

# check if this script itself has been changed 
checkAndHandleError $scriptWasChanged warning "regressiontest_davinci was changed"

# configure visit
cd $rootTestDir/src
rm -f config.log config.status config.cache config_log.txt make_log.txt
#find . -name .svn -prune -false -o -name '*.so' -o -name '*.o' -o -name '*.d' -exec rm {} \;
echo -e "LD_LIBRARY_PATH = $LD_LIBRARY_PATH" 1> config_log.txt 2>&1
echo -e "Forcing LD_LIBRARY_PATH to /usr/local/lib" 1>> config_log.txt 2>&1
LD_LIBRARY_PATH=/usr/local/lib
export LD_LIBRARY_PATH
./configure CXXFLAGS=-g MAKE=gmake --enable-parallel --enable-buildall 1>> config_log.txt 2>&1
checkAndHandleError $? warn "Pre-Update configure failed" config_log.txt 
cd ..

# test 'make clean' acts as expected
cd data
gmake clean 1> ../make_clean.out 2>&1
cd ../src
gmake clean 1>> ../make_clean.out 2>&1
rm -f uncleaned_files_log.txt
find . -name .svn -prune -false -o -name '*.so' -o -name '*.o' -o -name '*_moc.[Ch]' -o -name 'moc_*.[Ch]' 1> uncleaned_files_log.txt
uncleanedFilesCount=`wc -l uncleaned_files_log.txt | tr -s ' ' | sed -e 's/^ //' | cut -d' ' -f1`
checkAndHandleError $uncleanedFilesCount warning "Prior to svn update, make clean left files; removing manually" uncleaned_files_log.txt
if test -n "$uncleanedFilesCount" -a $uncleanedFilesCount -ne 0; then
    cat uncleaned_files_log.txt | tr -s ' ' | cut -d' ' -f2 | xargs -n 1 rm -f
fi

# Update our current 'checkout' of visit trunk from NERSC
cd $rootTestDir
rm -f nersc_repo_update_$dateTag
rm -f nersc_repo_modifiers_$dateTag
touch nersc_repo_update_$dateTag
touch nersc_repo_modifiers_$dateTag
for dir in src data test; do
    pushd $dir 1> /dev/null 2>&1
    svn update >> $rootTestDir/nersc_repo_update_$dateTag 2> $rootTestDir/stderr.txt
    checkAndHandleError $? warning "svn update on $dir failed" $rootTestDir/stderr.txt
    rm -f stderr.txt
    cat $rootTestDir/nersc_repo_update_$dateTag | grep '^[ADUCG]' | grep -v revision | tr -s ' ' | cut -d' ' -f2 > files_to_status
    svn status -u -v >> $rootTestDir/nersc_repo_status_$dateTag 2> $rootTestDir/stderr.txt
    checkAndHandleError $? warning "svn status on $dir failed" $rootTestDir/stderr.txt
    rm -f $rootTestDir/stderr.txt
    grep -f files_to_status $rootTestDir/nersc_repo_status_$dateTag | tr -s ' ' | cut -d' ' -f4 > $rootTestDir/nersc_repo_modifiers_$dateTag
    rm -f files_to_status stderr.txt

    popd 1> /dev/null 2>&1
done

# build list of modifiers email addresses
cat $modifiersFile nersc_repo_modifiers_$dateTag | grep -v '^[0-9]*$' | sort | uniq > ${modifiersFile}_tmp
rm -f $modifiersFile
mv ${modifiersFile}_tmp $modifiersFile 
for u in `cat $modifiersFile`; do
    uemail=`./src/svn_bin/nersc_username_to_email $u`
    modifiersEmails="$modifiersEmails $uemail" 
done

# See if any files actually changed
numChangedFiles=`grep '^[ADUCG]' nersc_repo_update_$dateTag | grep -v '^At revision' | wc -l | tr -s ' ' | cut -d' ' -f2`
if test $scriptWasChanged -eq 1; then
    numChangedFiles=`expr $numChangedFiles + 1`
fi
if test $numChangedFiles -eq 0; then
    if test $skipIfNoChanges -eq 1; then
        checkAndHandleError 1 flush "No Changes Found. Skipping Tests"
        exit 0
    else
        checkAndHandleError 1 warning "Although no changes were found, testing forced"
    fi
else
    checkAndHandleError $numChangedFiles warning "$numChangedFiles files changed" nersc_repo_update_$dateTag
fi

# make sure we don't have conflicts
conflictFiles=`grep '^C' nersc_repo_update_$dateTag | tr -s ' ' | cut -d' ' -f2`
checkAndHandleError "`eqzero $conflictFiles`" fatal "conflicting files" "$conflictFiles"

# ok, now really go ahead and configure and build VisIt
cd src
./configure CXXFLAGS=-g MAKE=gmake --enable-parallel --enable-buildall 1>> config_log.txt 2>&1
checkAndHandleError $? fatal "Post-Update configure failed" config_log.txt 
gmake VERBOSE=1 -j 6 1>> make_log.txt 2>&1
checkAndHandleError $? fatal "gmake VERBOSE=1 -j 6 failed in src" make_log.txt 1000
cd ..

# build data
cd data
rm -f make_log.txt
gmake clean 1> /dev/null 2>&1
gmake VERBOSE=1 -j 6 test 1>> make_log.txt 2>&1
checkAndHandleError $? fatal "gmake VERBOSE=1 -j 6 failed in data" make_log.txt  1000
cd ..

# set the datestamp used for all test modes
curdate=`date +%Y-%m-%d-%p%I%M`
theDay=`date +%A`

modes="davinci,serial"
case $theDay in
    Monday)    modes="davinci,serial" ;; 
    Tuesday)   modes="davinci,parallel" ;;
    Wednesday) modes="davinci,scalable,parallel,icet" ;;
    Thursday)  modes="davinci,serial" ;;
    Friday)    modes="davinci,parallel" ;;
    Saturday)  modes="davinci,scalable,parallel,icet" ;;
    Sunday)    modes="davinci,scalable,parallel,icet" ;;
esac

# set list of tests/modes to skip
skipList_dueTo_clientServer="\
          davinci,scalable,parallel,icet:tests/hybrid/locus.py \
          davinci,scalable,parallel,icet:tests/rendering/saveformats.py \
          davinci,scalable,parallel,icet:tests/queries/bestfitline.py"
# Temporarily overrid above (MCM - 02Apr10)
skipList_dueTo_clientServer=""

skipList_dueTo_parallelRendering="\
          davinci,scalable,parallel,icet:tests/databases/boxlib.py"

skipList_dueTo_temporaryBugs="\
          davinci,scalable,parallel,icet:tests/operators/transform.py:ops_transform05 \
          davinci,serial:tests/rendering/tuvok.py:tuvok_1,tuvok_2,tuvok_3,tuvok_4,tuvok_6 \
          davinci,parallel:tests/rendering/tuvok.py:tuvok_1,tuvok_2,tuvok_3,tuvok_4,tuvok_6 \
          davinci,scalable,parallel,icet:tests/rendering/tuvok.py:tuvok_1,tuvok_2,tuvok_3,tuvok_4,tuvok_6 \
          davinci,serial:tests/rendering/specular.py:specular_02,specular_03,specular_04,specular_05,specular_10 \
          davinci,parallel:tests/rendering/specular.py:specular_02,specular_03,specular_04,specular_05,specular_10 \
          davinci,scalable,parallel,icet:tests/rendering/specular.py:specular_02,specular_03,specular_04,specular_05,specular_10"

skipList_for_Davinci="\
          davinci,serial:tests/databases/itaps.py:itaps_14,itaps_15 \
          davinci,parallel:tests/databases/itaps.py:itaps_14,itaps_15 \
          davinci,scalable,parallel,icet:tests/databases/itaps.py:itaps_14,itaps_15 \
          davinci,parallel:tests/rendering/compositing.py \
          davinci,scalable,parallel,icet:tests/rendering/compositing.py \
          davinci,parallel:tests/queries/line_scan.py:ls_1_16,ls_1_18 \
          davinci,scalable,parallel,icet:tests/queries/line_scan.py:ls_1_16,ls_1_18"
          
skipList="$skipList_dueTo_clientServer $skipList_dueTo_parallelRendering $skipList_dueTo_temporaryBugs $skipList_for_Davinci"

#
# Make directory for where to store core files
#
rm -rf test/cores
mkdir test/cores

#
# run the test(s)
error=0
cd test

for m in $modes; do

    # If we're just testing this script, just run one simple test
    testsToRun=""
    if test -n "$testEmail"; then
        testsToRun="tests/databases/image.py"
    fi

    # set pixel diff-tolerances slightly above zero for mode containing 'scalable'
    diffTols=""
    if test -n "$(echo $modes | grep scalable)"; then
        diffTols="-pixdiff 2 -avgdiff 1"
    fi

    if test "$skipList" = ""; then
        ./runtest -q -m "$m" -l 1200 $diffTols -notrackmem $testsToRun
    else
        ./runtest -q -s "$skipList" -m "$m" -l 1200 $diffTols -notrackmem $testsToRun
    fi
    curerror=$?
    theMode=`echo $m | tr ',' '_'`
    rm -f short_summary
    touch short_summary
    echo "Failed results should be obtainable on davinci.nersc.gov" >> short_summary
    echo "http://portal.nersc.gov/project/visit/${dateTag}/${theMode}/index.html" >> short_summary
    grep 'category\|failed' summary >> short_summary
    checkAndHandleError $curerror warning "test mode $m failed" short_summary 
    failedPyFiles=`grep failed short_summary | cut -d':' -f1`

    # Determine the mode directory name in the baselines.
    parallel_found=`echo $m | grep "parallel"`
    scalable_found=`echo $m | grep "scalable"`
    theBaselineModeDir="serial"
    if test "$parallel_found" != ""; then
        if test "$scalable_found" != ""; then
            theBaselineModeDir="scalable_parallel"
        else
            theBaselineModeDir="parallel"
        fi
    fi

    # Do the work to copy html results and post them only if this is NOT
    # a test of the regressiontest script itself.
    if test -z "$testEmail"; then

        mkdir html_$dateTag
        cp html/index.html html_$dateTag/.

        for thePyFile in tests/*/*.py; do
            theCat=`echo $thePyFile | cut -d'/' -f2`
            thePyFileBase=`echo $thePyFile | cut -d'/' -f3 | cut -d'.' -f1`
            for baseFile in ./baseline/$theCat/$thePyFileBase/*; do
                if test -d $baseFile; then
                    continue
                fi
                baseNameTxt=`basename $baseFile .txt`
                baseNamePng=`basename $baseFile .png`
                baseName=""
                if test -f ./baseline/$theCat/$thePyFileBase/$baseNameTxt.txt; then
                    baseName=$baseNameTxt
                elif test -f ./baseline/$theCat/$thePyFileBase/$theBaselineModeDir/$baseNamePng.png; then
                    baseName=$baseNamePng
                    if test -f ./html/d_$baseNamePng.png; then
                        cp html/[cd]_${baseName}.png html_${dateTag}/. 1>/dev/null 2>&1
                    fi
                    ln /project/projectdirs/visit/test/trunk/test/baseline/$theCat/$thePyFileBase/$theBaselineModeDir/$baseNamePng.png html_${dateTag}/b_${baseName}.png
                    cp html/[cdb]_${baseName}_thumb.png html_${dateTag}/. 1>/dev/null 2>&1
                elif test -f ./baseline/$theCat/$thePyFileBase/$baseNamePng.png; then
                    baseName=$baseNamePng
                    if test -f ./html/d_$baseNamePng.png; then
                        cp html/[cd]_${baseName}.png html_${dateTag}/. 1>/dev/null 2>&1
                    fi
                    ln /project/projectdirs/visit/test/trunk/test/baseline/$theCat/$thePyFileBase/$baseNamePng.png html_${dateTag}/b_${baseName}.png
                    cp html/[cdb]_${baseName}_thumb.png html_${dateTag}/. 1>/dev/null 2>&1
                fi
                cp html/$baseName.html html_$dateTag/. 1>/dev/null 2>&1
            done
            cp html/${theCat}_${thePyFileBase}.html html_${dateTag}/. 1>/dev/null 2>&1
            cp html/${theCat}_${thePyFileBase}_py.html html_${dateTag}/. 1>/dev/null 2>&1
        done

        # 'post' the results by copying to public directory
        mkdir /project/projectdirs/visit/www/${dateTag}
        chgrp visit /project/projectdirs/visit/www/${dateTag}
        chmod g+rwX,o+rX /project/projectdirs/visit/www/${dateTag}

        #cp -dr html_$dateTag /project/projectdirs/visit/www/${dateTag}/${theMode}
        mv html_$dateTag /project/projectdirs/visit/www/${dateTag}/${theMode}
        chgrp -R visit /project/projectdirs/visit/www/${dateTag}/${theMode}
        chmod -R g+rwX,o+rX /project/projectdirs/visit/www/${dateTag}/${theMode}
        rm -rf html_$dateTag

        # update top-level index.html file
        rm -f index.html
        echo "<html><head><title>VisIt Regression Results For Davinci</title></head><body bgcolor=\"#a0a0f0\">" > index.html
        echo "<H1>Results of VisIt Regression Tests on Davinci</H1>" >> index.html
        echo "<table border>" >> index.html
        echo "<tr>" >> index.html
        echo "   <td><b>Timestamp</b></td>" >> index.html
        echo "   <td><b>Mode</b></td>" >> index.html
        echo "   <td><b>Result</b></td>" >> index.html
        echo "</tr>" >> index.html

        ls -1t /project/projectdirs/visit/www/*/*/index.html | while read testLine; do

            theMode=`echo $testLine | cut -d'/' -f7 | cut -d'_' -f2-`
            theRelDir=`echo $testLine | cut -d'/' -f6-`
            theRelDir="./${theRelDir}"
            theTimeStamp=`ls -l --time-style=+"%A_%b_%d_%I:%M%p" $testLine | tr -s ' ' | cut -d' ' -f6`

            echo "<tr>" >> index.html
            echo "<td>${theTimeStamp}</td>" >> index.html
            echo "<td><a href=\"${theRelDir}\">${theMode}</a></td>" >> index.html

            hasFailed=`grep -m 1 Failed $testLine 2>/dev/null`
            if test "$hasFailed" = ""; then
                hasFailed=`grep -m 1 Unacceptable $testLine 2>/dev/null`
            fi
            hasSkips=`grep -m 1 Skipped $testLine 2>/dev/null`
            if test "$hasFailed" = ""; then
                if test "$hasSkips" = ""; then
                    echo "<td bgcolor="#00ff00"><b>Passed</b></td>" >> index.html
                else
                    echo "<td bgcolor="#0000ff"><b>Passed w/Skips</b></td>" >> index.html
                fi
            else
                echo "<td bgcolor="#ff0000"><b>Failed</b></td>" >> index.html
            fi
            echo "</tr>" >> index.html
        done

        echo "</table>" >> index.html
        echo "</html>" >> index.html
        cp index.html /project/projectdirs/visit/www/.

    fi

    if test ! "$curerror" = "0"; then
        error=1
    else
        checkAndHandleError 1 warning "test mode $m succeeded"
    fi
done

#
# Reset last pass modifiers if everything passed
#
if test $error -eq 0; then
    rm -f $modifiersFile
    touch $modifiersFile
    if test "$skipList" = ""; then
        checkAndHandleError 1 flush "Complete Pass -- All tests Passed!!!"
    else
        checkAndHandleError 1 flush "Partial Pass - Some tests skipped"
    fi
    exit 0
else
    checkAndHandleError 1 flush "Tests failed"
    exit 1
fi
