#!/bin/sh -x
#
# Modifications:
#   Mark C. Miller, Thu Jul 29 23:35:19 PDT 2010
#   Added option to ignore third party lib problems to cmake invokation.
#
#   Eric Brugger, Thu Mar 31 09:38:38 PDT 2011
#   Modified the script to work with the 2.2RC instead of the 2.0RC.
#
#   Eric Brugger, Tue Apr 12 10:53:02 PDT 2011
#   Removed the specular lighting tests from the skip list.
#
#   Jeremy Meredith, Tue Aug  9 17:19:55 EDT 2011
#   Updated the subversion host/path.
#
#   Eric Brugger, Thu Aug 25 11:22:39 PDT 2011
#   Modified the script to work on edge.
#
#   Eric Brugger, Thu Aug 25 14:11:16 PDT 2011
#   Modified the script to send out the e-mail only after the results
#   were posted.
#
#   Eric Brugger, Wed Jan 25 13:26:36 PST 2012
#   Modified the command that submits the batch job to run the test suite
#   to specify that it isn't using any of the Lustre file systems so that
#   the job will run even if one of the Lustre file systems is down.
#
#   Eric Brugger, Fri Jun 22 07:54:19 PDT 2012
#   I modified the script to use cmake 2.8.8. I also added several failing
#   tests to the skip list.
#
#   Mark C. Miller, Wed Jun 27 12:30:13 PDT 2012
#   Removed zipwrapper tests from skip list because the tests were updated
#   to use STL in place of VTK.
#
#   Cyrus Harrison, Wed Aug 15 12:31:00 PDT 2012
#   Update to use python test suite
#
#   Eric Brugger, Fri Oct 11 14:10:51 PDT 2013
#   Change the path to the cmake used to build visit to the latest 2.8.10.2
#   version.
#
#   Eric Brugger, Fri Oct 18 15:01:56 PDT 2013
#   I changed "make test" to "make testdata" in the data directory since the
#   name of the target changed.
#
#   Eric Brugger, Fri Aug 15 11:02:18 PDT 2014
#   I added "--parallel-launch srun" to the runtest command to specify srun
#   as the parallel launch method.
#
#   Eric Brugger, Fri Aug 15 12:54:39 PDT 2014
#   I modified the script to use cmake 2.8.12.2 to configure visit.
#
#   Eric Brugger, Wed Oct 29 08:04:01 PDT 2014
#   I modified the script to use surface.
#
#   Eric Brugger, Tue Jan 13 14:09:04 PST 2015
#   I modified the script to use the account views for submitting the batch
#   job.
#
#   Eric Brugger, Thu Feb 12 08:17:18 PST 2015
#   I modified the script to use cmake 3.0.2 to configure visit.
#
#   Eric Brugger, Fri Sep 11 14:05:18 PDT 2015
#   I added the command line option "-branch", which takes a branch name.
#   While I was at it I also modified the script to use 16 cores instead of
#   12 since it is grabbing an entire node of surface, which has 16 cores.
#
#   Eric Brugger, Wed Sep  7 09:17:29 PDT 2016
#   I updated the script to build using the newly built 2.12 libraries.
#


#
# Determine the users name and e-mail address.
#
user=`whoami`

#
# Set the user e-mail address.
#
userNersc=unknown
userEmail=unknown
case $user in
    bonnell)
        userNersc=bonnell
        userEmail=biagas2@llnl.gov
        ;;
    kbonnell)
        userNersc=bonnell
        userEmail=biagas2@llnl.gov
        ;;
    brugger)
        userNersc=brugger
        userEmail=brugger1@llnl.gov
        ;;
    cyrush)
        userNersc=cyrush
        userEmail=cyrush@llnl.gov
        ;;
    hrchilds)
        userNersc=hrchilds
        userEmail=hankchilds3@gmail.com
        ;;
    mcmiller)
        userNersc=miller86
        userEmail=miller86@llnl.gov
        ;;
    miller)
        userNersc=miller86
        userEmail=miller86@llnl.gov
        ;;
    miller86)
        userNersc=miller86
        userEmail=miller86@llnl.gov
        ;;
    harrison37)
        userNersc=cyrush
        userEmail=cyrush@llnl.gov
        ;;
    cyrush)
        userNersc=cyrush
        userEmail=cyrush@llnl.gov
        ;;
esac

if test "$userEmail" = "unknown" ; then
    echo "Unknown user name. Exiting."
    exit 1
fi

#
# Parse the command line to determine if we should run the test suite
# on the trunk or RC.
#
debug="false"
branch="trunk"
revision="latest"
serial="false"
testHost="surface"
testDir="/nfs/tmp2/"
post="true"
nerscHost="edison.nersc.gov"

for abc
do
    case $1 in
        -trunk)
            branch="trunk"
            shift
            ;;
        -branch)
            branch=$2
            shift 2
            ;;
        -serial)
            serial="true"
            shift
            ;;
        -parallel)
            serial="false"
            shift
            ;;
        -d)
            debug="true"
            shift
            ;;
        -r)
            revision=$2
            shift 2
            ;;
        -host)
            testHost=$2
            shift 2
            ;;
        -no-post)
            post="false"
            shift
            ;;
    esac
done

if test "$branch" = "trunk" ; then
    workingDir="test_trunk"
else
    workingDir="test_branch"
fi

#
# Run the test suite on $testHost.
#
rm -f $testHost
cat <<EOF > $testHost
#!/bin/sh

dateTag=\`date +%Y-%m-%d-%H:%M\`

if test "$serial" = "true" ; then
    modes="$testHost,$branch,serial"
else
    modes="$testHost,$branch,serial $testHost,$branch,parallel $testHost,$branch,scalable,parallel,icet"
fi

# Check out the source
if test "$debug" = "true" ; then
   echo "Removing any old test files at: $testDir/$user/$workingDir" | mail -s "Checkout started" $userEmail
fi

echo "Removing any old test files at: $testDir/$user/$workingDir"

# Create the test directory.
if test ! -e $testDir/$user; then
    mkdir $testDir/$user
fi
rm -rf $testDir/$user/$workingDir
mkdir $testDir/$user/$workingDir
cd $testDir/$user/$workingDir

# Check out the source
if test "$debug" = "true" ; then
   echo "Checkout of $branch started at: \`date\`" | mail -s "Checkout started" $userEmail
fi

echo "Checkout of $branch started at: \`date\`"

if test "$branch" = "trunk" ; then
    if test "$revision" = "latest" ; then
        svn co svn+ssh://$userNersc@edison.nersc.gov/project/projectdirs/visit/svn/visit/trunk/src > buildlog 2>&1
        svn co svn+ssh://$userNersc@edison.nersc.gov/project/projectdirs/visit/svn/visit/trunk/data >> buildlog 2>&1
        svn co svn+ssh://$userNersc@edison.nersc.gov/project/projectdirs/visit/svn/visit/trunk/test >> buildlog 2>&1
    else
        svn co svn+ssh://$userNersc@edison.nersc.gov/project/projectdirs/visit/svn/visit/trunk/src -r $revision > buildlog 2>&1
        svn co svn+ssh://$userNersc@edison.nersc.gov/project/projectdirs/visit/svn/visit/trunk/data -r $revision >> buildlog 2>&1
        svn co svn+ssh://$userNersc@edison.nersc.gov/project/projectdirs/visit/svn/visit/trunk/test -r $revision >> buildlog 2>&1
    fi
else
    if test "$revision" = "latest" ; then
        svn co svn+ssh://$userNersc@edison.nersc.gov/project/projectdirs/visit/svn/visit/branches/$branch/src > buildlog 2>&1
        svn co svn+ssh://$userNersc@edison.nersc.gov/project/projectdirs/visit/svn/visit/branches/$branch/data >> buildlog 2>&1
        svn co svn+ssh://$userNersc@edison.nersc.gov/project/projectdirs/visit/svn/visit/branches/$branch/test >> buildlog 2>&1
    else
        svn co svn+ssh://$userNersc@edison.nersc.gov/project/projectdirs/visit/svn/visit/branches/$branch/src -r $revision > buildlog 2>&1
        svn co svn+ssh://$userNersc@edison.nersc.gov/project/projectdirs/visit/svn/visit/branches/$branch/data -r $revision >> buildlog 2>&1
        svn co svn+ssh://$userNersc@edison.nersc.gov/project/projectdirs/visit/svn/visit/branches/$branch/test -r $revision >> buildlog 2>&1
    fi
fi
if test "$debug" = "true" ; then
    echo "Checkout of $branch completed at: \`date\`" | mail -s "Checkout completed" $userEmail
fi

echo "Checkout of $branch completed at: \`date\`"

# Build the code
cd src
export PATH=\$PATH:/usr/bin/X11
export PATH=/opt/cudatoolkit-6.0/open64/bin:/opt/cudatoolkit-6.0/bin:\$PATH
export CUDA_INCLUDES=/opt/cudatoolkit-6.0/include
export CUDA_INSTALL_PATH=/opt/cudatoolkit-6.0
export CUDA_LIBS=/opt/cudatoolkit-6.0/lib64
export CUDA_PATH=/opt/cudatoolkit-6.0
export LD_LIBRARY_PATH=/opt/cudatoolkit-6.0/lib64:/usr/lib64/nvidia
export NVCC=/opt/cudatoolkit-6.0/bin/nvcc
export OPENGLLIB=-L/opt/cudatoolkit-6.0/lib64 -lcudart -lcuda

if test "$serial" = "true" ; then
    sed -i "s/VISIT_PARALLEL ON/VISIT_PARALLEL OFF/" config-site/surface86.cmake
fi
if test "$branch" = "trunk" ; then
    export PATH=/opt/rh/devtoolset-3/root/usr/bin:\$PATH
    export GNU_INCLUDES=/opt/rh/devtoolset-3/root/usr/include/c++/4.9.2
    export GNU_LIBS=/opt/rh/devtoolset-3/root/usr/lib64:/opt/rh/devtoolset-3/root/usr/lib/gcc/x86_64-redhat-linux/4.9.2
    export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/opt/rh/devtoolset-3/root/usr/lib64:/opt/rh/devtoolset-3/root/usr/lib/gcc/x86_64-redhat-linux/4.9.2
    export _LMFILES_=/opt/modules/modulefiles/gnu/4.9.2
    /usr/workspace/wsa/visit/visit/thirdparty_shared/2.13.0/cmake/3.0.2/linux-x86_64_gcc-4.9/bin/cmake \
        -DVISIT_CONFIG_SITE=config-site/surface86-test.cmake \
        -DCMAKE_BUILD_TYPE=Release \
        -DVISIT_BUILD_ALL_PLUGINS=1 \
        -DVISIT_DATA_MANUAL_EXAMPLES:BOOL=ON \
        -DIGNORE_THIRD_PARTY_LIB_PROBLEMS:BOOL=ON \
        -DCMAKE_CXX_FLAGS_RELEASE:STRING="-O3 -DNDEBUG -Wall -pedantic -Wextra -Wno-long-long -Wno-unused-parameter" \
        -DCMAKE_C_FLAGS_RELEASE:STRING="-O3 -DNDEBUG -Wall -pedantic -Wextra -Wno-long-long -Wno-unused-parameter" \
        -DCMAKE_MAKE_PROGRAM:FILEPATH="/usr/gapps/visit/gmake/4.0/chaos_5_x86_64_ib/bin/make" \
        . >> ../buildlog 2>&1
else
    /usr/gapps/visit/thirdparty_shared/2.12.0/cmake/3.0.2/linux-x86_64_gcc-4.4/bin/cmake \
        -DVISIT_CONFIG_SITE=config-site/surface86.cmake \
        -DCMAKE_BUILD_TYPE=Release \
        -DVISIT_BUILD_ALL_PLUGINS=1 \
        -DVISIT_DATA_MANUAL_EXAMPLES:BOOL=ON \
        -DIGNORE_THIRD_PARTY_LIB_PROBLEMS:BOOL=ON \
        -DCMAKE_CXX_FLAGS_RELEASE:STRING="-O3 -DNDEBUG -Wall -pedantic -Wextra -Wno-long-long -Wno-unused-parameter" \
        -DCMAKE_C_FLAGS_RELEASE:STRING="-O3 -DNDEBUG -Wall -pedantic -Wextra -Wno-long-long -Wno-unused-parameter" \
        -DCMAKE_MAKE_PROGRAM:FILEPATH="/usr/gapps/visit/gmake/4.0/chaos_5_x86_64_ib/bin/make" \
        . >> ../buildlog 2>&1
fi
rm -rf ../make.out ../make.err
make -k -j 16 1>../make.out 2>../make.err
if test $? -ne 0; then
    echo "Source build FAILED at: \`date\`" | mail -s "Source build FAILED" $userEmail
    exit 1
fi
if test "$debug" = "true" ; then
    echo "Source build completed at: \`date\`" | mail -s "Source build completed" $userEmail
fi
cat ../make.out >> ../buildlog
cat ../make.err >> ../buildlog

echo "Source build completed at: \`date\`"

# Build the test data
cd ../data
make -j 8 testdata >> ../buildlog 2>&1
if test "$debug" = "true" ; then
    echo "Data build completed at: \`date\`" | mail -s "Data build completed" $userEmail
fi

echo "Data build completed at: \`date\`"

if test "$debug" = "true" ; then
    echo "Starting tests:\`date\`" | mail -s "Starting tests" $userEmail
fi

echo "Starting tests:\`date\`"

# Run the tests, consolidating the results in one directory
cd ../test
rm -rf \$dateTag
mkdir \$dateTag
mkdir output
for m in \$modes; do
    theMode=\`echo \$m | tr ',' '_'\`
    resDir=output/\$theMode

    if test "$debug" = "true" ; then
        echo "runtest with mode = \$m" | mail -s "runtest with mode = \$m" $userEmail
    fi

    echo "runtest with mode = \$m"


    # Run the test
    ./runtest --parallel-launch "srun" -o \$resDir -m "\$m" -n 1 >> buildlog 2>&1
    mv \$resDir/html \$dateTag/\$theMode

    # Move the results to the consolidation directory
    if test "$debug" = "true" ; then
        echo "Test \$m completed at: \`date\`" | mail -s "Tests completed" $userEmail
    fi
    echo "Test \$m completed at: \`date\`"
done

# Determine if the test suite passed or failed.
hasFailed=""
for m in \$modes; do
    theMode=\`echo \$m | tr ',' '_'\`
    if test "\$hasFailed" = ""; then
        hasFailed=\`cat "\$dateTag/\$theMode/index.html" | grep -m 1 Failed 2>/dev/null\`
    fi
    if test "\$hasFailed" = ""; then
        hasFailed=\`cat "\$dateTag/\$theMode/index.html" | grep -m 1 Unacceptable 2>/dev/null\`
    fi
    if test "\$hasFailed" = ""; then
        hasFailed=\`cat "\$dateTag/\$theMode/index.html" | grep -m 1 OS-Killed 2>/dev/null\`
    fi
done

if test "$post" = "true" ; then
    # If the test suite passed then update the revision number indicating the
    # last pass.
    cd ..
    if test "\$hasFailed" = ""; then
        svn update src/svn_bin/lastpass_$testHost
        svnRevision=\`svn info src | grep "Revision:" | cut -d' ' -f2\`
        rm -f src/svn_bin/lastpass_$testHost
        echo "\$svnRevision" > src/svn_bin/lastpass_$testHost
        svn commit -m "Update the revision number of the last test suite pass." src/svn_bin/lastpass_$testHost
    fi

    # Tar up the results
    cd test
    tar -czvf  html.tar.gz \$dateTag

    # Copy the results to nersc
    scp html.tar.gz $userNersc@$nerscHost:${branch}_html.tar.gz
    if test "$debug" = "true" ; then
        echo "Results copied to $nerscHost at: `date`" | mail -s "Results copied" $userEmail
    fi

    echo "Results copied to $nerscHost at: `date`"

    # Post the results.
    cd ..
    scp src/svn_bin/visit-copy-test-results $userNersc@$nerscHost:
    ssh $userNersc@$nerscHost "./visit-copy-test-results ${branch}_html.tar.gz"
    if test "$debug" = "true" ; then
        echo "Results posted at: `date`" | mail -s "Results posted" $userEmail
    fi

    echo "Results posted at: `date`"

    # Send out an e-mail notifying the users of the test suite status.
    if test "\$hasFailed" = ""; then
        src/svn_bin/visit-notify-test-failure -pass -host $testHost
    else
        src/svn_bin/visit-notify-test-failure -host $testHost
    fi

    # If its a sunday, purge old test results
    if test "$(date +%A)" = "Sunday" ; then
        scp src/svn_bin/purge_old_test_results.sh $userNersc@$nerscHost:
        ssh $userNersc@$nerscHost "./purge_old_test_results.sh"
    fi

    # Set the permissions so that others may access the test directory.
    cd ..
    chgrp -R visit $workingDir
    chmod -R g+rwX,o+rX $workingDir
fi

EOF

chmod 750 $testHost

# Run the test suite.
if test "$testHost" = "surface" ; then
    scp surface surface:surface_testit
    subTag=`date +%Y-%m-%d-%H:%M`
    ssh surface "msub -l nodes=1:ppn=16 -l gres=ignore -l walltime=12:00:00 -o surface_testit_$subTag.out -q pbatch -A views -z ./surface_testit"
fi
