regressiontest_nersc 15.8 KB
Newer Older
1 2 3 4 5 6 7 8 9
#!/bin/sh -x

# ----------------------------------------------------------------------------
#      Run VisIt regression tests from NERSC source repo on LLNL systems 
#
#  Programmer: Mark C. Miller 
#  Date:       May 16, 2007 
#
#  Modifications:
10 11
#    Mark C. Miller, Thu Aug 30 06:07:23 PDT 2007
#    Added logging of 'changed' message when script itself is changed. 
miller86's avatar
miller86 committed
12 13 14
#
#    Mark C. Miller, Thu Aug 30 13:32:54 PDT 2007
#    Added testing of 'make distclean' target
miller86's avatar
miller86 committed
15 16 17
#
#    Mark C. Miller, Thu Sep 13 10:20:08 PDT 2007
#    Added filtering of distclean for known special cases
18 19 20 21
#
#    Mark C. Miller, Fri Sep 14 08:53:32 PDT 2007
#    Fixed poor use of 'tr -d' in construction of 'files_to_status' input
#    to generating list of modifiers.
22 23 24
#
#    Mark C. Miller, Tue Nov  6 08:31:40 PST 2007
#    Fixed problem removing uncleaned files manually.
25
#   
26 27 28 29
#    Hank Childs, Mon Dec 17 17:05:53 PST 2007
#    Remove whole scripts from the skipList and replace them with the 
#    individually failing tests.
#
30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116
# ----------------------------------------------------------------------------
dateTag=`date +%y%b%d`
svnHost=svn.nersc.gov
rootTestDir=/gscratch/miller/nersc_repo_test
errorLogFile=$rootTestDir/testLogFile_$dateTag
modifiersFile=$rootTestDir/nersc_repo_modifiers_since_last_pass
modifiersEmails=
subjectLeader="NERSC-VisIt Testing:"
rm -f $rootTestDir/testLogFile_*
errorMsgCount=0;
touch $errorLogFile

# list of users who want email every night with the log file
logrecipients="miller86@llnl.gov childs3@llnl.gov jsmeredith@ornl.gov"

eqzero() {
    if test "$1" = "0"; then
        echo 0
	return
    fi
    if test -z "$1"; then
        echo 0
	return
    fi
    echo 1
}

nezero() {
    if test "$1" != "0"; then
        echo 0
	return
    fi
    if test -n "$1"; then
        echo 0
	return
    fi
    echo 1
}

#
# Check current error status and, if set, log the given error
# checkAndHandleError args...
#     $1 <error-status-flag>
#     $2 <select warning/fatal/email behavior (other behaviors could be added)>
#     $3 <short-subject-text>
#     $4 <optional file or string containing details>
#
checkAndHandleError() {

    local errorStatus=$1
    local handlerBehavior=$2
    local shortSubject=$3
    local errorDetails=$4
    local timeOfError=`date '+%l:%M:%S %p'`

    # there is nothing to do if the error status flag is zero
    if test "$errorStatus" = "0"; then
        return
    fi
    errorMsgCount=`expr $errorMsgCount + 1`

    # put this error's information in the log file
    echo "********************************************************************************" >> $errorLogFile
    echo "Message #$errorMsgCount [$timeOfError]: $shortSubject" >> $errorLogFile
    if test -n "$errorDetails"; then
        echo "Details follow..." >> $errorLogFile
        if test -e $errorDetails; then
            cat $errorDetails >> $errorLogFile 
        else
            echo "$errorDetails" >> $errorLogFile 
        fi
    else
        echo "No details given." >> $errorLogFile
    fi
    echo "" >> $errorLogFile
    echo "" >> $errorLogFile
    echo "" >> $errorLogFile

    # if this is just a warning, return now
    if test -n "`echo $handlerBehavior | grep warn`"; then
        return
    fi

    # we get here either because the error is fatal or we're flushing the log
    # at the end of a run
    emailSubject=$shortSubject
    if test -n "`echo $handlerBehavior | grep flush`"; then
117
        if test `wc -l $errorLogFile | tr -s ' ' | cut -d' ' -f2` -eq 0; then
118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133
            return
	else
	    emailSubject="Message(s) from VisIt's Automatic Test Run"
	fi
    fi

    # put logfile contents in an email and send it
    rm -rf mailmsg
    cat > mailmsg << EOF
From: visit@llnl.gov 
Subject: $subjectLeader $emailSubject
This email was generated from VisIt's Automatic Test Run

One or more messages were logged.

The list of users who have modified VisIt since the
134 135
test suite last succesfully PASSED or PASSED w/SKIPS
is...
136 137

EOF
138 139 140
    echo $modifiersEmails | tr ' ' '\n' >> mailmsg
    echo "" >> mailmsg

141 142 143 144 145
    cat $errorLogFile >> mailmsg

    # make sure we don't wind up including a user from logrecipients
    # and modifiers twice
    emailList="`echo $logrecipients` `echo $modifiersEmails`"
146
    emailList=`echo $emailList | tr ' ' '\n' | sort | uniq`
147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164

    cat mailmsg | /usr/sbin/sendmail $emailList
    rm -f mailmsg

    if test -n "`echo $handlerBehavior | grep fatal`"; then
        arg1Len=`expr length "$errorStatus"`
        if test $arg1Len -lt 3; then
            exit $errorStatus
        else
            exit 1
        fi
    fi
}

# set up the environment
PATH=/misc/gapps/mpich/1.2.4/Linux/serial/64/debug/bin:/data_vobs/VisIt/clearcase_bin:/usr/atria/bin:/usr/security/bin:/sbin:/usr/sbin:/usr/bsd:/usr/local/bin:/usr/bin:/bin:/etc:/usr/bin/X11:/usr/local/X11:/usr/etc:/usr/lib:/usr/atria/bin:/usr/ccs/bin:/usr/SUNWspro/bin:.
export PATH

165 166
optError=0
skipIfNoChanges=1
167
scriptWasChanged=0
168 169 170 171 172 173 174 175 176 177
for options
do
   case $1 in
      "")
         # handle empty argument
         ;;
      -force)
         skipIfNoChanges=0;
	 shift
	 ;;
178 179 180 181
      -changed)
         scriptWasChanged=1;
	 shift
	 ;;
182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198
      -help)
         optError=1
         shift
         ;;
      -*)
         echo "Unknown option $1"
         optError=1
         shift
         ;;
   esac
done

if test $optError -eq 1; then
    echo "Usage:  $0 <options>"
    echo "Available options:"
    echo "        -help             display this help message"
    echo "        -force            force a test even if no files have changed since last run."
199
    echo "        -changed          Used to tell this script to issue the 'changed' log message."
200 201 202
    exit 1
fi

203 204 205 206 207 208 209 210 211
# clean up ipc resources
/home/visit/clearcase_bin/cleanipcs 1> /dev/null 2>&1

# check connectivity to nersc
ping -c 1 $svnHost 1> /dev/null 2>&1
checkAndHandleError $? fatal "ping of $svnHost failed"
pingWorks=`ping -c 5 $svnHost | grep "0% packet loss"`
checkAndHandleError "`nezero $pingWorks`" fatal "dropped packets to $svnHost"

212 213 214
# check if this script itself has been changed 
checkAndHandleError $scriptWasChanged warning "regressiontest_nersc was changed"

215 216
# Update our current 'checkout' of visit trunk from NERSC
pushd $rootTestDir 1> /dev/null 2>&1
217 218
rm -f nersc_repo_update_$dateTag
rm -f nersc_repo_modifiers_$dateTag
219 220 221 222
touch nersc_repo_update_$dateTag
touch nersc_repo_modifiers_$dateTag
for dir in src src/data src/test; do
    pushd $dir 1> /dev/null 2>&1
223 224 225 226 227 228 229 230 231
#    svn status -u -v >> $rootTestDir/svn_status.out 2> $rootTestDir/stderr.txt
#    checkAndHandleError $? warning "svn status on $dir failed" $rootTestDir/stderr.txt
#    rm -f $rootTestDir/stderr.txt
#    cat $rootTestDir/svn_status.out | grep '^ *\*' | tr -s ' ' | cut -d' ' -f5 >> $rootTestDir/nersc_repo_modifiers_$dateTag
#    rm -rf $rootTestDir/svn_status.out
#    svn update >> $rootTestDir/nersc_repo_update_$dateTag 2> $rootTestDir/stderr.txt
#    checkAndHandleError $? warning "svn update on $dir failed" $rootTestDir/stderr.txt
#    rm -f stderr.txt

232 233 234
    svn update >> $rootTestDir/nersc_repo_update_$dateTag 2> $rootTestDir/stderr.txt
    checkAndHandleError $? warning "svn update on $dir failed" $rootTestDir/stderr.txt
    rm -f stderr.txt
235
    cat $rootTestDir/nersc_repo_update_$dateTag | grep '^[ADUCG]' | grep -v revision | tr -s ' ' | cut -d' ' -f2 > files_to_status
236 237 238 239 240 241
    svn status -u -v >> $rootTestDir/nersc_repo_status_$dateTag 2> $rootTestDir/stderr.txt
    checkAndHandleError $? warning "svn status on $dir failed" $rootTestDir/stderr.txt
    rm -f $rootTestDir/stderr.txt
    grep -f files_to_status $rootTestDir/nersc_repo_status_$dateTag | tr -s ' ' | cut -d' ' -f4 > $rootTestDir/nersc_repo_modifiers_$dateTag
    rm -f files_to_status stderr.txt

242 243 244 245
    popd 1> /dev/null 2>&1
done

# build list of modifiers email addresses
246
cat $modifiersFile nersc_repo_modifiers_$dateTag | grep -v '^[0-9]*$' | sort | uniq > ${modifiersFile}_tmp
247 248 249 250 251 252 253
rm -f $modifiersFile
mv ${modifiersFile}_tmp $modifiersFile 
for u in `cat $modifiersFile`; do
    uemail=`./nersc_username_to_email $u`
    modifiersEmails="$modifiersEmails $uemail" 
done

254
# See if any files actually changed
255
numChangedFiles=`grep '^[ADUCG]' nersc_repo_update_$dateTag | grep -v '^At revision' | wc -l | tr -s ' ' | cut -d' ' -f2`
256 257 258 259 260 261 262 263 264 265 266 267 268 269 270
if test $numChangedFiles -eq 0; then
    if test $skipIfNoChanges -eq 1; then
        checkAndHandleError 1 flush "No Changes Found. Skipping Tests"
        exit 0
    else
        checkAndHandleError 1 warning "Although no changes were found, testing forced"
    fi
else
    checkAndHandleError $numChangedFiles warning "$numChangedFiles files changed" nersc_repo_update_$dateTag
fi

# make sure we don't have conflicts
conflictFiles=`grep '^C' nersc_repo_update_$dateTag | tr -s ' ' | cut -d' ' -f2`
checkAndHandleError "`eqzero $conflictFiles`" fatal "conflicting files" "$conflictFiles"

271 272 273
# build visit
cd src
rm -f config.log config.status config.cache config_log.txt make_log.txt
274
#find . -name .svn -prune -false -o -name '*.so' -o -name '*.o' -o -name '*.d' -exec rm {} \;
275 276 277 278 279
echo -e "LD_LIBRARY_PATH = $LD_LIBRARY_PATH" 1> config_log.txt 2>&1
echo -e "Forcing LD_LIBRARY_PATH to /usr/local/lib" 1>> config_log.txt 2>&1
LD_LIBRARY_PATH=/usr/local/lib
export LD_LIBRARY_PATH
env CXXFLAGS="-g" MAKE=gmake ./configure --enable-parallel 1>> config_log.txt 2>&1
280
checkAndHandleError $? fatal "FIRST configure failed" config_log.txt 
miller86's avatar
miller86 committed
281 282

# test 'make clean' acts as expected
283
cd data
miller86's avatar
miller86 committed
284
gmake clean 1> ./make_clean.out 2>&1
285
cd ..
miller86's avatar
miller86 committed
286
gmake clean 1>> ./make_clean.out 2>&1
287 288 289 290 291
rm -f uncleaned_files_log.txt
find . -name .svn -prune -false -o -name '*.so' -o -name '*.o' -o -name '*_moc.[Ch]' -o -name 'moc_*.[Ch]' 1> uncleaned_files_log.txt
uncleanedFilesCount=`wc -l uncleaned_files_log.txt | tr -s ' ' | cut -d' ' -f2`
checkAndHandleError $uncleanedFilesCount warning "make clean left files; removing manually" uncleaned_files_log.txt
if test $uncleanedFilesCount -ne 0; then
292
    cat uncleaned_files_log.txt | tr -s ' ' | cut -d' ' -f2 | xargs -n 1 rm -f
293
fi
miller86's avatar
miller86 committed
294 295

# test 'make distclean' acts as expected
miller86's avatar
miller86 committed
296 297 298 299
cd data
gmake distclean 1> ./make_distclean.out 2>&1
cd ..
make distclean 1>> ./make_distclean.out 2>&1
miller86's avatar
miller86 committed
300 301 302 303
rm -f undistcleaned_files_log.txt
find . -name .svn -prune -false -o -name '*.d' -o -name '.depend' -o -name '.pardepend' 1> undistcleaned_files_log.txt
# The 2nd find, below, finds only those Makefiles for which there is also
# a coorsponding Makefile.in. 'make distclean' should have removed these
miller86's avatar
miller86 committed
304 305 306
# It also includes some special grep filtering for special cases of some
# Makefiles that aren't easy for distclean to clean
find . -name .svn -prune -false -o \( -name 'Makefile' -exec test -e \{\}.in \; \) -print | grep -v ./components/Preprocessor/Makefile\\\|./tools/windowmaker/Makefile\\\|./tools/prep/Makefile\\\|./sim/examples/Makefile\\\|./common/siloobj/Makefile\\\|./common/siloobj_vtk_db/Makefile\\\|./cqscore/Makefile 1>> undistcleaned_files_log.txt
miller86's avatar
miller86 committed
307 308 309
undistcleanedFilesCount=`wc -l undistcleaned_files_log.txt | tr -s ' ' | cut -d' ' -f2`
checkAndHandleError $undistcleanedFilesCount warning "make distclean left files; removing manually" undistcleaned_files_log.txt
if test $undistcleanedFilesCount -ne 0; then
miller86's avatar
miller86 committed
310
    cat undistcleaned_files_log.txt | tr -s ' ' | cut -d' ' -f2 | xargs -n 1 rm -f
miller86's avatar
miller86 committed
311 312
fi

miller86's avatar
miller86 committed
313
# ok, now really go ahead and configure and build VisIt
314 315
env CXXFLAGS="-g" MAKE=gmake ./configure --enable-parallel 1>> config_log.txt 2>&1
checkAndHandleError $? fatal "SECOND configure failed" config_log.txt 
316 317 318
gmake -j 4 1>> make_log.txt 2>&1
checkAndHandleError $? fatal "gmake -j 4 failed in src" make_log.txt

319 320 321
# short term fix for repo src tree changes
sed -i -e 's|../src/exe/visitconvert_ser|../exe/visitconvert_ser|' data/Makefile
sed -i -e 's|../src/bin/visitconvert|../bin/visitconvert|' data/Makefile
322 323 324
# build data
cd data
rm -f make_log.txt
325
gmake clean 1> /dev/null 2>&1
326 327 328 329 330 331 332 333 334 335 336 337
gmake -j 4 test 1>> make_log.txt 2>&1
checkAndHandleError $? fatal "gmake -j 4 failed in data" make_log.txt 
cd ..

# set the datestamp used for all test modes
curdate=`date +%Y-%m-%d-%p%I%M`
theDay=`date +%A`

# set mode to run in
modes="nersc,serial nersc,parallel nersc,scalable,parallel"

# set list of tests/modes to skip
338
skipList_dueTo_clientServer="\
339 340
          nersc,scalable,parallel:tests/hybrid/locus.py \
          nersc,scalable,parallel:tests/rendering/saveformats.py \
341 342 343 344 345 346 347 348
          nersc,scalable,parallel:tests/queries/bestfitline.py"

skipList_dueTo_parallelRendering="\
          nersc,scalable,parallel:tests/databases/boxlib.py"

skipList_dueTo_temporaryBugs="\
	  nersc,scalable,parallel:tests/databases/netcdf.py:netcdf_1_00,netcdf_1_02 \
	  nersc,scalable,parallel:tests/operators/transform.py:ops_transform05 \
349 350 351 352 353 354
	  nersc,serial:tests/databases/silo.py:silo_03,silo_04 \
	  nersc,serial:tests/rendering/scalable.py:scalable_01,scalable_02,scalable_03,scalable_04,scalable_05 \
	  nersc,parallel:tests/databases/silo.py:silo_03,silo_04 \
	  nersc,parallel:tests/rendering/scalable.py:scalable_01,scalable_02,scalable_03,scalable_04,scalable_05 \
	  nersc,scalable,parallel:tests/databases/silo.py:silo_03,silo_04 \
	  nersc,scalable,parallel:tests/rendering/scalable.py:scalable_01,scalable_02,scalable_03,scalable_04,scalable_05"
355

356 357
skipList="$skipList_dueTo_clientServer $skipList_dueTo_parallelRendering $skipList_dueTo_temporaryBugs"

358 359 360 361 362 363 364 365 366 367 368 369
#
# Make directory for where to store core files
#
rm -rf test/cores
mkdir test/cores

#
# run the test(s)
error=0
cd test

#
370
# Short term fixes for differences in old/new directory structure
371 372
#
sed -i -e 's|^exepath="../src/bin/visit"|exepath="../bin/visit"|' runtest
373
sed -i -e 's|^#  CLARGS: -cli -s ../src/bin/visitdiff.py \(.*\)|#  CLARGS: -cli -s ../bin/visitdiff.py \1|' tests/databases/diff.py
374
rm -f ~miller/.visit/linux-intel/plugins/databases/*.so
375 376 377 378 379 380 381

for m in $modes; do

    # clean up ipc resources
    /home/visit/clearcase_bin/cleanipcs 1> /dev/null 2>&1

    if test "$skipList" = ""; then
382
        ./runtest -q -p -m "$m" -d "$curdate" -notrackmem
383
    else
384
        ./runtest -q -s "$skipList" -p -m "$m" -d "$curdate" -notrackmem
385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431
    fi
    curerror=$?
    theMode=`echo $m | tr ',' '_'`
    rm -f short_summary
    touch short_summary
    echo "Failed results should be obtainable from" >> short_summary
    echo "ftp://ftp.llnl.gov/outgoing/visit_results_${theMode}_${dateTag}.tar.gz" >> short_summary
    grep 'category\|failed' summary >> short_summary
    checkAndHandleError $curerror warning "test mode $m failed" short_summary 

    # try to use auto-login, anonymous ftp to
    # put failed results on ftp.llnl.gov
    failedPyFiles=`grep failed short_summary | cut -d':' -f1`
    if test -n "$failedPyFiles"; then
        mkdir html_$dateTag
        cp html/index.html html_$dateTag/.
        for f in $failedPyFiles; do
            fileKey=`basename $f .py`
	    cp html/*$fileKey* html_$dateTag/.
        done
	tar cf - html_$dateTag | gzip > visit_results_${theMode}_${dateTag}.tar.gz
	hasOrigNetrc=0
	if test -e .netrc; then
	    hasOrigNetrc=1
	    mv .netrc .netrc.orig
	fi
	touch .netrc
	chmod 600 .netrc
	echo "machine ftp.llnl.gov login anonymous password miller86@llnl.gov" >> ./.netrc
        rm -f ftp_commands.txt 
	echo "cd outgoing" >> ftp_commands.txt
	echo "bin" >> ftp_commands.txt
	echo "put visit_results_${theMode}_${dateTag}.tar.gz" >> ftp_commands.txt
	env HOME=. ftp ftp.llnl.gov < ftp_commands.txt
        checkAndHandleError $? "ftp transfer of results failed" sftp.log
	rm -f visit_results_${theMode}_${dateTag}.tar.gz
	rm -f .netrc
	rm -f ftp_commands.txt
	if test $hasOrigNetrc -eq 1; then
	    mv .netrc.orig .netrc
	fi
        rm -f sftp_batch sftp.log
        rm -rf html_$dateTag
    fi

    if test ! "$curerror" = "0"; then
        error=1
432 433
    else
        checkAndHandleError 1 warning "test mode $m succeeded"
434 435 436 437 438 439 440 441 442
    fi
done

#
# Reset last pass modifiers if everything passed
#
if test $error -eq 0; then
    rm -f $modifiersFile
    touch $modifiersFile
443 444 445 446 447 448 449 450 451
    if test "$skipList" = ""; then
        checkAndHandleError 1 flush "Complete Pass -- All tests Passed!!!"
    else
        checkAndHandleError 1 flush "Partial Pass - Some tests skipped"
    fi
    exit 0
else
    checkAndHandleError 1 flush "Tests failed"
    exit 1
452
fi