regressiontest_surface 12.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
#!/bin/sh
#
# Modifications:
#   Mark C. Miller, Thu Jul 29 23:35:19 PDT 2010
#   Added option to ignore third party lib problems to cmake invokation.
#
#   Eric Brugger, Thu Mar 31 09:38:38 PDT 2011
#   Modified the script to work with the 2.2RC instead of the 2.0RC.
#
#   Eric Brugger, Tue Apr 12 10:53:02 PDT 2011
#   Removed the specular lighting tests from the skip list.
#
#   Jeremy Meredith, Tue Aug  9 17:19:55 EDT 2011
#   Updated the subversion host/path.
#
#   Eric Brugger, Thu Aug 25 11:22:39 PDT 2011
#   Modified the script to work on edge.
18 19 20 21
#
#   Eric Brugger, Thu Aug 25 14:11:16 PDT 2011
#   Modified the script to send out the e-mail only after the results
#   were posted.
22 23 24 25 26
#
#   Eric Brugger, Wed Jan 25 13:26:36 PST 2012
#   Modified the command that submits the batch job to run the test suite
#   to specify that it isn't using any of the Lustre file systems so that
#   the job will run even if one of the Lustre file systems is down.
27 28 29 30
#
#   Eric Brugger, Fri Jun 22 07:54:19 PDT 2012
#   I modified the script to use cmake 2.8.8. I also added several failing
#   tests to the skip list.
31 32 33 34
#
#   Mark C. Miller, Wed Jun 27 12:30:13 PDT 2012
#   Removed zipwrapper tests from skip list because the tests were updated
#   to use STL in place of VTK.
35
#
cyrush's avatar
cyrush committed
36
#   Cyrus Harrison, Wed Aug 15 12:31:00 PDT 2012
37 38
#   Update to use python test suite
#
39 40 41 42
#   Eric Brugger, Fri Oct 11 14:10:51 PDT 2013
#   Change the path to the cmake used to build visit to the latest 2.8.10.2
#   version.
#
43 44 45 46
#   Eric Brugger, Fri Oct 18 15:01:56 PDT 2013
#   I changed "make test" to "make testdata" in the data directory since the
#   name of the target changed.
#
47 48 49 50
#   Eric Brugger, Fri Aug 15 11:02:18 PDT 2014
#   I added "--parallel-launch srun" to the runtest command to specify srun
#   as the parallel launch method.
#
51 52 53
#   Eric Brugger, Fri Aug 15 12:54:39 PDT 2014
#   I modified the script to use cmake 2.8.12.2 to configure visit.
#
54 55 56
#   Eric Brugger, Wed Oct 29 08:04:01 PDT 2014
#   I modified the script to use surface.
#
57 58 59 60
#   Eric Brugger, Tue Jan 13 14:09:04 PST 2015
#   I modified the script to use the account views for submitting the batch
#   job.
#
61 62 63
#   Eric Brugger, Thu Feb 12 08:17:18 PST 2015
#   I modified the script to use cmake 3.0.2 to configure visit.
#
64 65 66 67 68
#   Eric Brugger, Fri Sep 11 14:05:18 PDT 2015
#   I added the command line option "-branch", which takes a branch name.
#   While I was at it I also modified the script to use 16 cores instead of
#   12 since it is grabbing an entire node of surface, which has 16 cores.
#
69

70 71 72 73 74 75 76 77 78 79 80 81 82 83

#
# Determine the users name and e-mail address.
#
user=`whoami`

#
# Set the user e-mail address.
#
userNersc=unknown
userEmail=unknown
case $user in
    bonnell)
        userNersc=bonnell
bonnell's avatar
bonnell committed
84
        userEmail=biagas2@llnl.gov
85 86 87
        ;;
    kbonnell)
        userNersc=bonnell
bonnell's avatar
bonnell committed
88
        userEmail=biagas2@llnl.gov
89 90 91 92 93
        ;;
    brugger)
        userNersc=brugger
        userEmail=brugger1@llnl.gov
        ;;
94 95 96 97
    cyrush)
        userNersc=cyrush
        userEmail=cyrush@llnl.gov
        ;;
98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113
    hrchilds)
        userNersc=hrchilds
        userEmail=hankchilds3@gmail.com
        ;;
    mcmiller)
        userNersc=miller86
        userEmail=miller86@llnl.gov
        ;;
    miller)
        userNersc=miller86
        userEmail=miller86@llnl.gov
        ;;
    miller86)
        userNersc=miller86
        userEmail=miller86@llnl.gov
        ;;
114 115 116 117 118 119 120 121
    harrison37)
        userNersc=cyrush
        userEmail=cyrush@llnl.gov
        ;;
    cyrush)
        userNersc=cyrush
        userEmail=cyrush@llnl.gov
        ;;
122 123 124 125 126 127 128 129 130 131 132 133 134 135 136
esac

if test "$userEmail" = "unknown" ; then
    echo "Unknown user name. Exiting."
    exit 1
fi

#
# Parse the command line to determine if we should run the test suite
# on the trunk or RC.
#
debug="false"
branch="trunk"
revision="latest"
serial="false"
137
testHost="surface"
cyrush's avatar
cyrush committed
138
testDir="/nfs/tmp2/"
139
post="true"
140
nerscHost="edison.nersc.gov"
141 142 143 144 145 146 147 148

for abc
do
    case $1 in
        -trunk)
            branch="trunk"
            shift
            ;;
149 150 151
        -branch)
            branch=$2
            shift 2
152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172
            ;;
        -serial)
            serial="true"
            shift
            ;;
        -parallel)
            serial="false"
            shift
            ;;
        -d)
            debug="true"
            shift
            ;;
        -r)
            revision=$2
            shift 2
            ;;
        -host)
            testHost=$2
            shift 2
            ;;
173 174 175 176
        -no-post)
            post="false"
            shift
            ;;
177 178 179 180 181 182
    esac
done

if test "$branch" = "trunk" ; then
    workingDir="test_trunk"
else
183
    workingDir="test_branch"
184 185 186 187 188 189 190 191 192
fi

#
# Run the test suite on $testHost.
#
rm -f $testHost
cat <<EOF > $testHost
#!/bin/sh

cyrush's avatar
cyrush committed
193
dateTag=\`date +%Y-%m-%d-%H:%M\`
194 195 196 197 198 199 200

if test "$serial" = "true" ; then
    modes="$testHost,serial"
else
    modes="$testHost,serial $testHost,parallel $testHost,scalable,parallel,icet"
fi

201 202 203 204
# Check out the source
if test "$debug" = "true" ; then
   echo "Removing any old test files at: $testDir/$user/$workingDir" | mail -s "Checkout started" $userEmail
fi
205

cyrush's avatar
cyrush committed
206 207
echo "Removing any old test files at: $testDir/$user/$workingDir"

208 209 210 211 212 213 214 215 216 217 218 219
# Create the test directory.
if test ! -e $testDir/$user; then
    mkdir $testDir/$user
fi
rm -rf $testDir/$user/$workingDir
mkdir $testDir/$user/$workingDir
cd $testDir/$user/$workingDir

# Check out the source
if test "$debug" = "true" ; then
   echo "Checkout of $branch started at: \`date\`" | mail -s "Checkout started" $userEmail
fi
cyrush's avatar
cyrush committed
220 221 222

echo "Checkout of $branch started at: \`date\`"

223 224 225 226 227 228 229 230 231 232 233 234
if test "$branch" = "trunk" ; then
    if test "$revision" = "latest" ; then
        svn co svn+ssh://$userNersc@portal-auth.nersc.gov/project/projectdirs/visit/svn/visit/trunk/src > buildlog 2>&1
        svn co svn+ssh://$userNersc@portal-auth.nersc.gov/project/projectdirs/visit/svn/visit/trunk/data >> buildlog 2>&1
        svn co svn+ssh://$userNersc@portal-auth.nersc.gov/project/projectdirs/visit/svn/visit/trunk/test >> buildlog 2>&1
    else
        svn co svn+ssh://$userNersc@portal-auth.nersc.gov/project/projectdirs/visit/svn/visit/trunk/src -r $revision > buildlog 2>&1
        svn co svn+ssh://$userNersc@portal-auth.nersc.gov/project/projectdirs/visit/svn/visit/trunk/data -r $revision >> buildlog 2>&1
        svn co svn+ssh://$userNersc@portal-auth.nersc.gov/project/projectdirs/visit/svn/visit/trunk/test -r $revision >> buildlog 2>&1
    fi
else
    if test "$revision" = "latest" ; then
235 236 237
        svn co svn+ssh://$userNersc@portal-auth.nersc.gov/project/projectdirs/visit/svn/visit/branches/$branch/src > buildlog 2>&1
        svn co svn+ssh://$userNersc@portal-auth.nersc.gov/project/projectdirs/visit/svn/visit/branches/$branch/data >> buildlog 2>&1
        svn co svn+ssh://$userNersc@portal-auth.nersc.gov/project/projectdirs/visit/svn/visit/branches/$branch/test >> buildlog 2>&1
238
    else
239 240 241
        svn co svn+ssh://$userNersc@portal-auth.nersc.gov/project/projectdirs/visit/svn/visit/branches/$branch/src -r $revision > buildlog 2>&1
        svn co svn+ssh://$userNersc@portal-auth.nersc.gov/project/projectdirs/visit/svn/visit/branches/$branch/data -r $revision >> buildlog 2>&1
        svn co svn+ssh://$userNersc@portal-auth.nersc.gov/project/projectdirs/visit/svn/visit/branches/$branch/test -r $revision >> buildlog 2>&1
242 243 244 245 246 247
    fi
fi
if test "$debug" = "true" ; then
    echo "Checkout of $branch completed at: \`date\`" | mail -s "Checkout completed" $userEmail
fi

cyrush's avatar
cyrush committed
248 249
echo "Checkout of $branch completed at: \`date\`"

250 251 252
# Build the code
cd src
export PATH=\$PATH:/usr/bin/X11
253 254 255 256 257 258 259 260
export PATH=/opt/cudatoolkit-6.0/open64/bin:/opt/cudatoolkit-6.0/bin:\$PATH
export CUDA_INCLUDES=/opt/cudatoolkit-6.0/include
export CUDA_INSTALL_PATH=/opt/cudatoolkit-6.0
export CUDA_LIBS=/opt/cudatoolkit-6.0/lib64
export CUDA_PATH=/opt/cudatoolkit-6.0
export LD_LIBRARY_PATH=/opt/cudatoolkit-6.0/lib64:/usr/lib64/nvidia
export NVCC=/opt/cudatoolkit-6.0/bin/nvcc
export OPENGLLIB=-L/opt/cudatoolkit-6.0/lib64 -lcudart -lcuda
261

262
if test "$serial" = "true" ; then
263
    sed -i "s/VISIT_PARALLEL ON/VISIT_PARALLEL OFF/" config-site/surface86.cmake
264
fi
265
/usr/gapps/visit/thirdparty_shared/2.9.0/cmake/3.0.2/linux-x86_64_gcc-4.4/bin/cmake \
266
    -DVISIT_CONFIG_SITE=config-site/surface86.cmake \
cyrush's avatar
cyrush committed
267 268
    -DCMAKE_BUILD_TYPE=Release \
    -DVISIT_BUILD_ALL_PLUGINS=1 \
269
    -DVISIT_DATA_MANUAL_EXAMPLES:BOOL=ON \
cyrush's avatar
cyrush committed
270
    -DIGNORE_THIRD_PARTY_LIB_PROBLEMS:BOOL=ON \
271 272
    -DCMAKE_CXX_FLAGS_RELEASE:STRING="-O3 -DNDEBUG -Wall -pedantic -Wextra -Wno-long-long -Wno-unused-parameter" \
    -DCMAKE_C_FLAGS_RELEASE:STRING="-O3 -DNDEBUG -Wall -pedantic -Wextra -Wno-long-long -Wno-unused-parameter" \
273
    -DCMAKE_MAKE_PROGRAM:FILEPATH="/usr/gapps/visit/gmake/4.0/chaos_5_x86_64_ib/bin/make" \
274
    -DVISIT_EAVL_DIR="/usr/gapps/visit/thirdparty_shared/2.10.0/eavl/4593630/linux-x86_64_gcc-4.4" \
cyrush's avatar
cyrush committed
275
    . >> ../buildlog 2>&1
276
rm -rf ../make.out ../make.err
277
/usr/gapps/visit/gmake/4.0/chaos_5_x86_64_ib/bin/make --output-sync=line -k -j 16 1>../make.out 2>../make.err
278 279 280 281
if test $? -ne 0; then
    echo "Source build FAILED at: \`date\`" | mail -s "Source build FAILED" $userEmail
    exit 1
fi
282 283 284
if test "$debug" = "true" ; then
    echo "Source build completed at: \`date\`" | mail -s "Source build completed" $userEmail
fi
miller86's avatar
miller86 committed
285 286
cat ../make.out >> ../buildlog
cat ../make.err >> ../buildlog
287

cyrush's avatar
cyrush committed
288 289
echo "Source build completed at: \`date\`"

290 291
# Build the test data
cd ../data
292
make -j 8 testdata >> ../buildlog 2>&1
293 294 295 296
if test "$debug" = "true" ; then
    echo "Data build completed at: \`date\`" | mail -s "Data build completed" $userEmail
fi

cyrush's avatar
cyrush committed
297
echo "Data build completed at: \`date\`"
298 299 300 301 302

if test "$debug" = "true" ; then
    echo "Starting tests:\`date\`" | mail -s "Starting tests" $userEmail
fi

cyrush's avatar
cyrush committed
303 304
echo "Starting tests:\`date\`"

305 306 307 308
# Run the tests, consolidating the results in one directory
cd ../test
rm -rf \$dateTag
mkdir \$dateTag
cyrush's avatar
cyrush committed
309
mkdir output
310
for m in \$modes; do
311
    theMode=\`echo \$m | tr ',' '_'\`
cyrush's avatar
cyrush committed
312
    resDir=output/\$theMode
313 314 315

    if test "$debug" = "true" ; then
        echo "runtest with mode = \$m" | mail -s "runtest with mode = \$m" $userEmail
316 317
    fi

cyrush's avatar
cyrush committed
318 319 320
    echo "runtest with mode = \$m"


321
    # Run the test
322
    ./runtest --parallel-launch "srun" -o \$resDir -m "\$m" -n 1 >> buildlog 2>&1
323
    mv \$resDir/html \$dateTag/\$theMode
324 325 326 327 328

    # Move the results to the consolidation directory
    if test "$debug" = "true" ; then
        echo "Test \$m completed at: \`date\`" | mail -s "Tests completed" $userEmail
    fi
cyrush's avatar
cyrush committed
329
    echo "Test \$m completed at: \`date\`"
330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346
done

# Determine if the test suite passed or failed.
hasFailed=""
for m in \$modes; do
    theMode=\`echo \$m | tr ',' '_'\`
    if test "\$hasFailed" = ""; then
        hasFailed=\`cat "\$dateTag/\$theMode/index.html" | grep -m 1 Failed 2>/dev/null\`
    fi
    if test "\$hasFailed" = ""; then
        hasFailed=\`cat "\$dateTag/\$theMode/index.html" | grep -m 1 Unacceptable 2>/dev/null\`
    fi
    if test "\$hasFailed" = ""; then
        hasFailed=\`cat "\$dateTag/\$theMode/index.html" | grep -m 1 OS-Killed 2>/dev/null\`
    fi
done

347 348 349 350 351 352 353 354 355 356 357
if test "$post" = "true" ; then
    # If the test suite passed then update the revision number indicating the
    # last pass.
    cd ..
    if test "\$hasFailed" = ""; then
        svn update src/svn_bin/lastpass_$testHost
        svnRevision=\`svn info src | grep "Revision:" | cut -d' ' -f2\`
        rm -f src/svn_bin/lastpass_$testHost
        echo "\$svnRevision" > src/svn_bin/lastpass_$testHost
        svn commit -m "Update the revision number of the last test suite pass." src/svn_bin/lastpass_$testHost
    fi
358

359 360
    # Tar up the results
    cd test
cyrush's avatar
cyrush committed
361
    tar -czvf  html.tar.gz \$dateTag
362

cyrush's avatar
cyrush committed
363 364
    # Copy the results to nersc
    scp html.tar.gz $userNersc@$nerscHost:
365
    if test "$debug" = "true" ; then
cyrush's avatar
cyrush committed
366
        echo "Results copied to $nerscHost at: `date`" | mail -s "Results copied" $userEmail
367
    fi
368

cyrush's avatar
cyrush committed
369 370
    echo "Results copied to $nerscHost at: `date`"

371 372
    # Post the results.
    cd ..
cyrush's avatar
cyrush committed
373 374
    scp src/svn_bin/visit-update-test-website $userNersc@$nerscHost:
    ssh $userNersc@$nerscHost "./visit-update-test-website"
375 376 377
    if test "$debug" = "true" ; then
        echo "Results posted at: `date`" | mail -s "Results posted" $userEmail
    fi
378

cyrush's avatar
cyrush committed
379 380
    echo "Results posted at: `date`"

381 382 383 384 385 386
    # Send out an e-mail notifying the users of the test suite status.
    if test "\$hasFailed" = ""; then
        src/svn_bin/visit-notify-test-failure -pass -host $testHost
    else
        src/svn_bin/visit-notify-test-failure -host $testHost
    fi
387

388
    # If its a sunday, purge old test results
389 390 391 392
    if test "$(date +%A)" = "Sunday" ; then
        scp src/svn_bin/purge_old_test_results.sh $userNersc@$nerscHost:
        ssh $userNersc@$nerscHost "./purge_old_test_results.sh"
    fi
393

394 395 396 397 398
    # Set the permissions so that others may access the test directory.
    cd ..
    chgrp -R visit $workingDir
    chmod -R g+rwX,o+rX $workingDir
fi
399 400 401 402 403 404

EOF

chmod 750 $testHost

# Run the test suite.
405 406
if test "$testHost" = "surface" ; then
    scp surface surface:surface_testit
cyrush's avatar
cyrush committed
407
    subTag=`date +%Y-%m-%d-%H:%M`
408
    ssh surface "msub -l nodes=1:ppn=16 -l gres=ignore -l walltime=12:00:00 -o surface_testit_$subTag.out -q pbatch -A views -z ./surface_testit"
409
fi