Commit 39ac8b4e authored by Brad King's avatar Brad King
Browse files

ctest: Add --repeat-after-timeout option

Add an option to re-run tests if they timeout.  This will help tolerate
sporadic timeouts on busy machines.
parent 80c2c9d1
...@@ -268,9 +268,17 @@ Options ...@@ -268,9 +268,17 @@ Options
``--repeat-until-pass <n>`` ``--repeat-until-pass <n>``
Allow each test to run up to ``<n>`` times in order to pass. Allow each test to run up to ``<n>`` times in order to pass.
Repeats tests if they fail for any reason.
This is useful in tolerating sporadic failures in test cases. This is useful in tolerating sporadic failures in test cases.
``--repeat-after-timeout <n>``
Allow each test to run up to ``<n>`` times in order to pass.
Repeats tests only if they timeout.
This is useful in tolerating sporadic timeouts in test cases
on busy machines.
``--max-width <width>`` ``--max-width <width>``
Set the max width for a test name to output. Set the max width for a test name to output.
......
ctest-repeat-until-pass ctest-repeat-until-pass
----------------------- -----------------------
* The :manual:`ctest(1)` tool learned a new ``--repeat-until-pass <n>`` * The :manual:`ctest(1)` tool learned new ``--repeat-until-pass <n>``
option to help tolerate sporadic test failures. and ``--repeat-after-timeout <n>`` options to help tolerate sporadic
test failures.
...@@ -345,7 +345,9 @@ bool cmCTestRunTest::NeedsToRerun() ...@@ -345,7 +345,9 @@ bool cmCTestRunTest::NeedsToRerun()
if ((this->RerunMode == cmCTest::Rerun::UntilFail && if ((this->RerunMode == cmCTest::Rerun::UntilFail &&
this->TestResult.Status == cmCTestTestHandler::COMPLETED) || this->TestResult.Status == cmCTestTestHandler::COMPLETED) ||
(this->RerunMode == cmCTest::Rerun::UntilPass && (this->RerunMode == cmCTest::Rerun::UntilPass &&
this->TestResult.Status != cmCTestTestHandler::COMPLETED)) { this->TestResult.Status != cmCTestTestHandler::COMPLETED) ||
(this->RerunMode == cmCTest::Rerun::AfterTimeout &&
this->TestResult.Status == cmCTestTestHandler::TIMEOUT)) {
this->RunAgain = true; this->RunAgain = true;
return true; return true;
} }
...@@ -745,10 +747,11 @@ void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total) ...@@ -745,10 +747,11 @@ void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total)
// then it will never print out the completed / total, same would // then it will never print out the completed / total, same would
// got for run until pass. Trick is when this is called we don't // got for run until pass. Trick is when this is called we don't
// yet know if we are passing or failing. // yet know if we are passing or failing.
if ((this->RerunMode != cmCTest::Rerun::UntilPass && bool const progressOnLast =
this->NumberOfRunsLeft == 1) || (this->RerunMode != cmCTest::Rerun::UntilPass &&
(this->RerunMode == cmCTest::Rerun::UntilPass && this->RerunMode != cmCTest::Rerun::AfterTimeout);
this->NumberOfRunsLeft == this->NumberOfRunsTotal) || if ((progressOnLast && this->NumberOfRunsLeft == 1) ||
(!progressOnLast && this->NumberOfRunsLeft == this->NumberOfRunsTotal) ||
this->CTest->GetTestProgressOutput()) { this->CTest->GetTestProgressOutput()) {
outputStream << std::setw(getNumWidth(total)) << completed << "/"; outputStream << std::setw(getNumWidth(total)) << completed << "/";
outputStream << std::setw(getNumWidth(total)) << total << " "; outputStream << std::setw(getNumWidth(total)) << total << " ";
......
...@@ -1884,6 +1884,28 @@ bool cmCTest::HandleCommandLineArguments(size_t& i, ...@@ -1884,6 +1884,28 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
} }
} }
if (this->CheckArgument(arg, "--repeat-after-timeout")) {
if (i >= args.size() - 1) {
errormsg = "'--repeat-after-timeout' requires an argument";
return false;
}
if (this->Impl->RerunMode != cmCTest::Rerun::Never) {
errormsg = "At most one '--repeat-*' option may be used.";
return false;
}
i++;
long repeat = 1;
if (!cmStrToLong(args[i], &repeat)) {
errormsg =
"'--repeat-after-timeout' given non-integer value '" + args[i] + "'";
return false;
}
this->Impl->RepeatTests = static_cast<int>(repeat);
if (repeat > 1) {
this->Impl->RerunMode = cmCTest::Rerun::AfterTimeout;
}
}
if (this->CheckArgument(arg, "--test-load") && i < args.size() - 1) { if (this->CheckArgument(arg, "--test-load") && i < args.size() - 1) {
i++; i++;
unsigned long load; unsigned long load;
......
...@@ -438,6 +438,7 @@ public: ...@@ -438,6 +438,7 @@ public:
Never, Never,
UntilFail, UntilFail,
UntilPass, UntilPass,
AfterTimeout,
}; };
Rerun GetRerunMode() const; Rerun GetRerunMode() const;
......
...@@ -102,6 +102,8 @@ static const char* cmDocumentationOptions[][2] = { ...@@ -102,6 +102,8 @@ static const char* cmDocumentationOptions[][2] = {
"Require each test to run <n> times without failing in order to pass" }, "Require each test to run <n> times without failing in order to pass" },
{ "--repeat-until-pass <n>", { "--repeat-until-pass <n>",
"Allow each test to run up to <n> times in order to pass" }, "Allow each test to run up to <n> times in order to pass" },
{ "--repeat-after-timeout <n>",
"Allow each test to run up to <n> times if it times out" },
{ "--max-width <width>", "Set the max width for a test name to output" }, { "--max-width <width>", "Set the max width for a test name to output" },
{ "--interactive-debug-mode [0|1]", "Set the interactive mode to 0 or 1." }, { "--interactive-debug-mode [0|1]", "Set the interactive mode to 0 or 1." },
{ "--hardware-spec-file <file>", "Set the hardware spec file to use." }, { "--hardware-spec-file <file>", "Set the hardware spec file to use." },
......
...@@ -24,12 +24,25 @@ run_cmake_command(repeat-until-fail-good ...@@ -24,12 +24,25 @@ run_cmake_command(repeat-until-fail-good
${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2
) )
run_cmake_command(repeat-after-timeout-bad1
${CMAKE_CTEST_COMMAND} --repeat-after-timeout
)
run_cmake_command(repeat-after-timeout-bad2
${CMAKE_CTEST_COMMAND} --repeat-after-timeout foo
)
run_cmake_command(repeat-after-timeout-good
${CMAKE_CTEST_COMMAND} --repeat-after-timeout 2
)
run_cmake_command(repeat-until-pass-and-fail run_cmake_command(repeat-until-pass-and-fail
${CMAKE_CTEST_COMMAND} --repeat-until-pass 2 --repeat-until-fail 2 ${CMAKE_CTEST_COMMAND} --repeat-until-pass 2 --repeat-until-fail 2
) )
run_cmake_command(repeat-until-fail-and-pass run_cmake_command(repeat-until-fail-and-pass
${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-until-pass 2 ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-until-pass 2
) )
run_cmake_command(repeat-until-fail-and-timeout
${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-after-timeout 2
)
function(run_repeat_until_pass_tests) function(run_repeat_until_pass_tests)
# Use a single build tree for a few tests without cleaning. # Use a single build tree for a few tests without cleaning.
...@@ -42,6 +55,17 @@ function(run_repeat_until_pass_tests) ...@@ -42,6 +55,17 @@ function(run_repeat_until_pass_tests)
endfunction() endfunction()
run_repeat_until_pass_tests() run_repeat_until_pass_tests()
function(run_repeat_after_timeout_tests)
# Use a single build tree for a few tests without cleaning.
set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-after-timeout-build)
run_cmake(repeat-after-timeout-cmake)
set(RunCMake_TEST_NO_CLEAN 1)
run_cmake_command(repeat-after-timeout-ctest
${CMAKE_CTEST_COMMAND} -C Debug --repeat-after-timeout 3
)
endfunction()
run_repeat_after_timeout_tests()
function(run_repeat_until_fail_tests) function(run_repeat_until_fail_tests)
# Use a single build tree for a few tests without cleaning. # Use a single build tree for a few tests without cleaning.
set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-fail-build) set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-fail-build)
......
^CMake Error: '--repeat-after-timeout' requires an argument$
^CMake Error: '--repeat-after-timeout' given non-integer value 'foo'$
enable_testing()
set(TEST_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/test_output.txt")
add_test(NAME initialization
COMMAND ${CMAKE_COMMAND}
"-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
-P "${CMAKE_CURRENT_SOURCE_DIR}/init.cmake")
add_test(NAME test1
COMMAND ${CMAKE_COMMAND}
"-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
-P "${CMAKE_CURRENT_SOURCE_DIR}/test1-timeout.cmake")
set_tests_properties(test1 PROPERTIES DEPENDS "initialization" TIMEOUT 2)
add_test(hello ${CMAKE_COMMAND} -E echo hello)
add_test(goodbye ${CMAKE_COMMAND} -E echo goodbye)
^Test project .*/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-build
Start 1: initialization
1/4 Test #1: initialization ................... Passed +[0-9.]+ sec
Start 2: test1
2/4 Test #2: test1 ............................\*\*\*Timeout +[0-9.]+ sec
Start 2: test1
Test #2: test1 ............................ Passed +[0-9.]+ sec
Start 3: hello
3/4 Test #3: hello ............................ Passed +[0-9.]+ sec
Start 4: goodbye
4/4 Test #4: goodbye .......................... Passed +[0-9.]+ sec
100% tests passed, 0 tests failed out of 4
Total Test time \(real\) = +[0-9.]+ sec$
^CMake Error: At most one '--repeat-\*' option may be used\.$
# This is run by test test1 in repeat-after-timeout-cmake.cmake with cmake -P.
# It reads the file TEST_OUTPUT_FILE and increments the number
# found in the file by 1. Unless the number is 2, then the
# code sends out a cmake error causing the test to not timeout only on
# the second time it is run.
message("TEST_OUTPUT_FILE = ${TEST_OUTPUT_FILE}")
file(READ "${TEST_OUTPUT_FILE}" COUNT)
message("COUNT= ${COUNT}")
math(EXPR COUNT "${COUNT} + 1")
file(WRITE "${TEST_OUTPUT_FILE}" "${COUNT}")
if(NOT COUNT EQUAL 2)
message("this test times out except on the 2nd run")
execute_process(COMMAND ${CMAKE_COMMAND} -E sleep 10)
endif()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment