ctest: Add --repeat-after-timeout option

Add an option to re-run tests if they timeout.  This will help tolerate
sporadic timeouts on busy machines.

Help/manual/ctest.1.rst

@@ -268,9 +268,17 @@ Options
 
 ``--repeat-until-pass <n>``
  Allow each test to run up to ``<n>`` times in order to pass.
+ Repeats tests if they fail for any reason.
 
  This is useful in tolerating sporadic failures in test cases.
 
+``--repeat-after-timeout <n>``
+ Allow each test to run up to ``<n>`` times in order to pass.
+ Repeats tests only if they timeout.
+
+ This is useful in tolerating sporadic timeouts in test cases
+ on busy machines.
+
 ``--max-width <width>``
  Set the max width for a test name to output.
 

Help/release/dev/ctest-repeat-until-pass.rst

 ctest-repeat-until-pass
 -----------------------
 
-* The :manual:`ctest(1)` tool learned a new ``--repeat-until-pass <n>``
-  option to help tolerate sporadic test failures.
+* The :manual:`ctest(1)` tool learned new ``--repeat-until-pass <n>``
+  and ``--repeat-after-timeout <n>`` options to help tolerate sporadic
+  test failures.

Source/CTest/cmCTestRunTest.cxx

@@ -345,7 +345,9 @@ bool cmCTestRunTest::NeedsToRerun()
   if ((this->RerunMode == cmCTest::Rerun::UntilFail &&
        this->TestResult.Status == cmCTestTestHandler::COMPLETED) ||
       (this->RerunMode == cmCTest::Rerun::UntilPass &&
-       this->TestResult.Status != cmCTestTestHandler::COMPLETED)) {
+       this->TestResult.Status != cmCTestTestHandler::COMPLETED) ||
+      (this->RerunMode == cmCTest::Rerun::AfterTimeout &&
+       this->TestResult.Status == cmCTestTestHandler::TIMEOUT)) {
     this->RunAgain = true;
     return true;
   }
@@ -745,10 +747,11 @@ void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total)
   // then it will never print out the completed / total, same would
   // got for run until pass.  Trick is when this is called we don't
   // yet know if we are passing or failing.
-  if ((this->RerunMode != cmCTest::Rerun::UntilPass &&
-       this->NumberOfRunsLeft == 1) ||
-      (this->RerunMode == cmCTest::Rerun::UntilPass &&
-       this->NumberOfRunsLeft == this->NumberOfRunsTotal) ||
+  bool const progressOnLast =
+    (this->RerunMode != cmCTest::Rerun::UntilPass &&
+     this->RerunMode != cmCTest::Rerun::AfterTimeout);
+  if ((progressOnLast && this->NumberOfRunsLeft == 1) ||
+      (!progressOnLast && this->NumberOfRunsLeft == this->NumberOfRunsTotal) ||
       this->CTest->GetTestProgressOutput()) {
     outputStream << std::setw(getNumWidth(total)) << completed << "/";
     outputStream << std::setw(getNumWidth(total)) << total << " ";

Source/cmCTest.cxx

@@ -1884,6 +1884,28 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
     }
   }
 
+  if (this->CheckArgument(arg, "--repeat-after-timeout")) {
+    if (i >= args.size() - 1) {
+      errormsg = "'--repeat-after-timeout' requires an argument";
+      return false;
+    }
+    if (this->Impl->RerunMode != cmCTest::Rerun::Never) {
+      errormsg = "At most one '--repeat-*' option may be used.";
+      return false;
+    }
+    i++;
+    long repeat = 1;
+    if (!cmStrToLong(args[i], &repeat)) {
+      errormsg =
+        "'--repeat-after-timeout' given non-integer value '" + args[i] + "'";
+      return false;
+    }
+    this->Impl->RepeatTests = static_cast<int>(repeat);
+    if (repeat > 1) {
+      this->Impl->RerunMode = cmCTest::Rerun::AfterTimeout;
+    }
+  }
+
   if (this->CheckArgument(arg, "--test-load") && i < args.size() - 1) {
     i++;
     unsigned long load;

Source/cmCTest.h

@@ -438,6 +438,7 @@ public:
     Never,
     UntilFail,
     UntilPass,
+    AfterTimeout,
   };
   Rerun GetRerunMode() const;
 

Source/ctest.cxx

@@ -102,6 +102,8 @@ static const char* cmDocumentationOptions[][2] = {
     "Require each test to run <n> times without failing in order to pass" },
   { "--repeat-until-pass <n>",
     "Allow each test to run up to <n> times in order to pass" },
+  { "--repeat-after-timeout <n>",
+    "Allow each test to run up to <n> times if it times out" },
   { "--max-width <width>", "Set the max width for a test name to output" },
   { "--interactive-debug-mode [0|1]", "Set the interactive mode to 0 or 1." },
   { "--hardware-spec-file <file>", "Set the hardware spec file to use." },

Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake

@@ -24,12 +24,25 @@ run_cmake_command(repeat-until-fail-good
   ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2
   )
 
+run_cmake_command(repeat-after-timeout-bad1
+  ${CMAKE_CTEST_COMMAND} --repeat-after-timeout
+  )
+run_cmake_command(repeat-after-timeout-bad2
+  ${CMAKE_CTEST_COMMAND} --repeat-after-timeout foo
+  )
+run_cmake_command(repeat-after-timeout-good
+  ${CMAKE_CTEST_COMMAND} --repeat-after-timeout 2
+  )
+
 run_cmake_command(repeat-until-pass-and-fail
   ${CMAKE_CTEST_COMMAND} --repeat-until-pass 2 --repeat-until-fail 2
   )
 run_cmake_command(repeat-until-fail-and-pass
   ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-until-pass 2
   )
+run_cmake_command(repeat-until-fail-and-timeout
+  ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-after-timeout 2
+  )
 
 function(run_repeat_until_pass_tests)
   # Use a single build tree for a few tests without cleaning.
@@ -42,6 +55,17 @@ function(run_repeat_until_pass_tests)
 endfunction()
 run_repeat_until_pass_tests()
 
+function(run_repeat_after_timeout_tests)
+  # Use a single build tree for a few tests without cleaning.
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-after-timeout-build)
+  run_cmake(repeat-after-timeout-cmake)
+  set(RunCMake_TEST_NO_CLEAN 1)
+  run_cmake_command(repeat-after-timeout-ctest
+    ${CMAKE_CTEST_COMMAND} -C Debug --repeat-after-timeout 3
+    )
+endfunction()
+run_repeat_after_timeout_tests()
+
 function(run_repeat_until_fail_tests)
   # Use a single build tree for a few tests without cleaning.
   set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-fail-build)

Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-result.txt

+1

Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad1-stderr.txt

+^CMake Error: '--repeat-after-timeout' requires an argument$

Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-result.txt

+1

Tests/RunCMake/CTestCommandLine/repeat-after-timeout-bad2-stderr.txt

+^CMake Error: '--repeat-after-timeout' given non-integer value 'foo'$

Tests/RunCMake/CTestCommandLine/repeat-after-timeout-cmake.cmake

+enable_testing()
+
+set(TEST_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/test_output.txt")
+add_test(NAME initialization
+  COMMAND ${CMAKE_COMMAND}
+  "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
+  -P "${CMAKE_CURRENT_SOURCE_DIR}/init.cmake")
+add_test(NAME test1
+  COMMAND ${CMAKE_COMMAND}
+  "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
+  -P "${CMAKE_CURRENT_SOURCE_DIR}/test1-timeout.cmake")
+set_tests_properties(test1 PROPERTIES DEPENDS "initialization" TIMEOUT 2)
+
+add_test(hello ${CMAKE_COMMAND} -E echo hello)
+add_test(goodbye ${CMAKE_COMMAND} -E echo goodbye)

Tests/RunCMake/CTestCommandLine/repeat-after-timeout-ctest-stdout.txt

+^Test project .*/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-build
+    Start 1: initialization
+1/4 Test #1: initialization ...................   Passed +[0-9.]+ sec
+    Start 2: test1
+2/4 Test #2: test1 ............................\*\*\*Timeout +[0-9.]+ sec
+    Start 2: test1
+    Test #2: test1 ............................   Passed +[0-9.]+ sec
+    Start 3: hello
+3/4 Test #3: hello ............................   Passed +[0-9.]+ sec
+    Start 4: goodbye
+4/4 Test #4: goodbye ..........................   Passed +[0-9.]+ sec
+
+100% tests passed, 0 tests failed out of 4
+
+Total Test time \(real\) = +[0-9.]+ sec$

Tests/RunCMake/CTestCommandLine/repeat-after-timeout-good-stderr.txt

+^No tests were found!!!$

Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-result.txt

+1

Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-timeout-stderr.txt

+^CMake Error: At most one '--repeat-\*' option may be used\.$

Tests/RunCMake/CTestCommandLine/test1-timeout.cmake

+# This is run by test test1 in repeat-after-timeout-cmake.cmake with cmake -P.
+# It reads the file TEST_OUTPUT_FILE and increments the number
+# found in the file by 1.  Unless the number is 2, then the
+# code sends out a cmake error causing the test to not timeout only on
+# the second time it is run.
+message("TEST_OUTPUT_FILE = ${TEST_OUTPUT_FILE}")
+file(READ "${TEST_OUTPUT_FILE}" COUNT)
+message("COUNT= ${COUNT}")
+math(EXPR COUNT "${COUNT} + 1")
+file(WRITE "${TEST_OUTPUT_FILE}" "${COUNT}")
+if(NOT COUNT EQUAL 2)
+  message("this test times out except on the 2nd run")
+  execute_process(COMMAND ${CMAKE_COMMAND} -E sleep 10)
+endif()