ctest: Add --repeat-until-pass option

Add an option to re-run tests if they fail.  This will help tolerate
sporadic failures.

Issue: #17010
Co-Author: Ben Boeckel <ben.boeckel@kitware.com>
Co-Author: Chuck Atkins <chuck.atkins@kitware.com>

Help/manual/ctest.1.rst

@@ -266,6 +266,11 @@ Options
 
  This is useful in finding sporadic failures in test cases.
 
+``--repeat-until-pass <n>``
+ Allow each test to run up to ``<n>`` times in order to pass.
+
+ This is useful in tolerating sporadic failures in test cases.
+
 ``--max-width <width>``
  Set the max width for a test name to output.
 

Help/release/dev/ctest-repeat-until-pass.rst

+ctest-repeat-until-pass
+-----------------------
+
+* The :manual:`ctest(1)` tool learned a new ``--repeat-until-pass <n>``
+  option to help tolerate sporadic test failures.

Source/CTest/cmCTestMultiProcessHandler.cxx

@@ -171,8 +171,8 @@ bool cmCTestMultiProcessHandler::StartTestProcess(int test)
   this->RunningCount += GetProcessorsUsed(test);
 
   cmCTestRunTest* testRun = new cmCTestRunTest(*this);
-  if (this->CTest->GetRepeatUntilFail()) {
-    testRun->SetRunUntilFailOn();
+  if (this->CTest->GetRerunMode() != cmCTest::Rerun::Never) {
+    testRun->SetRerunMode(this->CTest->GetRerunMode());
     testRun->SetNumberOfRuns(this->CTest->GetTestRepeat());
   }
   testRun->SetIndex(test);

Source/CTest/cmCTestRunTest.cxx

@@ -340,10 +340,12 @@ bool cmCTestRunTest::NeedsToRerun()
     return false;
   }
   // if number of runs left is not 0, and we are running until
-  // we find a failed test, then return true so the test can be
+  // we find a failed (or passed) test, then return true so the test can be
   // restarted
-  if (this->RunUntilFail &&
-      this->TestResult.Status == cmCTestTestHandler::COMPLETED) {
+  if ((this->RerunMode == cmCTest::Rerun::UntilFail &&
+       this->TestResult.Status == cmCTestTestHandler::COMPLETED) ||
+      (this->RerunMode == cmCTest::Rerun::UntilPass &&
+       this->TestResult.Status != cmCTestTestHandler::COMPLETED)) {
     this->RunAgain = true;
     return true;
   }
@@ -743,7 +745,11 @@ void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total)
   // then it will never print out the completed / total, same would
   // got for run until pass.  Trick is when this is called we don't
   // yet know if we are passing or failing.
-  if (this->NumberOfRunsLeft == 1 || this->CTest->GetTestProgressOutput()) {
+  if ((this->RerunMode != cmCTest::Rerun::UntilPass &&
+       this->NumberOfRunsLeft == 1) ||
+      (this->RerunMode == cmCTest::Rerun::UntilPass &&
+       this->NumberOfRunsLeft == this->NumberOfRunsTotal) ||
+      this->CTest->GetTestProgressOutput()) {
     outputStream << std::setw(getNumWidth(total)) << completed << "/";
     outputStream << std::setw(getNumWidth(total)) << total << " ";
   }

Source/CTest/cmCTestRunTest.h

@@ -13,13 +13,12 @@
 
 #include <stddef.h>
 
+#include "cmCTest.h"
 #include "cmCTestMultiProcessHandler.h"
 #include "cmCTestTestHandler.h"
 #include "cmDuration.h"
 #include "cmProcess.h"
 
-class cmCTest;
-
 /** \class cmRunTest
  * \brief represents a single test to be run
  *
@@ -30,8 +29,13 @@ class cmCTestRunTest
 public:
   explicit cmCTestRunTest(cmCTestMultiProcessHandler& multiHandler);
 
-  void SetNumberOfRuns(int n) { this->NumberOfRunsLeft = n; }
-  void SetRunUntilFailOn() { this->RunUntilFail = true; }
+  void SetNumberOfRuns(int n)
+  {
+    this->NumberOfRunsLeft = n;
+    this->NumberOfRunsTotal = n;
+  }
+
+  void SetRerunMode(cmCTest::Rerun r) { this->RerunMode = r; }
   void SetTestProperties(cmCTestTestHandler::cmCTestTestProperties* prop)
   {
     this->TestProperties = prop;
@@ -129,8 +133,9 @@ private:
   std::vector<std::map<
     std::string, std::vector<cmCTestMultiProcessHandler::HardwareAllocation>>>
     AllocatedHardware;
-  bool RunUntilFail = false; // default to run the test once
+  cmCTest::Rerun RerunMode = cmCTest::Rerun::Never;
   int NumberOfRunsLeft = 1;  // default to 1 run of the test
+  int NumberOfRunsTotal = 1; // default to 1 run of the test
   bool RunAgain = false;     // default to not having to run again
   size_t TotalNumberOfTests;
 };

Source/cmCTest.cxx

@@ -84,7 +84,7 @@ struct cmCTest::Private
   };
 
   int RepeatTests = 1; // default to run each test once
-  bool RepeatUntilFail = false;
+  cmCTest::Rerun RerunMode = cmCTest::Rerun::Never;
   std::string ConfigType;
   std::string ScheduleType;
   std::chrono::system_clock::time_point StopTime;
@@ -1839,11 +1839,16 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
     this->SetParallelLevel(plevel);
     this->Impl->ParallelLevelSetInCli = true;
   }
+
   if (this->CheckArgument(arg, "--repeat-until-fail")) {
     if (i >= args.size() - 1) {
       errormsg = "'--repeat-until-fail' requires an argument";
       return false;
     }
+    if (this->Impl->RerunMode != cmCTest::Rerun::Never) {
+      errormsg = "At most one '--repeat-*' option may be used.";
+      return false;
+    }
     i++;
     long repeat = 1;
     if (!cmStrToLong(args[i], &repeat)) {
@@ -1853,7 +1858,29 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
     }
     this->Impl->RepeatTests = static_cast<int>(repeat);
     if (repeat > 1) {
-      this->Impl->RepeatUntilFail = true;
+      this->Impl->RerunMode = cmCTest::Rerun::UntilFail;
+    }
+  }
+
+  if (this->CheckArgument(arg, "--repeat-until-pass")) {
+    if (i >= args.size() - 1) {
+      errormsg = "'--repeat-until-pass' requires an argument";
+      return false;
+    }
+    if (this->Impl->RerunMode != cmCTest::Rerun::Never) {
+      errormsg = "At most one '--repeat-*' option may be used.";
+      return false;
+    }
+    i++;
+    long repeat = 1;
+    if (!cmStrToLong(args[i], &repeat)) {
+      errormsg =
+        "'--repeat-until-pass' given non-integer value '" + args[i] + "'";
+      return false;
+    }
+    this->Impl->RepeatTests = static_cast<int>(repeat);
+    if (repeat > 1) {
+      this->Impl->RerunMode = cmCTest::Rerun::UntilPass;
     }
   }
 
@@ -2852,9 +2879,9 @@ int cmCTest::GetTestRepeat() const
   return this->Impl->RepeatTests;
 }
 
-bool cmCTest::GetRepeatUntilFail() const
+cmCTest::Rerun cmCTest::GetRerunMode() const
 {
-  return this->Impl->RepeatUntilFail;
+  return this->Impl->RerunMode;
 }
 
 void cmCTest::SetBuildID(const std::string& id)

Source/cmCTest.h

@@ -433,8 +433,13 @@ public:
   /** Return the number of times a test should be run */
   int GetTestRepeat() const;
 
-  /** Return true if test should run until fail */
-  bool GetRepeatUntilFail() const;
+  enum class Rerun
+  {
+    Never,
+    UntilFail,
+    UntilPass,
+  };
+  Rerun GetRerunMode() const;
 
   void GenerateSubprojectsOutput(cmXMLWriter& xml);
   std::vector<std::string> GetLabelsForSubprojects();

Source/ctest.cxx

@@ -99,8 +99,9 @@ static const char* cmDocumentationOptions[][2] = {
   { "-U, --union", "Take the Union of -I and -R" },
   { "--rerun-failed", "Run only the tests that failed previously" },
   { "--repeat-until-fail <n>",
-    "Require each test to run <n> "
-    "times without failing in order to pass" },
+    "Require each test to run <n> times without failing in order to pass" },
+  { "--repeat-until-pass <n>",
+    "Allow each test to run up to <n> times in order to pass" },
   { "--max-width <width>", "Set the max width for a test name to output" },
   { "--interactive-debug-mode [0|1]", "Set the interactive mode to 0 or 1." },
   { "--hardware-spec-file <file>", "Set the hardware spec file to use." },

Tests/RunCMake/CTestCommandLine/RunCMakeTest.cmake

@@ -4,6 +4,16 @@ set(RunCMake_TEST_TIMEOUT 60)
 unset(ENV{CTEST_PARALLEL_LEVEL})
 unset(ENV{CTEST_OUTPUT_ON_FAILURE})
 
+run_cmake_command(repeat-until-pass-bad1
+  ${CMAKE_CTEST_COMMAND} --repeat-until-pass
+  )
+run_cmake_command(repeat-until-pass-bad2
+  ${CMAKE_CTEST_COMMAND} --repeat-until-pass foo
+  )
+run_cmake_command(repeat-until-pass-good
+  ${CMAKE_CTEST_COMMAND} --repeat-until-pass 2
+  )
+
 run_cmake_command(repeat-until-fail-bad1
   ${CMAKE_CTEST_COMMAND} --repeat-until-fail
   )
@@ -14,14 +24,29 @@ run_cmake_command(repeat-until-fail-good
   ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2
   )
 
-function(run_repeat_until_fail_tests)
+run_cmake_command(repeat-until-pass-and-fail
+  ${CMAKE_CTEST_COMMAND} --repeat-until-pass 2 --repeat-until-fail 2
+  )
+run_cmake_command(repeat-until-fail-and-pass
+  ${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-until-pass 2
+  )
+
+function(run_repeat_until_pass_tests)
   # Use a single build tree for a few tests without cleaning.
-  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-fail-build)
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-pass-build)
+  run_cmake(repeat-until-pass-cmake)
   set(RunCMake_TEST_NO_CLEAN 1)
-  file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
-  file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
+  run_cmake_command(repeat-until-pass-ctest
+    ${CMAKE_CTEST_COMMAND} -C Debug --repeat-until-pass 3
+    )
+endfunction()
+run_repeat_until_pass_tests()
 
+function(run_repeat_until_fail_tests)
+  # Use a single build tree for a few tests without cleaning.
+  set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-fail-build)
   run_cmake(repeat-until-fail-cmake)
+  set(RunCMake_TEST_NO_CLEAN 1)
   run_cmake_command(repeat-until-fail-ctest
     ${CMAKE_CTEST_COMMAND} -C Debug --repeat-until-fail 3
     )

Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-result.txt

+1

Tests/RunCMake/CTestCommandLine/repeat-until-fail-and-pass-stderr.txt

+^CMake Error: At most one '--repeat-\*' option may be used\.$

Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-result.txt

+1

Tests/RunCMake/CTestCommandLine/repeat-until-pass-and-fail-stderr.txt

+^CMake Error: At most one '--repeat-\*' option may be used\.$

Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-result.txt

+1

Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad1-stderr.txt

+^CMake Error: '--repeat-until-pass' requires an argument$

Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-result.txt

+1

Tests/RunCMake/CTestCommandLine/repeat-until-pass-bad2-stderr.txt

+^CMake Error: '--repeat-until-pass' given non-integer value 'foo'$

Tests/RunCMake/CTestCommandLine/repeat-until-pass-cmake.cmake

+enable_testing()
+
+set(TEST_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/test_output.txt")
+add_test(NAME initialization
+  COMMAND ${CMAKE_COMMAND}
+  "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
+  -P "${CMAKE_CURRENT_SOURCE_DIR}/init.cmake")
+add_test(NAME test1
+  COMMAND ${CMAKE_COMMAND}
+  "-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
+  -P "${CMAKE_CURRENT_SOURCE_DIR}/test1-pass.cmake")
+set_tests_properties(test1 PROPERTIES DEPENDS "initialization")
+
+add_test(hello ${CMAKE_COMMAND} -E echo hello)
+add_test(goodbye ${CMAKE_COMMAND} -E echo goodbye)

Tests/RunCMake/CTestCommandLine/repeat-until-pass-ctest-stdout.txt

+^Test project .*/Tests/RunCMake/CTestCommandLine/repeat-until-pass-build
+    Start 1: initialization
+1/4 Test #1: initialization ...................   Passed +[0-9.]+ sec
+    Start 2: test1
+2/4 Test #2: test1 ............................\*\*\*Failed +[0-9.]+ sec
+    Start 2: test1
+    Test #2: test1 ............................   Passed +[0-9.]+ sec
+    Start 3: hello
+3/4 Test #3: hello ............................   Passed +[0-9.]+ sec
+    Start 4: goodbye
+4/4 Test #4: goodbye ..........................   Passed +[0-9.]+ sec
+
+100% tests passed, 0 tests failed out of 4
+
+Total Test time \(real\) = +[0-9.]+ sec$

Tests/RunCMake/CTestCommandLine/repeat-until-pass-good-stderr.txt

+^No tests were found!!!$

Tests/RunCMake/CTestCommandLine/test1-pass.cmake

+# This is run by test test1 in repeat-until-pass-cmake.cmake with cmake -P.
+# It reads the file TEST_OUTPUT_FILE and increments the number
+# found in the file by 1.  Unless the number is 2, then the
+# code sends out a cmake error causing the test to pass only on
+# the second time it is run.
+message("TEST_OUTPUT_FILE = ${TEST_OUTPUT_FILE}")
+file(READ "${TEST_OUTPUT_FILE}" COUNT)
+message("COUNT= ${COUNT}")
+math(EXPR COUNT "${COUNT} + 1")
+file(WRITE "${TEST_OUTPUT_FILE}" "${COUNT}")
+if(NOT COUNT EQUAL 2)
+  message(FATAL_ERROR "this test passes only on the 2nd run")
+endif()