ctest_test: Add option to REPEAT tests

Help/command/ctest_test.rst

@@ -23,6 +23,7 @@ Perform the :ref:`CTest Test Step` as a :ref:`Dashboard Client`.
              [STOP_TIME <time-of-day>]
              [RETURN_VALUE <result-var>]
              [CAPTURE_CMAKE_ERROR <result-var>]
+             [REPEAT <mode>:<n>]
              [QUIET]
              )
 
@@ -95,6 +96,25 @@ The options are:
   and then the ``--test-load`` command-line argument to :manual:`ctest(1)`.
   See also the ``TestLoad`` setting in the :ref:`CTest Test Step`.
 
+``REPEAT <mode>:<n>``
+  Run tests repeatedly based on the given ``<mode>`` up to ``<n>`` times.
+  The modes are:
+
+  ``UNTIL_FAIL``
+    Require each test to run ``<n>`` times without failing in order to pass.
+    This is useful in finding sporadic failures in test cases.
+
+  ``UNTIL_PASS``
+    Allow each test to run up to ``<n>`` times in order to pass.
+    Repeats tests if they fail for any reason.
+    This is useful in tolerating sporadic failures in test cases.
+
+  ``AFTER_TIMEOUT``
+    Allow each test to run up to ``<n>`` times in order to pass.
+    Repeats tests only if they timeout.
+    This is useful in tolerating sporadic timeouts in test cases
+    on busy machines.
+
 ``SCHEDULE_RANDOM <ON|OFF>``
   Launch tests in a random order.  This may be useful for detecting
   implicit test dependencies.

Help/release/dev/ctest-repeat-until-pass.rst

@@ -4,3 +4,6 @@ ctest-repeat-until-pass
 * The :manual:`ctest(1)` tool learned new ``--repeat-until-pass <n>``
   and ``--repeat-after-timeout <n>`` options to help tolerate sporadic
   test failures.
+
+* The :command:`ctest_test` command gained a ``REPEAT <mode>:<n>`` option
+  to specify conditions in which to repeat tests.

Source/CTest/cmCTestTestCommand.cxx

@@ -29,6 +29,7 @@ void cmCTestTestCommand::BindArguments()
   this->Bind("EXCLUDE_FIXTURE_SETUP"_s, this->ExcludeFixtureSetup);
   this->Bind("EXCLUDE_FIXTURE_CLEANUP"_s, this->ExcludeFixtureCleanup);
   this->Bind("PARALLEL_LEVEL"_s, this->ParallelLevel);
+  this->Bind("REPEAT"_s, this->Repeat);
   this->Bind("SCHEDULE_RANDOM"_s, this->ScheduleRandom);
   this->Bind("STOP_TIME"_s, this->StopTime);
   this->Bind("TEST_LOAD"_s, this->TestLoad);
@@ -85,6 +86,9 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler()
   if (!this->ParallelLevel.empty()) {
     handler->SetOption("ParallelLevel", this->ParallelLevel.c_str());
   }
+  if (!this->Repeat.empty()) {
+    handler->SetOption("Repeat", this->Repeat.c_str());
+  }
   if (!this->ScheduleRandom.empty()) {
     handler->SetOption("ScheduleRandom", this->ScheduleRandom.c_str());
   }

Source/CTest/cmCTestTestCommand.h

@@ -55,6 +55,7 @@ protected:
   std::string ExcludeFixtureSetup;
   std::string ExcludeFixtureCleanup;
   std::string ParallelLevel;
+  std::string Repeat;
   std::string ScheduleRandom;
   std::string StopTime;
   std::string TestLoad;

Source/CTest/cmCTestTestHandler.cxx

@@ -471,6 +471,30 @@ bool cmCTestTestHandler::ProcessOptions()
   if (cmIsOn(this->GetOption("ScheduleRandom"))) {
     this->CTest->SetScheduleType("Random");
   }
+  if (const char* repeat = this->GetOption("Repeat")) {
+    cmsys::RegularExpression repeatRegex(
+      "^(UNTIL_FAIL|UNTIL_PASS|AFTER_TIMEOUT):([0-9]+)$");
+    if (repeatRegex.find(repeat)) {
+      std::string const& count = repeatRegex.match(2);
+      unsigned long n = 1;
+      cmStrToULong(count, &n); // regex guarantees success
+      this->RepeatCount = static_cast<int>(n);
+      if (this->RepeatCount > 1) {
+        std::string const& mode = repeatRegex.match(1);
+        if (mode == "UNTIL_FAIL") {
+          this->RepeatMode = cmCTest::Repeat::UntilFail;
+        } else if (mode == "UNTIL_PASS") {
+          this->RepeatMode = cmCTest::Repeat::UntilPass;
+        } else if (mode == "AFTER_TIMEOUT") {
+          this->RepeatMode = cmCTest::Repeat::AfterTimeout;
+        }
+      }
+    } else {
+      cmCTestLog(this->CTest, ERROR_MESSAGE,
+                 "Repeat option invalid value: " << repeat << std::endl);
+      return false;
+    }
+  }
   if (this->GetOption("ParallelLevel")) {
     this->CTest->SetParallelLevel(atoi(this->GetOption("ParallelLevel")));
   }
@@ -1231,8 +1255,12 @@ void cmCTestTestHandler::ProcessDirectory(std::vector<std::string>& passed,
   parallel->SetCTest(this->CTest);
   parallel->SetParallelLevel(this->CTest->GetParallelLevel());
   parallel->SetTestHandler(this);
-  parallel->SetRepeatMode(this->CTest->GetRepeatMode(),
-                          this->CTest->GetRepeatCount());
+  if (this->RepeatMode != cmCTest::Repeat::Never) {
+    parallel->SetRepeatMode(this->RepeatMode, this->RepeatCount);
+  } else {
+    parallel->SetRepeatMode(this->CTest->GetRepeatMode(),
+                            this->CTest->GetRepeatCount());
+  }
   parallel->SetQuiet(this->Quiet);
   if (this->TestLoad > 0) {
     parallel->SetTestLoad(this->TestLoad);

Source/CTest/cmCTestTestHandler.h

@@ -18,12 +18,12 @@
 
 #include "cmsys/RegularExpression.hxx"
 
+#include "cmCTest.h"
 #include "cmCTestGenericHandler.h"
 #include "cmCTestResourceSpec.h"
 #include "cmDuration.h"
 #include "cmListFileCache.h"
 
-class cmCTest;
 class cmMakefile;
 class cmXMLWriter;
 
@@ -353,6 +353,8 @@ private:
 
   std::ostream* LogFile;
 
+  cmCTest::Repeat RepeatMode = cmCTest::Repeat::Never;
+  int RepeatCount = 1;
   bool RerunFailed;
 };
 

Tests/RunCMake/ctest_test/RunCMakeTest.cmake

 include(RunCTest)
 set(RunCMake_TEST_TIMEOUT 60)
 
+unset(ENV{CTEST_PARALLEL_LEVEL})
+unset(ENV{CTEST_OUTPUT_ON_FAILURE})
+
 set(CASE_CTEST_TEST_ARGS "")
 set(CASE_CTEST_TEST_LOAD "")
 
@@ -71,7 +74,24 @@ add_test(NAME PassingTest COMMAND ${CMAKE_COMMAND} -E echo PassingTestOutput)
 add_test(NAME FailingTest COMMAND ${CMAKE_COMMAND} -E no_such_command)
   ]])
 
-  unset(ENV{CTEST_PARALLEL_LEVEL})
   run_ctest(TestOutputSize)
 endfunction()
 run_TestOutputSize()
+
+run_ctest_test(TestRepeatBad1 REPEAT UNKNOWN:3)
+run_ctest_test(TestRepeatBad2 REPEAT UNTIL_FAIL:-1)
+
+function(run_TestRepeat case)
+  set(CASE_CTEST_TEST_ARGS EXCLUDE RunCMakeVersion ${ARGN})
+  string(CONCAT CASE_CMAKELISTS_SUFFIX_CODE [[
+add_test(NAME testRepeat
+  COMMAND ${CMAKE_COMMAND} -D COUNT_FILE=${CMAKE_CURRENT_BINARY_DIR}/count.cmake
+                           -P "]] "${RunCMake_SOURCE_DIR}/TestRepeat${case}" [[.cmake")
+set_property(TEST testRepeat PROPERTY TIMEOUT 5)
+  ]])
+
+  run_ctest(TestRepeat${case})
+endfunction()
+run_TestRepeat(UntilFail REPEAT UNTIL_FAIL:3)
+run_TestRepeat(UntilPass REPEAT UNTIL_PASS:3)
+run_TestRepeat(AfterTimeout REPEAT AFTER_TIMEOUT:3)

Tests/RunCMake/ctest_test/TestRepeatAfterTimeout-stdout.txt

+Test project [^
+]*/Tests/RunCMake/ctest_test/TestRepeatAfterTimeout-build
+    Start 1: testRepeat
+1/1 Test #1: testRepeat .......................\*\*\*Timeout +[0-9.]+ sec
+    Start 1: testRepeat
+    Test #1: testRepeat .......................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec$

Tests/RunCMake/ctest_test/TestRepeatAfterTimeout.cmake

+include("${COUNT_FILE}" OPTIONAL)
+if(NOT COUNT)
+  set(COUNT 0)
+endif()
+math(EXPR COUNT "${COUNT} + 1")
+file(WRITE "${COUNT_FILE}" "set(COUNT ${COUNT})\n")
+if(NOT COUNT EQUAL 2)
+  message("this test times out except on the 2nd run")
+  execute_process(COMMAND ${CMAKE_COMMAND} -E sleep 10)
+endif()

Tests/RunCMake/ctest_test/TestRepeatBad1-result.txt

+(-1|255)

Tests/RunCMake/ctest_test/TestRepeatBad1-stderr.txt

+Repeat option invalid value: UNKNOWN:3

Tests/RunCMake/ctest_test/TestRepeatBad2-result.txt

+(-1|255)

Tests/RunCMake/ctest_test/TestRepeatBad2-stderr.txt

+Repeat option invalid value: UNTIL_FAIL:-1

Tests/RunCMake/ctest_test/TestRepeatUntilFail-stdout.txt

+Test project [^
+]*/Tests/RunCMake/ctest_test/TestRepeatUntilFail-build
+    Start 1: testRepeat
+    Test #1: testRepeat .......................   Passed +[0-9.]+ sec
+    Start 1: testRepeat
+    Test #1: testRepeat .......................\*\*\*Failed +[0-9.]+ sec
++
+0% tests passed, 1 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec
++
+The following tests FAILED:
+[	 ]+1 - testRepeat \(Failed\)$

Tests/RunCMake/ctest_test/TestRepeatUntilFail.cmake

+include("${COUNT_FILE}" OPTIONAL)
+if(NOT COUNT)
+  set(COUNT 0)
+endif()
+math(EXPR COUNT "${COUNT} + 1")
+file(WRITE "${COUNT_FILE}" "set(COUNT ${COUNT})\n")
+if(COUNT EQUAL 2)
+  message(FATAL_ERROR "this test fails on the 2nd run")
+endif()

Tests/RunCMake/ctest_test/TestRepeatUntilPass-stdout.txt

+Test project [^
+]*/Tests/RunCMake/ctest_test/TestRepeatUntilPass-build
+    Start 1: testRepeat
+1/1 Test #1: testRepeat .......................\*\*\*Failed +[0-9.]+ sec
+    Start 1: testRepeat
+    Test #1: testRepeat .......................   Passed +[0-9.]+ sec
++
+100% tests passed, 0 tests failed out of 1
++
+Total Test time \(real\) = +[0-9.]+ sec$

Tests/RunCMake/ctest_test/TestRepeatUntilPass.cmake

+include("${COUNT_FILE}" OPTIONAL)
+if(NOT COUNT)
+  set(COUNT 0)
+endif()
+math(EXPR COUNT "${COUNT} + 1")
+file(WRITE "${COUNT_FILE}" "set(COUNT ${COUNT})\n")
+if(NOT COUNT EQUAL 2)
+  message(FATAL_ERROR "this test passes only on the 2nd run")
+endif()