Commit 28994115 authored by Brad King's avatar Brad King

ctest_test: Add option to REPEAT tests

parent 42d5d8f4
......@@ -23,6 +23,7 @@ Perform the :ref:`CTest Test Step` as a :ref:`Dashboard Client`.
[STOP_TIME <time-of-day>]
[RETURN_VALUE <result-var>]
[CAPTURE_CMAKE_ERROR <result-var>]
[REPEAT <mode>:<n>]
[QUIET]
)
......@@ -95,6 +96,25 @@ The options are:
and then the ``--test-load`` command-line argument to :manual:`ctest(1)`.
See also the ``TestLoad`` setting in the :ref:`CTest Test Step`.
``REPEAT <mode>:<n>``
Run tests repeatedly based on the given ``<mode>`` up to ``<n>`` times.
The modes are:
``UNTIL_FAIL``
Require each test to run ``<n>`` times without failing in order to pass.
This is useful in finding sporadic failures in test cases.
``UNTIL_PASS``
Allow each test to run up to ``<n>`` times in order to pass.
Repeats tests if they fail for any reason.
This is useful in tolerating sporadic failures in test cases.
``AFTER_TIMEOUT``
Allow each test to run up to ``<n>`` times in order to pass.
Repeats tests only if they timeout.
This is useful in tolerating sporadic timeouts in test cases
on busy machines.
``SCHEDULE_RANDOM <ON|OFF>``
Launch tests in a random order. This may be useful for detecting
implicit test dependencies.
......
......@@ -4,3 +4,6 @@ ctest-repeat-until-pass
* The :manual:`ctest(1)` tool learned new ``--repeat-until-pass <n>``
and ``--repeat-after-timeout <n>`` options to help tolerate sporadic
test failures.
* The :command:`ctest_test` command gained a ``REPEAT <mode>:<n>`` option
to specify conditions in which to repeat tests.
......@@ -29,6 +29,7 @@ void cmCTestTestCommand::BindArguments()
this->Bind("EXCLUDE_FIXTURE_SETUP"_s, this->ExcludeFixtureSetup);
this->Bind("EXCLUDE_FIXTURE_CLEANUP"_s, this->ExcludeFixtureCleanup);
this->Bind("PARALLEL_LEVEL"_s, this->ParallelLevel);
this->Bind("REPEAT"_s, this->Repeat);
this->Bind("SCHEDULE_RANDOM"_s, this->ScheduleRandom);
this->Bind("STOP_TIME"_s, this->StopTime);
this->Bind("TEST_LOAD"_s, this->TestLoad);
......@@ -85,6 +86,9 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler()
if (!this->ParallelLevel.empty()) {
handler->SetOption("ParallelLevel", this->ParallelLevel.c_str());
}
if (!this->Repeat.empty()) {
handler->SetOption("Repeat", this->Repeat.c_str());
}
if (!this->ScheduleRandom.empty()) {
handler->SetOption("ScheduleRandom", this->ScheduleRandom.c_str());
}
......
......@@ -55,6 +55,7 @@ protected:
std::string ExcludeFixtureSetup;
std::string ExcludeFixtureCleanup;
std::string ParallelLevel;
std::string Repeat;
std::string ScheduleRandom;
std::string StopTime;
std::string TestLoad;
......
......@@ -471,6 +471,30 @@ bool cmCTestTestHandler::ProcessOptions()
if (cmIsOn(this->GetOption("ScheduleRandom"))) {
this->CTest->SetScheduleType("Random");
}
if (const char* repeat = this->GetOption("Repeat")) {
cmsys::RegularExpression repeatRegex(
"^(UNTIL_FAIL|UNTIL_PASS|AFTER_TIMEOUT):([0-9]+)$");
if (repeatRegex.find(repeat)) {
std::string const& count = repeatRegex.match(2);
unsigned long n = 1;
cmStrToULong(count, &n); // regex guarantees success
this->RepeatCount = static_cast<int>(n);
if (this->RepeatCount > 1) {
std::string const& mode = repeatRegex.match(1);
if (mode == "UNTIL_FAIL") {
this->RepeatMode = cmCTest::Repeat::UntilFail;
} else if (mode == "UNTIL_PASS") {
this->RepeatMode = cmCTest::Repeat::UntilPass;
} else if (mode == "AFTER_TIMEOUT") {
this->RepeatMode = cmCTest::Repeat::AfterTimeout;
}
}
} else {
cmCTestLog(this->CTest, ERROR_MESSAGE,
"Repeat option invalid value: " << repeat << std::endl);
return false;
}
}
if (this->GetOption("ParallelLevel")) {
this->CTest->SetParallelLevel(atoi(this->GetOption("ParallelLevel")));
}
......@@ -1231,8 +1255,12 @@ void cmCTestTestHandler::ProcessDirectory(std::vector<std::string>& passed,
parallel->SetCTest(this->CTest);
parallel->SetParallelLevel(this->CTest->GetParallelLevel());
parallel->SetTestHandler(this);
parallel->SetRepeatMode(this->CTest->GetRepeatMode(),
this->CTest->GetRepeatCount());
if (this->RepeatMode != cmCTest::Repeat::Never) {
parallel->SetRepeatMode(this->RepeatMode, this->RepeatCount);
} else {
parallel->SetRepeatMode(this->CTest->GetRepeatMode(),
this->CTest->GetRepeatCount());
}
parallel->SetQuiet(this->Quiet);
if (this->TestLoad > 0) {
parallel->SetTestLoad(this->TestLoad);
......
......@@ -18,12 +18,12 @@
#include "cmsys/RegularExpression.hxx"
#include "cmCTest.h"
#include "cmCTestGenericHandler.h"
#include "cmCTestResourceSpec.h"
#include "cmDuration.h"
#include "cmListFileCache.h"
class cmCTest;
class cmMakefile;
class cmXMLWriter;
......@@ -353,6 +353,8 @@ private:
std::ostream* LogFile;
cmCTest::Repeat RepeatMode = cmCTest::Repeat::Never;
int RepeatCount = 1;
bool RerunFailed;
};
......
include(RunCTest)
set(RunCMake_TEST_TIMEOUT 60)
unset(ENV{CTEST_PARALLEL_LEVEL})
unset(ENV{CTEST_OUTPUT_ON_FAILURE})
set(CASE_CTEST_TEST_ARGS "")
set(CASE_CTEST_TEST_LOAD "")
......@@ -71,7 +74,24 @@ add_test(NAME PassingTest COMMAND ${CMAKE_COMMAND} -E echo PassingTestOutput)
add_test(NAME FailingTest COMMAND ${CMAKE_COMMAND} -E no_such_command)
]])
unset(ENV{CTEST_PARALLEL_LEVEL})
run_ctest(TestOutputSize)
endfunction()
run_TestOutputSize()
run_ctest_test(TestRepeatBad1 REPEAT UNKNOWN:3)
run_ctest_test(TestRepeatBad2 REPEAT UNTIL_FAIL:-1)
function(run_TestRepeat case)
set(CASE_CTEST_TEST_ARGS EXCLUDE RunCMakeVersion ${ARGN})
string(CONCAT CASE_CMAKELISTS_SUFFIX_CODE [[
add_test(NAME testRepeat
COMMAND ${CMAKE_COMMAND} -D COUNT_FILE=${CMAKE_CURRENT_BINARY_DIR}/count.cmake
-P "]] "${RunCMake_SOURCE_DIR}/TestRepeat${case}" [[.cmake")
set_property(TEST testRepeat PROPERTY TIMEOUT 5)
]])
run_ctest(TestRepeat${case})
endfunction()
run_TestRepeat(UntilFail REPEAT UNTIL_FAIL:3)
run_TestRepeat(UntilPass REPEAT UNTIL_PASS:3)
run_TestRepeat(AfterTimeout REPEAT AFTER_TIMEOUT:3)
Test project [^
]*/Tests/RunCMake/ctest_test/TestRepeatAfterTimeout-build
Start 1: testRepeat
1/1 Test #1: testRepeat .......................\*\*\*Timeout +[0-9.]+ sec
Start 1: testRepeat
Test #1: testRepeat ....................... Passed +[0-9.]+ sec
+
100% tests passed, 0 tests failed out of 1
+
Total Test time \(real\) = +[0-9.]+ sec$
include("${COUNT_FILE}" OPTIONAL)
if(NOT COUNT)
set(COUNT 0)
endif()
math(EXPR COUNT "${COUNT} + 1")
file(WRITE "${COUNT_FILE}" "set(COUNT ${COUNT})\n")
if(NOT COUNT EQUAL 2)
message("this test times out except on the 2nd run")
execute_process(COMMAND ${CMAKE_COMMAND} -E sleep 10)
endif()
Repeat option invalid value: UNKNOWN:3
Repeat option invalid value: UNTIL_FAIL:-1
Test project [^
]*/Tests/RunCMake/ctest_test/TestRepeatUntilFail-build
Start 1: testRepeat
Test #1: testRepeat ....................... Passed +[0-9.]+ sec
Start 1: testRepeat
Test #1: testRepeat .......................\*\*\*Failed +[0-9.]+ sec
+
0% tests passed, 1 tests failed out of 1
+
Total Test time \(real\) = +[0-9.]+ sec
+
The following tests FAILED:
[ ]+1 - testRepeat \(Failed\)$
include("${COUNT_FILE}" OPTIONAL)
if(NOT COUNT)
set(COUNT 0)
endif()
math(EXPR COUNT "${COUNT} + 1")
file(WRITE "${COUNT_FILE}" "set(COUNT ${COUNT})\n")
if(COUNT EQUAL 2)
message(FATAL_ERROR "this test fails on the 2nd run")
endif()
Test project [^
]*/Tests/RunCMake/ctest_test/TestRepeatUntilPass-build
Start 1: testRepeat
1/1 Test #1: testRepeat .......................\*\*\*Failed +[0-9.]+ sec
Start 1: testRepeat
Test #1: testRepeat ....................... Passed +[0-9.]+ sec
+
100% tests passed, 0 tests failed out of 1
+
Total Test time \(real\) = +[0-9.]+ sec$
include("${COUNT_FILE}" OPTIONAL)
if(NOT COUNT)
set(COUNT 0)
endif()
math(EXPR COUNT "${COUNT} + 1")
file(WRITE "${COUNT_FILE}" "set(COUNT ${COUNT})\n")
if(NOT COUNT EQUAL 2)
message(FATAL_ERROR "this test passes only on the 2nd run")
endif()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment