Commit 6660926f authored by Brad King's avatar Brad King Committed by Kitware Robot

Merge topic 'ctest-repeat-until-pass'

39ac8b4e ctest: Add --repeat-after-timeout option
80c2c9d1 ctest: Add --repeat-until-pass option
0187e522 cmCTestRunTest: Use inline member initializers
Acked-by: Kitware Robot's avatarKitware Robot <kwrobot@kitware.com>
Merge-request: !3960
parents b42da462 39ac8b4e
Pipeline #149840 passed with stage
in 0 seconds
......@@ -266,6 +266,19 @@ Options
This is useful in finding sporadic failures in test cases.
``--repeat-until-pass <n>``
Allow each test to run up to ``<n>`` times in order to pass.
Repeats tests if they fail for any reason.
This is useful in tolerating sporadic failures in test cases.
``--repeat-after-timeout <n>``
Allow each test to run up to ``<n>`` times in order to pass.
Repeats tests only if they timeout.
This is useful in tolerating sporadic timeouts in test cases
on busy machines.
``--max-width <width>``
Set the max width for a test name to output.
......
ctest-repeat-until-pass
-----------------------
* The :manual:`ctest(1)` tool learned new ``--repeat-until-pass <n>``
and ``--repeat-after-timeout <n>`` options to help tolerate sporadic
test failures.
......@@ -171,8 +171,8 @@ bool cmCTestMultiProcessHandler::StartTestProcess(int test)
this->RunningCount += GetProcessorsUsed(test);
cmCTestRunTest* testRun = new cmCTestRunTest(*this);
if (this->CTest->GetRepeatUntilFail()) {
testRun->SetRunUntilFailOn();
if (this->CTest->GetRerunMode() != cmCTest::Rerun::Never) {
testRun->SetRerunMode(this->CTest->GetRerunMode());
testRun->SetNumberOfRuns(this->CTest->GetTestRepeat());
}
testRun->SetIndex(test);
......
......@@ -34,9 +34,6 @@ cmCTestRunTest::cmCTestRunTest(cmCTestMultiProcessHandler& multiHandler)
this->TestResult.Status = cmCTestTestHandler::NOT_RUN;
this->TestResult.TestCount = 0;
this->TestResult.Properties = nullptr;
this->NumberOfRunsLeft = 1; // default to 1 run of the test
this->RunUntilFail = false; // default to run the test once
this->RunAgain = false; // default to not having to run again
}
void cmCTestRunTest::CheckOutput(std::string const& line)
......@@ -343,10 +340,14 @@ bool cmCTestRunTest::NeedsToRerun()
return false;
}
// if number of runs left is not 0, and we are running until
// we find a failed test, then return true so the test can be
// we find a failed (or passed) test, then return true so the test can be
// restarted
if (this->RunUntilFail &&
this->TestResult.Status == cmCTestTestHandler::COMPLETED) {
if ((this->RerunMode == cmCTest::Rerun::UntilFail &&
this->TestResult.Status == cmCTestTestHandler::COMPLETED) ||
(this->RerunMode == cmCTest::Rerun::UntilPass &&
this->TestResult.Status != cmCTestTestHandler::COMPLETED) ||
(this->RerunMode == cmCTest::Rerun::AfterTimeout &&
this->TestResult.Status == cmCTestTestHandler::TIMEOUT)) {
this->RunAgain = true;
return true;
}
......@@ -746,7 +747,12 @@ void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total)
// then it will never print out the completed / total, same would
// got for run until pass. Trick is when this is called we don't
// yet know if we are passing or failing.
if (this->NumberOfRunsLeft == 1 || this->CTest->GetTestProgressOutput()) {
bool const progressOnLast =
(this->RerunMode != cmCTest::Rerun::UntilPass &&
this->RerunMode != cmCTest::Rerun::AfterTimeout);
if ((progressOnLast && this->NumberOfRunsLeft == 1) ||
(!progressOnLast && this->NumberOfRunsLeft == this->NumberOfRunsTotal) ||
this->CTest->GetTestProgressOutput()) {
outputStream << std::setw(getNumWidth(total)) << completed << "/";
outputStream << std::setw(getNumWidth(total)) << total << " ";
}
......
......@@ -13,13 +13,12 @@
#include <stddef.h>
#include "cmCTest.h"
#include "cmCTestMultiProcessHandler.h"
#include "cmCTestTestHandler.h"
#include "cmDuration.h"
#include "cmProcess.h"
class cmCTest;
/** \class cmRunTest
* \brief represents a single test to be run
*
......@@ -30,8 +29,13 @@ class cmCTestRunTest
public:
explicit cmCTestRunTest(cmCTestMultiProcessHandler& multiHandler);
void SetNumberOfRuns(int n) { this->NumberOfRunsLeft = n; }
void SetRunUntilFailOn() { this->RunUntilFail = true; }
void SetNumberOfRuns(int n)
{
this->NumberOfRunsLeft = n;
this->NumberOfRunsTotal = n;
}
void SetRerunMode(cmCTest::Rerun r) { this->RerunMode = r; }
void SetTestProperties(cmCTestTestHandler::cmCTestTestProperties* prop)
{
this->TestProperties = prop;
......@@ -129,9 +133,10 @@ private:
std::vector<std::map<
std::string, std::vector<cmCTestMultiProcessHandler::HardwareAllocation>>>
AllocatedHardware;
bool RunUntilFail;
int NumberOfRunsLeft;
bool RunAgain;
cmCTest::Rerun RerunMode = cmCTest::Rerun::Never;
int NumberOfRunsLeft = 1; // default to 1 run of the test
int NumberOfRunsTotal = 1; // default to 1 run of the test
bool RunAgain = false; // default to not having to run again
size_t TotalNumberOfTests;
};
......
......@@ -84,7 +84,7 @@ struct cmCTest::Private
};
int RepeatTests = 1; // default to run each test once
bool RepeatUntilFail = false;
cmCTest::Rerun RerunMode = cmCTest::Rerun::Never;
std::string ConfigType;
std::string ScheduleType;
std::chrono::system_clock::time_point StopTime;
......@@ -1839,11 +1839,16 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
this->SetParallelLevel(plevel);
this->Impl->ParallelLevelSetInCli = true;
}
if (this->CheckArgument(arg, "--repeat-until-fail")) {
if (i >= args.size() - 1) {
errormsg = "'--repeat-until-fail' requires an argument";
return false;
}
if (this->Impl->RerunMode != cmCTest::Rerun::Never) {
errormsg = "At most one '--repeat-*' option may be used.";
return false;
}
i++;
long repeat = 1;
if (!cmStrToLong(args[i], &repeat)) {
......@@ -1853,7 +1858,51 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
}
this->Impl->RepeatTests = static_cast<int>(repeat);
if (repeat > 1) {
this->Impl->RepeatUntilFail = true;
this->Impl->RerunMode = cmCTest::Rerun::UntilFail;
}
}
if (this->CheckArgument(arg, "--repeat-until-pass")) {
if (i >= args.size() - 1) {
errormsg = "'--repeat-until-pass' requires an argument";
return false;
}
if (this->Impl->RerunMode != cmCTest::Rerun::Never) {
errormsg = "At most one '--repeat-*' option may be used.";
return false;
}
i++;
long repeat = 1;
if (!cmStrToLong(args[i], &repeat)) {
errormsg =
"'--repeat-until-pass' given non-integer value '" + args[i] + "'";
return false;
}
this->Impl->RepeatTests = static_cast<int>(repeat);
if (repeat > 1) {
this->Impl->RerunMode = cmCTest::Rerun::UntilPass;
}
}
if (this->CheckArgument(arg, "--repeat-after-timeout")) {
if (i >= args.size() - 1) {
errormsg = "'--repeat-after-timeout' requires an argument";
return false;
}
if (this->Impl->RerunMode != cmCTest::Rerun::Never) {
errormsg = "At most one '--repeat-*' option may be used.";
return false;
}
i++;
long repeat = 1;
if (!cmStrToLong(args[i], &repeat)) {
errormsg =
"'--repeat-after-timeout' given non-integer value '" + args[i] + "'";
return false;
}
this->Impl->RepeatTests = static_cast<int>(repeat);
if (repeat > 1) {
this->Impl->RerunMode = cmCTest::Rerun::AfterTimeout;
}
}
......@@ -2852,9 +2901,9 @@ int cmCTest::GetTestRepeat() const
return this->Impl->RepeatTests;
}
bool cmCTest::GetRepeatUntilFail() const
cmCTest::Rerun cmCTest::GetRerunMode() const
{
return this->Impl->RepeatUntilFail;
return this->Impl->RerunMode;
}
void cmCTest::SetBuildID(const std::string& id)
......
......@@ -433,8 +433,14 @@ public:
/** Return the number of times a test should be run */
int GetTestRepeat() const;
/** Return true if test should run until fail */
bool GetRepeatUntilFail() const;
enum class Rerun
{
Never,
UntilFail,
UntilPass,
AfterTimeout,
};
Rerun GetRerunMode() const;
void GenerateSubprojectsOutput(cmXMLWriter& xml);
std::vector<std::string> GetLabelsForSubprojects();
......
......@@ -99,8 +99,11 @@ static const char* cmDocumentationOptions[][2] = {
{ "-U, --union", "Take the Union of -I and -R" },
{ "--rerun-failed", "Run only the tests that failed previously" },
{ "--repeat-until-fail <n>",
"Require each test to run <n> "
"times without failing in order to pass" },
"Require each test to run <n> times without failing in order to pass" },
{ "--repeat-until-pass <n>",
"Allow each test to run up to <n> times in order to pass" },
{ "--repeat-after-timeout <n>",
"Allow each test to run up to <n> times if it times out" },
{ "--max-width <width>", "Set the max width for a test name to output" },
{ "--interactive-debug-mode [0|1]", "Set the interactive mode to 0 or 1." },
{ "--hardware-spec-file <file>", "Set the hardware spec file to use." },
......
......@@ -4,6 +4,16 @@ set(RunCMake_TEST_TIMEOUT 60)
unset(ENV{CTEST_PARALLEL_LEVEL})
unset(ENV{CTEST_OUTPUT_ON_FAILURE})
run_cmake_command(repeat-until-pass-bad1
${CMAKE_CTEST_COMMAND} --repeat-until-pass
)
run_cmake_command(repeat-until-pass-bad2
${CMAKE_CTEST_COMMAND} --repeat-until-pass foo
)
run_cmake_command(repeat-until-pass-good
${CMAKE_CTEST_COMMAND} --repeat-until-pass 2
)
run_cmake_command(repeat-until-fail-bad1
${CMAKE_CTEST_COMMAND} --repeat-until-fail
)
......@@ -14,14 +24,53 @@ run_cmake_command(repeat-until-fail-good
${CMAKE_CTEST_COMMAND} --repeat-until-fail 2
)
function(run_repeat_until_fail_tests)
run_cmake_command(repeat-after-timeout-bad1
${CMAKE_CTEST_COMMAND} --repeat-after-timeout
)
run_cmake_command(repeat-after-timeout-bad2
${CMAKE_CTEST_COMMAND} --repeat-after-timeout foo
)
run_cmake_command(repeat-after-timeout-good
${CMAKE_CTEST_COMMAND} --repeat-after-timeout 2
)
run_cmake_command(repeat-until-pass-and-fail
${CMAKE_CTEST_COMMAND} --repeat-until-pass 2 --repeat-until-fail 2
)
run_cmake_command(repeat-until-fail-and-pass
${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-until-pass 2
)
run_cmake_command(repeat-until-fail-and-timeout
${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-after-timeout 2
)
function(run_repeat_until_pass_tests)
# Use a single build tree for a few tests without cleaning.
set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-fail-build)
set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-pass-build)
run_cmake(repeat-until-pass-cmake)
set(RunCMake_TEST_NO_CLEAN 1)
file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
run_cmake_command(repeat-until-pass-ctest
${CMAKE_CTEST_COMMAND} -C Debug --repeat-until-pass 3
)
endfunction()
run_repeat_until_pass_tests()
function(run_repeat_after_timeout_tests)
# Use a single build tree for a few tests without cleaning.
set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-after-timeout-build)
run_cmake(repeat-after-timeout-cmake)
set(RunCMake_TEST_NO_CLEAN 1)
run_cmake_command(repeat-after-timeout-ctest
${CMAKE_CTEST_COMMAND} -C Debug --repeat-after-timeout 3
)
endfunction()
run_repeat_after_timeout_tests()
function(run_repeat_until_fail_tests)
# Use a single build tree for a few tests without cleaning.
set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-fail-build)
run_cmake(repeat-until-fail-cmake)
set(RunCMake_TEST_NO_CLEAN 1)
run_cmake_command(repeat-until-fail-ctest
${CMAKE_CTEST_COMMAND} -C Debug --repeat-until-fail 3
)
......
^CMake Error: '--repeat-after-timeout' requires an argument$
^CMake Error: '--repeat-after-timeout' given non-integer value 'foo'$
enable_testing()
set(TEST_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/test_output.txt")
add_test(NAME initialization
COMMAND ${CMAKE_COMMAND}
"-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
-P "${CMAKE_CURRENT_SOURCE_DIR}/init.cmake")
add_test(NAME test1
COMMAND ${CMAKE_COMMAND}
"-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
-P "${CMAKE_CURRENT_SOURCE_DIR}/test1-timeout.cmake")
set_tests_properties(test1 PROPERTIES DEPENDS "initialization" TIMEOUT 2)
add_test(hello ${CMAKE_COMMAND} -E echo hello)
add_test(goodbye ${CMAKE_COMMAND} -E echo goodbye)
^Test project .*/Tests/RunCMake/CTestCommandLine/repeat-after-timeout-build
Start 1: initialization
1/4 Test #1: initialization ................... Passed +[0-9.]+ sec
Start 2: test1
2/4 Test #2: test1 ............................\*\*\*Timeout +[0-9.]+ sec
Start 2: test1
Test #2: test1 ............................ Passed +[0-9.]+ sec
Start 3: hello
3/4 Test #3: hello ............................ Passed +[0-9.]+ sec
Start 4: goodbye
4/4 Test #4: goodbye .......................... Passed +[0-9.]+ sec
100% tests passed, 0 tests failed out of 4
Total Test time \(real\) = +[0-9.]+ sec$
^CMake Error: At most one '--repeat-\*' option may be used\.$
^CMake Error: At most one '--repeat-\*' option may be used\.$
^CMake Error: At most one '--repeat-\*' option may be used\.$
^CMake Error: '--repeat-until-pass' requires an argument$
^CMake Error: '--repeat-until-pass' given non-integer value 'foo'$
enable_testing()
set(TEST_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/test_output.txt")
add_test(NAME initialization
COMMAND ${CMAKE_COMMAND}
"-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
-P "${CMAKE_CURRENT_SOURCE_DIR}/init.cmake")
add_test(NAME test1
COMMAND ${CMAKE_COMMAND}
"-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
-P "${CMAKE_CURRENT_SOURCE_DIR}/test1-pass.cmake")
set_tests_properties(test1 PROPERTIES DEPENDS "initialization")
add_test(hello ${CMAKE_COMMAND} -E echo hello)
add_test(goodbye ${CMAKE_COMMAND} -E echo goodbye)
^Test project .*/Tests/RunCMake/CTestCommandLine/repeat-until-pass-build
Start 1: initialization
1/4 Test #1: initialization ................... Passed +[0-9.]+ sec
Start 2: test1
2/4 Test #2: test1 ............................\*\*\*Failed +[0-9.]+ sec
Start 2: test1
Test #2: test1 ............................ Passed +[0-9.]+ sec
Start 3: hello
3/4 Test #3: hello ............................ Passed +[0-9.]+ sec
Start 4: goodbye
4/4 Test #4: goodbye .......................... Passed +[0-9.]+ sec
100% tests passed, 0 tests failed out of 4
Total Test time \(real\) = +[0-9.]+ sec$
# This is run by test test1 in repeat-until-pass-cmake.cmake with cmake -P.
# It reads the file TEST_OUTPUT_FILE and increments the number
# found in the file by 1. Unless the number is 2, then the
# code sends out a cmake error causing the test to pass only on
# the second time it is run.
message("TEST_OUTPUT_FILE = ${TEST_OUTPUT_FILE}")
file(READ "${TEST_OUTPUT_FILE}" COUNT)
message("COUNT= ${COUNT}")
math(EXPR COUNT "${COUNT} + 1")
file(WRITE "${TEST_OUTPUT_FILE}" "${COUNT}")
if(NOT COUNT EQUAL 2)
message(FATAL_ERROR "this test passes only on the 2nd run")
endif()
# This is run by test test1 in repeat-after-timeout-cmake.cmake with cmake -P.
# It reads the file TEST_OUTPUT_FILE and increments the number
# found in the file by 1. Unless the number is 2, then the
# code sends out a cmake error causing the test to not timeout only on
# the second time it is run.
message("TEST_OUTPUT_FILE = ${TEST_OUTPUT_FILE}")
file(READ "${TEST_OUTPUT_FILE}" COUNT)
message("COUNT= ${COUNT}")
math(EXPR COUNT "${COUNT} + 1")
file(WRITE "${TEST_OUTPUT_FILE}" "${COUNT}")
if(NOT COUNT EQUAL 2)
message("this test times out except on the 2nd run")
execute_process(COMMAND ${CMAKE_COMMAND} -E sleep 10)
endif()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment