Commit fde70a1b authored by Bill Hoffman's avatar Bill Hoffman Committed by Brad King

ctest: Add a new --repeat-until-fail option

This option tells ctest to run each test N times until the test fails or
the N times have run. This is useful for finding random failing tests.
parent 56be75b3
......@@ -194,6 +194,11 @@ Options
subsequent calls to ctest with the --rerun-failed option will run
the set of tests that most recently failed (if any).
``--repeat-until-fail <n>``
Require each test to run ``<n>`` times without failing in order to pass.
This is useful in finding sporadic failures in test cases.
``--max-width <width>``
Set the max width for a test name to output
......
......@@ -121,6 +121,11 @@ void cmCTestMultiProcessHandler::StartTestProcess(int test)
this->RunningCount += GetProcessorsUsed(test);
cmCTestRunTest* testRun = new cmCTestRunTest(this->TestHandler);
if(this->CTest->GetRepeatUntilFail())
{
testRun->SetRunUntilFailOn();
testRun->SetNumberOfRuns(this->CTest->GetTestRepeat());
}
testRun->SetIndex(test);
testRun->SetTestProperties(this->Properties[test]);
......@@ -289,7 +294,13 @@ bool cmCTestMultiProcessHandler::CheckOutput()
cmCTestRunTest* p = *i;
int test = p->GetIndex();
if(p->EndTest(this->Completed, this->Total, true))
bool testResult = p->EndTest(this->Completed, this->Total, true);
if(p->StartAgain())
{
this->Completed--; // remove the completed test because run again
continue;
}
if(testResult)
{
this->Passed->push_back(p->GetTestProperties()->Name);
}
......
......@@ -33,6 +33,9 @@ cmCTestRunTest::cmCTestRunTest(cmCTestTestHandler* handler)
this->CompressedOutput = "";
this->CompressionRatio = 2;
this->StopTimePassed = false;
this->NumberOfRunsLeft = 1; // default to 1 run of the test
this->RunUntilFail = false; // default to run the test once
this->RunAgain = false; // default to not having to run again
}
cmCTestRunTest::~cmCTestRunTest()
......@@ -357,13 +360,50 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started)
this->MemCheckPostProcess();
this->ComputeWeightedCost();
}
// Always push the current TestResult onto the
// If the test does not need to rerun push the current TestResult onto the
// TestHandler vector
this->TestHandler->TestResults.push_back(this->TestResult);
if(!this->NeedsToRerun())
{
this->TestHandler->TestResults.push_back(this->TestResult);
}
delete this->TestProcess;
return passed;
}
bool cmCTestRunTest::StartAgain()
{
if(!this->RunAgain)
{
return false;
}
this->RunAgain = false; // reset
// change to tests directory
std::string current_dir = cmSystemTools::GetCurrentWorkingDirectory();
cmSystemTools::ChangeDirectory(this->TestProperties->Directory);
this->StartTest(this->TotalNumberOfTests);
// change back
cmSystemTools::ChangeDirectory(current_dir);
return true;
}
bool cmCTestRunTest::NeedsToRerun()
{
this->NumberOfRunsLeft--;
if(this->NumberOfRunsLeft == 0)
{
return false;
}
// if number of runs left is not 0, and we are running until
// we find a failed test, then return true so the test can be
// restarted
if(this->RunUntilFail
&& this->TestResult.Status == cmCTestTestHandler::COMPLETED)
{
this->RunAgain = true;
return true;
}
return false;
}
//----------------------------------------------------------------------
void cmCTestRunTest::ComputeWeightedCost()
{
......@@ -400,6 +440,7 @@ void cmCTestRunTest::MemCheckPostProcess()
// Starts the execution of a test. Returns once it has started
bool cmCTestRunTest::StartTest(size_t total)
{
this->TotalNumberOfTests = total; // save for rerun case
cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(2*getNumWidth(total) + 8)
<< "Start "
<< std::setw(getNumWidth(this->TestHandler->GetMaxIndex()))
......@@ -494,10 +535,10 @@ bool cmCTestRunTest::StartTest(size_t total)
//----------------------------------------------------------------------
void cmCTestRunTest::ComputeArguments()
{
this->Arguments.clear(); // reset becaue this might be a rerun
std::vector<std::string>::const_iterator j =
this->TestProperties->Args.begin();
++j; // skip test name
// find the test executable
if(this->TestHandler->MemCheck)
{
......@@ -682,10 +723,28 @@ bool cmCTestRunTest::ForkProcess(double testTimeOut, bool explicitTimeout,
void cmCTestRunTest::WriteLogOutputTop(size_t completed, size_t total)
{
cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
<< completed << "/");
cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
<< total << " ");
// if this is the last or only run of this test
// then print out completed / total
// Only issue is if a test fails and we are running until fail
// then it will never print out the completed / total, same would
// got for run until pass. Trick is when this is called we don't
// yet know if we are passing or failing.
if(this->NumberOfRunsLeft == 1)
{
cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
<< completed << "/");
cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
<< total << " ");
}
// if this is one of several runs of a test just print blank space
// to keep things neat
else
{
cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
<< " " << " ");
cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(getNumWidth(total))
<< " " << " ");
}
if ( this->TestHandler->MemCheck )
{
......
......@@ -27,6 +27,8 @@ public:
cmCTestRunTest(cmCTestTestHandler* handler);
~cmCTestRunTest();
void SetNumberOfRuns(int n) {this->NumberOfRunsLeft = n;}
void SetRunUntilFailOn() { this->RunUntilFail = true;}
void SetTestProperties(cmCTestTestHandler::cmCTestTestProperties * prop)
{ this->TestProperties = prop; }
......@@ -58,7 +60,10 @@ public:
void ComputeArguments();
void ComputeWeightedCost();
bool StartAgain();
private:
bool NeedsToRerun();
void DartProcessing();
void ExeNotFound(std::string exe);
// Figures out a final timeout which is min(STOP_TIME, NOW+TIMEOUT)
......@@ -92,6 +97,10 @@ private:
std::string ActualCommand;
std::vector<std::string> Arguments;
bool StopTimePassed;
bool RunUntilFail;
int NumberOfRunsLeft;
bool RunAgain;
size_t TotalNumberOfTests;
};
inline int getNumWidth(size_t n)
......
......@@ -328,6 +328,8 @@ cmCTest::cmCTest()
this->OutputTestOutputOnTestFailure = false;
this->ComputedCompressTestOutput = false;
this->ComputedCompressMemCheckOutput = false;
this->RepeatTests = 1; // default to run each test once
this->RepeatUntilFail = false;
if(cmSystemTools::GetEnv("CTEST_OUTPUT_ON_FAILURE"))
{
this->OutputTestOutputOnTestFailure = true;
......@@ -1983,11 +1985,11 @@ bool cmCTest::CheckArgument(const std::string& arg, const char* varg1,
//----------------------------------------------------------------------
// Processes one command line argument (and its arguments if any)
// for many simple options and then returns
void cmCTest::HandleCommandLineArguments(size_t &i,
std::vector<std::string> &args)
bool cmCTest::HandleCommandLineArguments(size_t &i,
std::vector<std::string> &args,
std::string& errormsg)
{
std::string arg = args[i];
if(this->CheckArgument(arg, "-F"))
{
this->Failover = true;
......@@ -2005,6 +2007,27 @@ void cmCTest::HandleCommandLineArguments(size_t &i,
this->SetParallelLevel(plevel);
this->ParallelLevelSetInCli = true;
}
if(this->CheckArgument(arg, "--repeat-until-fail"))
{
if( i >= args.size() - 1)
{
errormsg = "'--repeat-until-fail' requires an argument";
return false;
}
i++;
long repeat = 1;
if(!cmSystemTools::StringToLong(args[i].c_str(), &repeat))
{
errormsg = "'--repeat-until-fail' given non-integer value '"
+ args[i] + "'";
return false;
}
this->RepeatTests = static_cast<int>(repeat);
if(repeat > 1)
{
this->RepeatUntilFail = true;
}
}
if(this->CheckArgument(arg, "--no-compress-output"))
{
......@@ -2190,6 +2213,7 @@ void cmCTest::HandleCommandLineArguments(size_t &i,
this->GetHandler("test")->SetPersistentOption("RerunFailed", "true");
this->GetHandler("memcheck")->SetPersistentOption("RerunFailed", "true");
}
return true;
}
//----------------------------------------------------------------------
......@@ -2272,7 +2296,12 @@ int cmCTest::Run(std::vector<std::string> &args, std::string* output)
for(size_t i=1; i < args.size(); ++i)
{
// handle the simple commandline arguments
this->HandleCommandLineArguments(i,args);
std::string errormsg;
if(!this->HandleCommandLineArguments(i,args, errormsg))
{
cmSystemTools::Error(errormsg.c_str());
return 1;
}
// handle the script arguments -S -SR -SP
this->HandleScriptArguments(i,args,SRArgumentSpecified);
......
......@@ -429,8 +429,13 @@ public:
{
return this->Definitions;
}
// return the number of times a test should be run
int GetTestRepeat() { return this->RepeatTests;}
// return true if test should run until fail
bool GetRepeatUntilFail() { return this->RepeatUntilFail;}
private:
int RepeatTests;
bool RepeatUntilFail;
std::string ConfigType;
std::string ScheduleType;
std::string StopTime;
......@@ -535,8 +540,9 @@ private:
bool AddVariableDefinition(const std::string &arg);
//! parse and process most common command line arguments
void HandleCommandLineArguments(size_t &i,
std::vector<std::string> &args);
bool HandleCommandLineArguments(size_t &i,
std::vector<std::string> &args,
std::string& errormsg);
//! hande the -S -SP and -SR arguments
void HandleScriptArguments(size_t &i,
......
......@@ -75,6 +75,8 @@ static const char * cmDocumentationOptions[][2] =
"Run a specific number of tests by number."},
{"-U, --union", "Take the Union of -I and -R"},
{"--rerun-failed", "Run only the tests that failed previously"},
{"--repeat-until-fail <n>", "Require each test to run <n> "
"times without failing in order to pass"},
{"--max-width <width>", "Set the max width for a test name to output"},
{"--interactive-debug-mode [0|1]", "Set the interactive mode to 0 or 1."},
{"--no-label-summary", "Disable timing summary information for labels."},
......
......@@ -198,6 +198,7 @@ add_RunCMake_test(CommandLine)
add_RunCMake_test(install)
add_RunCMake_test(CPackInstallProperties)
add_RunCMake_test(ExternalProject)
add_RunCMake_test(CTestCommandLine)
set(IfacePaths_INCLUDE_DIRECTORIES_ARGS -DTEST_PROP=INCLUDE_DIRECTORIES)
add_RunCMake_test(IfacePaths_INCLUDE_DIRECTORIES TEST_DIR IfacePaths)
......
cmake_minimum_required(VERSION 3.0)
project(${RunCMake_TEST} NONE)
include(${RunCMake_TEST}.cmake)
include(RunCMake)
run_cmake_command(repeat-until-fail-bad1
${CMAKE_CTEST_COMMAND} --repeat-until-fail
)
run_cmake_command(repeat-until-fail-bad2
${CMAKE_CTEST_COMMAND} --repeat-until-fail foo
)
run_cmake_command(repeat-until-fail-good
${CMAKE_CTEST_COMMAND} --repeat-until-fail 2
)
function(run_repeat_until_fail_tests)
# Use a single build tree for a few tests without cleaning.
set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-fail-build)
set(RunCMake_TEST_NO_CLEAN 1)
file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
run_cmake(repeat-until-fail-cmake)
run_cmake_command(repeat-until-fail-ctest
${CMAKE_CTEST_COMMAND} -C Debug --repeat-until-fail 3
)
endfunction()
run_repeat_until_fail_tests()
# This is run by test initialization in repeat-until-fail-cmake.cmake
# with cmake -P. It creates TEST_OUTPUT_FILE with a 0 in it.
file(WRITE "${TEST_OUTPUT_FILE}" "0")
^CMake Error: '--repeat-until-fail' requires an argument$
^CMake Error: '--repeat-until-fail' given non-integer value 'foo'$
enable_testing()
set(TEST_OUTPUT_FILE "${CMAKE_CURRENT_BINARY_DIR}/test_output.txt")
add_test(NAME initialization
COMMAND ${CMAKE_COMMAND}
"-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
-P "${CMAKE_CURRENT_SOURCE_DIR}/init.cmake")
add_test(NAME test1
COMMAND ${CMAKE_COMMAND}
"-DTEST_OUTPUT_FILE=${TEST_OUTPUT_FILE}"
-P "${CMAKE_CURRENT_SOURCE_DIR}/test1.cmake")
set_tests_properties(test1 PROPERTIES DEPENDS "initialization")
add_test(hello ${CMAKE_COMMAND} -E echo hello)
add_test(goodbye ${CMAKE_COMMAND} -E echo goodbye)
^Test project .*/Tests/RunCMake/CTestCommandLine/repeat-until-fail-build
Start 1: initialization
Test #1: initialization ................... Passed [0-9.]+ sec
Start 1: initialization
Test #1: initialization ................... Passed [0-9.]+ sec
Start 1: initialization
1/4 Test #1: initialization ................... Passed [0-9.]+ sec
Start 2: test1
Test #2: test1 ............................ Passed [0-9.]+ sec
Start 2: test1
Test #2: test1 ............................\*\*\*Failed [0-9.]+ sec
Start 3: hello
Test #3: hello ............................ Passed [0-9.]+ sec
Start 3: hello
Test #3: hello ............................ Passed [0-9.]+ sec
Start 3: hello
3/4 Test #3: hello ............................ Passed [0-9.]+ sec
Start 4: goodbye
Test #4: goodbye .......................... Passed [0-9.]+ sec
Start 4: goodbye
Test #4: goodbye .......................... Passed [0-9.]+ sec
Start 4: goodbye
4/4 Test #4: goodbye .......................... Passed [0-9.]+ sec
+
75% tests passed, 1 tests failed out of 4
+
Total Test time \(real\) = +[0-9.]+ sec
+
The following tests FAILED:
[ ]+2 - test1 \(Failed\)$
# This is run by test test1 in repeat-until-fail-cmake.cmake with cmake -P.
# It reads the file TEST_OUTPUT_FILE and increments the number
# found in the file by 1. When the number is 2, then the
# code sends out a cmake error causing the test to fail
# the second time it is run.
message("TEST_OUTPUT_FILE = ${TEST_OUTPUT_FILE}")
file(READ "${TEST_OUTPUT_FILE}" COUNT)
message("COUNT= ${COUNT}")
math(EXPR COUNT "${COUNT} + 1")
file(WRITE "${TEST_OUTPUT_FILE}" "${COUNT}")
if(${COUNT} EQUAL 2)
message(FATAL_ERROR "this test fails on the 2nd run")
endif()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment