Commit 9e9787f1 authored by Craig Scott's avatar Craig Scott Committed by Kitware Robot

Merge topic 'ctest-repeat'

32c165d2 CTest: Consolidate '--repeat-* n' options as `--repeat *:n`
Acked-by: Kitware Robot's avatarKitware Robot <kwrobot@kitware.com>
Merge-request: !4053
parents 3520208c 32c165d2
Pipeline #151889 passed with stage
in 0 seconds
......@@ -261,23 +261,27 @@ Options
fail, subsequent calls to CTest with the ``--rerun-failed`` option will run
the set of tests that most recently failed (if any).
``--repeat-until-fail <n>``
Require each test to run ``<n>`` times without failing in order to pass.
This is useful in finding sporadic failures in test cases.
``--repeat <mode>:<n>``
Run tests repeatedly based on the given ``<mode>`` up to ``<n>`` times.
The modes are:
``--repeat-until-pass <n>``
Allow each test to run up to ``<n>`` times in order to pass.
Repeats tests if they fail for any reason.
``until-fail``
Require each test to run ``<n>`` times without failing in order to pass.
This is useful in finding sporadic failures in test cases.
This is useful in tolerating sporadic failures in test cases.
``until-pass``
Allow each test to run up to ``<n>`` times in order to pass.
Repeats tests if they fail for any reason.
This is useful in tolerating sporadic failures in test cases.
``--repeat-after-timeout <n>``
Allow each test to run up to ``<n>`` times in order to pass.
Repeats tests only if they timeout.
``after-timeout``
Allow each test to run up to ``<n>`` times in order to pass.
Repeats tests only if they timeout.
This is useful in tolerating sporadic timeouts in test cases
on busy machines.
This is useful in tolerating sporadic timeouts in test cases
on busy machines.
``--repeat-until-fail <n>``
Equivalent to ``--repeat until-fail:<n>``.
``--max-width <width>``
Set the max width for a test name to output.
......
ctest-repeat-until-pass
-----------------------
ctest-repeat
------------
* The :manual:`ctest(1)` tool learned new ``--repeat-until-pass <n>``
and ``--repeat-after-timeout <n>`` options to help tolerate sporadic
test failures.
* The :manual:`ctest(1)` tool gained a ``--repeat <mode>:<n>`` option
to specify conditions in which to repeat tests. This generalizes
the existing ``--repeat-until-fail <n>`` option to add modes for
``until-pass`` and ``after-timeout``.
* The :command:`ctest_test` command gained a ``REPEAT <mode>:<n>`` option
to specify conditions in which to repeat tests.
......@@ -21,6 +21,7 @@
#include "cmsys/FStream.hxx"
#include "cmsys/Glob.hxx"
#include "cmsys/Process.h"
#include "cmsys/RegularExpression.hxx"
#include "cmsys/SystemInformation.hxx"
#include "cm_curl.h"
......@@ -1846,7 +1847,7 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
return false;
}
if (this->Impl->RepeatMode != cmCTest::Repeat::Never) {
errormsg = "At most one '--repeat-*' option may be used.";
errormsg = "At most one '--repeat' option may be used.";
return false;
}
i++;
......@@ -1862,48 +1863,37 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
}
}
if (this->CheckArgument(arg, "--repeat-until-pass")) {
if (this->CheckArgument(arg, "--repeat")) {
if (i >= args.size() - 1) {
errormsg = "'--repeat-until-pass' requires an argument";
errormsg = "'--repeat' requires an argument";
return false;
}
if (this->Impl->RepeatMode != cmCTest::Repeat::Never) {
errormsg = "At most one '--repeat-*' option may be used.";
errormsg = "At most one '--repeat' option may be used.";
return false;
}
i++;
long repeat = 1;
if (!cmStrToLong(args[i], &repeat)) {
errormsg =
"'--repeat-until-pass' given non-integer value '" + args[i] + "'";
return false;
}
this->Impl->RepeatCount = static_cast<int>(repeat);
if (repeat > 1) {
this->Impl->RepeatMode = cmCTest::Repeat::UntilPass;
}
}
if (this->CheckArgument(arg, "--repeat-after-timeout")) {
if (i >= args.size() - 1) {
errormsg = "'--repeat-after-timeout' requires an argument";
return false;
}
if (this->Impl->RepeatMode != cmCTest::Repeat::Never) {
errormsg = "At most one '--repeat-*' option may be used.";
return false;
}
i++;
long repeat = 1;
if (!cmStrToLong(args[i], &repeat)) {
errormsg =
"'--repeat-after-timeout' given non-integer value '" + args[i] + "'";
cmsys::RegularExpression repeatRegex(
"^(until-fail|until-pass|after-timeout):([0-9]+)$");
if (repeatRegex.find(args[i])) {
std::string const& count = repeatRegex.match(2);
unsigned long n = 1;
cmStrToULong(count, &n); // regex guarantees success
this->Impl->RepeatCount = static_cast<int>(n);
if (this->Impl->RepeatCount > 1) {
std::string const& mode = repeatRegex.match(1);
if (mode == "until-fail") {
this->Impl->RepeatMode = cmCTest::Repeat::UntilFail;
} else if (mode == "until-pass") {
this->Impl->RepeatMode = cmCTest::Repeat::UntilPass;
} else if (mode == "after-timeout") {
this->Impl->RepeatMode = cmCTest::Repeat::AfterTimeout;
}
}
} else {
errormsg = "'--repeat' given invalid value '" + args[i] + "'";
return false;
}
this->Impl->RepeatCount = static_cast<int>(repeat);
if (repeat > 1) {
this->Impl->RepeatMode = cmCTest::Repeat::AfterTimeout;
}
}
if (this->CheckArgument(arg, "--test-load") && i < args.size() - 1) {
......
......@@ -98,11 +98,11 @@ static const char* cmDocumentationOptions[][2] = {
"Run a specific number of tests by number." },
{ "-U, --union", "Take the Union of -I and -R" },
{ "--rerun-failed", "Run only the tests that failed previously" },
{ "--repeat-until-fail <n>",
{ "--repeat until-fail:<n>, --repeat-until-fail <n>",
"Require each test to run <n> times without failing in order to pass" },
{ "--repeat-until-pass <n>",
{ "--repeat until-pass:<n>",
"Allow each test to run up to <n> times in order to pass" },
{ "--repeat-after-timeout <n>",
{ "--repeat after-timeout:<n>",
"Allow each test to run up to <n> times if it times out" },
{ "--max-width <width>", "Set the max width for a test name to output" },
{ "--interactive-debug-mode [0|1]", "Set the interactive mode to 0 or 1." },
......
......@@ -4,14 +4,26 @@ set(RunCMake_TEST_TIMEOUT 60)
unset(ENV{CTEST_PARALLEL_LEVEL})
unset(ENV{CTEST_OUTPUT_ON_FAILURE})
run_cmake_command(repeat-until-pass-bad1
${CMAKE_CTEST_COMMAND} --repeat-until-pass
run_cmake_command(repeat-opt-bad1
${CMAKE_CTEST_COMMAND} --repeat until-pass
)
run_cmake_command(repeat-until-pass-bad2
${CMAKE_CTEST_COMMAND} --repeat-until-pass foo
run_cmake_command(repeat-opt-bad2
${CMAKE_CTEST_COMMAND} --repeat until-pass:foo
)
run_cmake_command(repeat-until-pass-good
${CMAKE_CTEST_COMMAND} --repeat-until-pass 2
run_cmake_command(repeat-opt-bad3
${CMAKE_CTEST_COMMAND} --repeat until-fail:2 --repeat-until-fail 2
)
run_cmake_command(repeat-opt-bad4
${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat until-fail:2
)
run_cmake_command(repeat-opt-until-pass
${CMAKE_CTEST_COMMAND} --repeat until-pass:2
)
run_cmake_command(repeat-opt-until-fail
${CMAKE_CTEST_COMMAND} --repeat until-fail:2
)
run_cmake_command(repeat-opt-after-timeout
${CMAKE_CTEST_COMMAND} --repeat after-timeout:2
)
run_cmake_command(repeat-until-fail-bad1
......@@ -24,33 +36,13 @@ run_cmake_command(repeat-until-fail-good
${CMAKE_CTEST_COMMAND} --repeat-until-fail 2
)
run_cmake_command(repeat-after-timeout-bad1
${CMAKE_CTEST_COMMAND} --repeat-after-timeout
)
run_cmake_command(repeat-after-timeout-bad2
${CMAKE_CTEST_COMMAND} --repeat-after-timeout foo
)
run_cmake_command(repeat-after-timeout-good
${CMAKE_CTEST_COMMAND} --repeat-after-timeout 2
)
run_cmake_command(repeat-until-pass-and-fail
${CMAKE_CTEST_COMMAND} --repeat-until-pass 2 --repeat-until-fail 2
)
run_cmake_command(repeat-until-fail-and-pass
${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-until-pass 2
)
run_cmake_command(repeat-until-fail-and-timeout
${CMAKE_CTEST_COMMAND} --repeat-until-fail 2 --repeat-after-timeout 2
)
function(run_repeat_until_pass_tests)
# Use a single build tree for a few tests without cleaning.
set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/repeat-until-pass-build)
run_cmake(repeat-until-pass-cmake)
set(RunCMake_TEST_NO_CLEAN 1)
run_cmake_command(repeat-until-pass-ctest
${CMAKE_CTEST_COMMAND} -C Debug --repeat-until-pass 3
${CMAKE_CTEST_COMMAND} -C Debug --repeat until-pass:3
)
endfunction()
run_repeat_until_pass_tests()
......@@ -61,7 +53,7 @@ function(run_repeat_after_timeout_tests)
run_cmake(repeat-after-timeout-cmake)
set(RunCMake_TEST_NO_CLEAN 1)
run_cmake_command(repeat-after-timeout-ctest
${CMAKE_CTEST_COMMAND} -C Debug --repeat-after-timeout 3
${CMAKE_CTEST_COMMAND} -C Debug --repeat after-timeout:3
)
endfunction()
run_repeat_after_timeout_tests()
......@@ -72,10 +64,11 @@ function(run_repeat_until_fail_tests)
run_cmake(repeat-until-fail-cmake)
set(RunCMake_TEST_NO_CLEAN 1)
run_cmake_command(repeat-until-fail-ctest
${CMAKE_CTEST_COMMAND} -C Debug --repeat-until-fail 3
${CMAKE_CTEST_COMMAND} -C Debug ${ARGN}
)
endfunction()
run_repeat_until_fail_tests()
run_repeat_until_fail_tests(--repeat-until-fail 3)
run_repeat_until_fail_tests(--repeat until-fail:3)
function(run_BadCTestTestfile)
set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/BadCTestTestfile)
......
^CMake Error: '--repeat-after-timeout' requires an argument$
^CMake Error: '--repeat-after-timeout' given non-integer value 'foo'$
^CMake Error: '--repeat' given invalid value 'until-pass'$
^CMake Error: '--repeat' given invalid value 'until-pass:foo'$
^CMake Error: At most one '--repeat' option may be used\.$
^CMake Error: At most one '--repeat' option may be used\.$
^CMake Error: At most one '--repeat-\*' option may be used\.$
^CMake Error: At most one '--repeat-\*' option may be used\.$
^CMake Error: At most one '--repeat-\*' option may be used\.$
^CMake Error: '--repeat-until-pass' requires an argument$
^CMake Error: '--repeat-until-pass' given non-integer value 'foo'$
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment