Commit c1b2b7c0 authored by Craig Scott's avatar Craig Scott
Browse files

Add ctest options for limiting which tests fixtures add

The new options allow the user to restrict the setup and cleanup tests
automatically added for fixtures.
parent 02d11868
......@@ -13,6 +13,9 @@ Perform the :ref:`CTest MemCheck Step` as a :ref:`Dashboard Client`.
[INCLUDE <include-regex>]
[EXCLUDE_LABEL <label-exclude-regex>]
[INCLUDE_LABEL <label-include-regex>]
[EXCLUDE_FIXTURE <regex>]
[EXCLUDE_FIXTURE_SETUP <regex>]
[EXCLUDE_FIXTURE_CLEANUP <regex>]
[PARALLEL_LEVEL <level>]
[TEST_LOAD <threshold>]
[SCHEDULE_RANDOM <ON|OFF>]
......
......@@ -13,6 +13,9 @@ Perform the :ref:`CTest Test Step` as a :ref:`Dashboard Client`.
[INCLUDE <include-regex>]
[EXCLUDE_LABEL <label-exclude-regex>]
[INCLUDE_LABEL <label-include-regex>]
[EXCLUDE_FIXTURE <regex>]
[EXCLUDE_FIXTURE_SETUP <regex>]
[EXCLUDE_FIXTURE_CLEANUP <regex>]
[PARALLEL_LEVEL <level>]
[TEST_LOAD <threshold>]
[SCHEDULE_RANDOM <ON|OFF>]
......@@ -61,6 +64,20 @@ The options are:
Specify a regular expression matching test labels to include.
Tests not matching this expression are excluded.
``EXCLUDE_FIXTURE <regex>``
If a test in the set of tests to be executed requires a particular fixture,
that fixture's setup and cleanup tests would normally be added to the test
set automatically. This option prevents adding setup or cleanup tests for
fixtures matching the ``<regex>``. Note that all other fixture behavior is
retained, including test dependencies and skipping tests that have fixture
setup tests that fail.
``EXCLUDE_FIXTURE_SETUP <regex>``
Same as ``EXCLUDE_FIXTURE`` except only matching setup tests are excluded.
``EXCLUDE_FIXTURE_CLEANUP <regex>``
Same as ``EXCLUDE_FIXTURE`` except only matching cleanup tests are excluded.
``PARALLEL_LEVEL <level>``
Specify a positive number representing the number of tests to
be run in parallel.
......
......@@ -117,6 +117,23 @@ Options
This option tells ctest to NOT run the tests whose labels match the
given regular expression.
``-FA <regex>, --fixture-exclude-any <regex>``
Exclude fixtures matching ``<regex>`` from automatically adding any tests to
the test set.
If a test in the set of tests to be executed requires a particular fixture,
that fixture's setup and cleanup tests would normally be added to the test set
automatically. This option prevents adding setup or cleanup tests for fixtures
matching the ``<regex>``. Note that all other fixture behavior is retained,
including test dependencies and skipping tests that have fixture setup tests
that fail.
``-FS <regex>, --fixture-exclude-setup <regex>``
Same as ``-FA`` except only matching setup tests are excluded.
``-FC <regex>, --fixture-exclude-cleanup <regex>``
Same as ``-FA`` except only matching cleanup tests are excluded.
``-D <dashboard>, --dashboard <dashboard>``
Execute dashboard test.
......
excludeFixtures
---------------
* The :manual:`ctest(1)` executable gained new options which allow the
developer to disable automatically adding tests to the test set to satisfy
fixture dependencies. ``-FS`` prevents adding setup tests for fixtures
matching the provided regular expression, ``-FC`` prevents adding cleanup
tests for matching fixtures and ``-FA`` prevents adding any test for matching
fixtures.
......@@ -20,6 +20,9 @@ cmCTestTestCommand::cmCTestTestCommand()
this->Arguments[ctt_INCLUDE] = "INCLUDE";
this->Arguments[ctt_EXCLUDE_LABEL] = "EXCLUDE_LABEL";
this->Arguments[ctt_INCLUDE_LABEL] = "INCLUDE_LABEL";
this->Arguments[ctt_EXCLUDE_FIXTURE] = "EXCLUDE_FIXTURE";
this->Arguments[ctt_EXCLUDE_FIXTURE_SETUP] = "EXCLUDE_FIXTURE_SETUP";
this->Arguments[ctt_EXCLUDE_FIXTURE_CLEANUP] = "EXCLUDE_FIXTURE_CLEANUP";
this->Arguments[ctt_PARALLEL_LEVEL] = "PARALLEL_LEVEL";
this->Arguments[ctt_SCHEDULE_RANDOM] = "SCHEDULE_RANDOM";
this->Arguments[ctt_STOP_TIME] = "STOP_TIME";
......@@ -76,6 +79,18 @@ cmCTestGenericHandler* cmCTestTestCommand::InitializeHandler()
handler->SetOption("LabelRegularExpression",
this->Values[ctt_INCLUDE_LABEL]);
}
if (this->Values[ctt_EXCLUDE_FIXTURE]) {
handler->SetOption("ExcludeFixtureRegularExpression",
this->Values[ctt_EXCLUDE_FIXTURE]);
}
if (this->Values[ctt_EXCLUDE_FIXTURE_SETUP]) {
handler->SetOption("ExcludeFixtureSetupRegularExpression",
this->Values[ctt_EXCLUDE_FIXTURE_SETUP]);
}
if (this->Values[ctt_EXCLUDE_FIXTURE_CLEANUP]) {
handler->SetOption("ExcludeFixtureCleanupRegularExpression",
this->Values[ctt_EXCLUDE_FIXTURE_CLEANUP]);
}
if (this->Values[ctt_PARALLEL_LEVEL]) {
handler->SetOption("ParallelLevel", this->Values[ctt_PARALLEL_LEVEL]);
}
......
......@@ -53,6 +53,9 @@ protected:
ctt_INCLUDE,
ctt_EXCLUDE_LABEL,
ctt_INCLUDE_LABEL,
ctt_EXCLUDE_FIXTURE,
ctt_EXCLUDE_FIXTURE_SETUP,
ctt_EXCLUDE_FIXTURE_CLEANUP,
ctt_PARALLEL_LEVEL,
ctt_SCHEDULE_RANDOM,
ctt_STOP_TIME,
......
......@@ -362,6 +362,9 @@ void cmCTestTestHandler::Initialize()
this->ExcludeLabelRegularExpression = "";
this->IncludeRegExp = "";
this->ExcludeRegExp = "";
this->ExcludeFixtureRegExp.clear();
this->ExcludeFixtureSetupRegExp.clear();
this->ExcludeFixtureCleanupRegExp.clear();
TestsToRunString = "";
this->UseUnion = false;
......@@ -439,6 +442,18 @@ int cmCTestTestHandler::ProcessHandler()
this->UseExcludeRegExp();
this->SetExcludeRegExp(val);
}
val = this->GetOption("ExcludeFixtureRegularExpression");
if (val) {
this->ExcludeFixtureRegExp = val;
}
val = this->GetOption("ExcludeFixtureSetupRegularExpression");
if (val) {
this->ExcludeFixtureSetupRegExp = val;
}
val = this->GetOption("ExcludeFixtureCleanupRegularExpression");
if (val) {
this->ExcludeFixtureCleanupRegExp = val;
}
this->SetRerunFailed(cmSystemTools::IsOn(this->GetOption("RerunFailed")));
this->TestResults.clear();
......@@ -828,13 +843,35 @@ void cmCTestTestHandler::UpdateForFixtures(ListOfTests& tests) const
"Updating test list for fixtures" << std::endl,
this->Quiet);
// Prepare regular expression evaluators
std::string setupRegExp(this->ExcludeFixtureRegExp);
std::string cleanupRegExp(this->ExcludeFixtureRegExp);
if (!this->ExcludeFixtureSetupRegExp.empty()) {
if (setupRegExp.empty()) {
setupRegExp = this->ExcludeFixtureSetupRegExp;
} else {
setupRegExp.append("(" + setupRegExp + ")|(" +
this->ExcludeFixtureSetupRegExp + ")");
}
}
if (!this->ExcludeFixtureCleanupRegExp.empty()) {
if (cleanupRegExp.empty()) {
cleanupRegExp = this->ExcludeFixtureCleanupRegExp;
} else {
cleanupRegExp.append("(" + cleanupRegExp + ")|(" +
this->ExcludeFixtureCleanupRegExp + ")");
}
}
cmsys::RegularExpression excludeSetupRegex(setupRegExp);
cmsys::RegularExpression excludeCleanupRegex(cleanupRegExp);
// Prepare some maps to help us find setup and cleanup tests for
// any given fixture
typedef ListOfTests::const_iterator TestIterator;
typedef std::multimap<std::string, TestIterator> FixtureDependencies;
typedef FixtureDependencies::const_iterator FixtureDepsIterator;
FixtureDependencies fixtureSetups;
FixtureDependencies fixtureDeps;
FixtureDependencies fixtureCleanups;
for (ListOfTests::const_iterator it = this->TestList.begin();
it != this->TestList.end(); ++it) {
......@@ -844,13 +881,12 @@ void cmCTestTestHandler::UpdateForFixtures(ListOfTests& tests) const
for (std::set<std::string>::const_iterator depsIt = setups.begin();
depsIt != setups.end(); ++depsIt) {
fixtureSetups.insert(std::make_pair(*depsIt, it));
fixtureDeps.insert(std::make_pair(*depsIt, it));
}
const std::set<std::string>& cleanups = p.FixturesCleanup;
for (std::set<std::string>::const_iterator depsIt = cleanups.begin();
depsIt != cleanups.end(); ++depsIt) {
fixtureDeps.insert(std::make_pair(*depsIt, it));
fixtureCleanups.insert(std::make_pair(*depsIt, it));
}
}
......@@ -924,34 +960,72 @@ void cmCTestTestHandler::UpdateForFixtures(ListOfTests& tests) const
// added from a previously checked test). A fixture isn't required
// to have setup/cleanup tests.
if (!addedFixtures.insert(requiredFixtureName).second) {
// Already added this fixture
// Already seen this fixture, no need to check it again
continue;
}
std::pair<FixtureDepsIterator, FixtureDepsIterator> fixtureRange =
fixtureDeps.equal_range(requiredFixtureName);
for (FixtureDepsIterator it = fixtureRange.first;
it != fixtureRange.second; ++it) {
ListOfTests::const_iterator lotIt = it->second;
const cmCTestTestProperties& p = *lotIt;
if (!addedTests.insert(p.Name).second) {
// Already have p in our test list
continue;
// Only add setup tests if this fixture has not been excluded
if (setupRegExp.empty() ||
!excludeSetupRegex.find(requiredFixtureName)) {
std::pair<FixtureDepsIterator, FixtureDepsIterator> fixtureRange =
fixtureSetups.equal_range(requiredFixtureName);
for (FixtureDepsIterator it = fixtureRange.first;
it != fixtureRange.second; ++it) {
ListOfTests::const_iterator lotIt = it->second;
const cmCTestTestProperties& p = *lotIt;
if (!addedTests.insert(p.Name).second) {
// Already have p in our test list
continue;
}
// This is a test not yet in our list, so add it and
// update its index to reflect where it was in the original
// full list of all tests (needed to track individual tests
// across ctest runs for re-run failed, etc.)
tests.push_back(p);
tests.back().Index =
1 + static_cast<int>(std::distance(this->TestList.begin(), lotIt));
++fixtureTestsAdded;
cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
"Added setup test "
<< p.Name << " required by fixture "
<< requiredFixtureName << std::endl,
this->Quiet);
}
}
// This is a test not yet in our list, so add it and
// update its index to reflect where it was in the original
// full list of all tests (needed to track individual tests
// across ctest runs for re-run failed, etc.)
tests.push_back(p);
tests.back().Index =
1 + static_cast<int>(std::distance(this->TestList.begin(), lotIt));
++fixtureTestsAdded;
cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT, "Added test "
<< p.Name << " required by fixture "
<< requiredFixtureName << std::endl,
this->Quiet);
// Only add cleanup tests if this fixture has not been excluded
if (cleanupRegExp.empty() ||
!excludeCleanupRegex.find(requiredFixtureName)) {
std::pair<FixtureDepsIterator, FixtureDepsIterator> fixtureRange =
fixtureCleanups.equal_range(requiredFixtureName);
for (FixtureDepsIterator it = fixtureRange.first;
it != fixtureRange.second; ++it) {
ListOfTests::const_iterator lotIt = it->second;
const cmCTestTestProperties& p = *lotIt;
if (!addedTests.insert(p.Name).second) {
// Already have p in our test list
continue;
}
// This is a test not yet in our list, so add it and
// update its index to reflect where it was in the original
// full list of all tests (needed to track individual tests
// across ctest runs for re-run failed, etc.)
tests.push_back(p);
tests.back().Index =
1 + static_cast<int>(std::distance(this->TestList.begin(), lotIt));
++fixtureTestsAdded;
cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
"Added cleanup test "
<< p.Name << " required by fixture "
<< requiredFixtureName << std::endl,
this->Quiet);
}
}
}
......
......@@ -280,6 +280,9 @@ private:
std::string ExcludeLabelRegExp;
std::string IncludeRegExp;
std::string ExcludeRegExp;
std::string ExcludeFixtureRegExp;
std::string ExcludeFixtureSetupRegExp;
std::string ExcludeFixtureCleanupRegExp;
cmsys::RegularExpression IncludeLabelRegularExpression;
cmsys::RegularExpression ExcludeLabelRegularExpression;
cmsys::RegularExpression IncludeTestsRegularExpression;
......
......@@ -1899,6 +1899,34 @@ bool cmCTest::HandleCommandLineArguments(size_t& i,
->SetPersistentOption("ExcludeRegularExpression", args[i].c_str());
}
if (this->CheckArgument(arg, "-FA", "--fixture-exclude-any") &&
i < args.size() - 1) {
i++;
this->GetHandler("test")->SetPersistentOption(
"ExcludeFixtureRegularExpression", args[i].c_str());
this->GetHandler("memcheck")
->SetPersistentOption("ExcludeFixtureRegularExpression",
args[i].c_str());
}
if (this->CheckArgument(arg, "-FS", "--fixture-exclude-setup") &&
i < args.size() - 1) {
i++;
this->GetHandler("test")->SetPersistentOption(
"ExcludeFixtureSetupRegularExpression", args[i].c_str());
this->GetHandler("memcheck")
->SetPersistentOption("ExcludeFixtureSetupRegularExpression",
args[i].c_str());
}
if (this->CheckArgument(arg, "-FC", "--fixture-exclude-cleanup") &&
i < args.size() - 1) {
i++;
this->GetHandler("test")->SetPersistentOption(
"ExcludeFixtureCleanupRegularExpression", args[i].c_str());
this->GetHandler("memcheck")
->SetPersistentOption("ExcludeFixtureCleanupRegularExpression",
args[i].c_str());
}
if (this->CheckArgument(arg, "--rerun-failed")) {
this->GetHandler("test")->SetPersistentOption("RerunFailed", "true");
this->GetHandler("memcheck")->SetPersistentOption("RerunFailed", "true");
......
......@@ -52,6 +52,18 @@ static const char* cmDocumentationOptions[][2] = {
"expression." },
{ "-LE <regex>, --label-exclude <regex>", "Exclude tests with labels "
"matching regular expression." },
{ "-FA <regex>, --fixture-exclude-any <regex>", "Do not automatically "
"add any tests for "
"fixtures matching "
"regular expression." },
{ "-FS <regex>, --fixture-exclude-setup <regex>", "Do not automatically "
"add setup tests for "
"fixtures matching "
"regular expression." },
{ "-FC <regex>, --fixture-exclude-cleanup <regex>", "Do not automatically "
"add cleanup tests for "
"fixtures matching "
"regular expression." },
{ "-D <dashboard>, --dashboard <dashboard>", "Execute dashboard test" },
{ "-D <var>:<type>=<value>", "Define a variable for script mode" },
{ "-M <model>, --test-model <model>", "Sets the model for a dashboard" },
......
cmake_minimum_required (VERSION 3.6.2)
cmake_minimum_required (VERSION 3.8.0)
project(ctest_fixtures LANGUAGES NONE)
include(CTest)
......
......@@ -19,6 +19,41 @@ run_ctest_test(setupFoo INCLUDE setupFoo)
run_ctest_test(wontRun INCLUDE wontRun)
run_ctest_test(unused INCLUDE Unused)
run_ctest_test(exclude_setup_foo
INCLUDE "one|two"
EXCLUDE_FIXTURE_SETUP "Foo"
)
run_ctest_test(exclude_setup_bar
INCLUDE "one|two"
EXCLUDE_FIXTURE_SETUP "Bar"
)
run_ctest_test(exclude_cleanup_foo
INCLUDE "one|two"
EXCLUDE_FIXTURE_CLEANUP "Foo"
)
run_ctest_test(exclude_cleanup_bar
INCLUDE "one|two"
EXCLUDE_FIXTURE_CLEANUP "Bar"
)
run_ctest_test(exclude_any_foo
INCLUDE "one|two"
EXCLUDE_FIXTURE "Foo"
)
run_ctest_test(exclude_any_bar
INCLUDE "one|two"
EXCLUDE_FIXTURE "Bar"
)
run_ctest_test(exclude_any_foobar
INCLUDE "one|two"
EXCLUDE_FIXTURE "Foo|Bar"
)
#------------------------------------------------------------
# CMake configure will fail due to cyclic test dependencies
#------------------------------------------------------------
......@@ -35,3 +70,18 @@ set(CASE_CMAKELISTS_CYCLIC_CODE [[
FIXTURES_REQUIRED "Foo")
]])
run_ctest(cyclicCleanup)
#------------------------------------------------------------
# Repeat some of the exclusion tests with ctest command line
# options instead of arguments to ctest_test(). This verifies
# that the command line options make it through as well.
#------------------------------------------------------------
unset(CASE_CMAKELISTS_CYCLIC_CODE)
set(CASE_CTEST_FIXTURES_ARGS "")
run_ctest(exclude_setup_foo -R "one|two" -FS Foo)
run_ctest(exclude_setup_foo -R "one|two" --fixture-exclude-setup Foo)
run_ctest(exclude_cleanup_foo -R "one|two" -FC Foo)
run_ctest(exclude_cleanup_foo -R "one|two" --fixture-exclude-cleanup Foo)
run_ctest(exclude_any_foo -R "one|two" -FA Foo)
run_ctest(exclude_any_foo -R "one|two" --fixture-exclude-any Foo)
Test project .*/Tests/RunCMake/ctest_fixtures/exclude_any_bar-build
Start 3: setupFoo
1/5 Test #3: setupFoo +\.+ +Passed +[0-9.]+ sec
Start 2: setupBoth
2/5 Test #2: setupBoth +\.+ +Passed +[0-9.]+ sec
Start 1: one
3/5 Test #1: one +\.+ +Passed +[0-9.]+ sec
Start 5: cleanupFoo
4/5 Test #5: cleanupFoo +\.+ +Passed +[0-9.]+ sec
Start 6: two
5/5 Test #6: two +\.+ +Passed +[0-9.]+ sec
+
100% tests passed, 0 tests failed out of 5
+
Total Test time \(real\) = +[0-9.]+ sec$
Test project .*/Tests/RunCMake/ctest_fixtures/exclude_any_foo-build
Start 2: setupBoth
1/4 Test #2: setupBoth +\.+ +Passed +[0-9.]+ sec
Start 1: one
2/4 Test #1: one +\.+ +Passed +[0-9.]+ sec
Start 6: two
3/4 Test #6: two +\.+ +Passed +[0-9.]+ sec
Start 7: cleanupBar
4/4 Test #7: cleanupBar +\.+ +Passed +[0-9.]+ sec
+
100% tests passed, 0 tests failed out of 4
+
Total Test time \(real\) = +[0-9.]+ sec$
Test project .*/Tests/RunCMake/ctest_fixtures/exclude_any_foobar-build
Start 1: one
1/2 Test #1: one +\.+ +Passed +[0-9.]+ sec
Start 6: two
2/2 Test #6: two +\.+ +Passed +[0-9.]+ sec
+
100% tests passed, 0 tests failed out of 2
+
Total Test time \(real\) = +[0-9.]+ sec$
Test project .*/Tests/RunCMake/ctest_fixtures/exclude_cleanup_bar-build
Start 3: setupFoo
1/5 Test #3: setupFoo +\.+ +Passed +[0-9.]+ sec
Start 2: setupBoth
2/5 Test #2: setupBoth +\.+ +Passed +[0-9.]+ sec
Start 1: one
3/5 Test #1: one +\.+ +Passed +[0-9.]+ sec
Start 5: cleanupFoo
4/5 Test #5: cleanupFoo +\.+ +Passed +[0-9.]+ sec
Start 6: two
5/5 Test #6: two +\.+ +Passed +[0-9.]+ sec
+
100% tests passed, 0 tests failed out of 5
+
Total Test time \(real\) = +[0-9.]+ sec$
Test project .*/Tests/RunCMake/ctest_fixtures/exclude_cleanup_foo-build
Start 3: setupFoo
1/5 Test #3: setupFoo +\.+ +Passed +[0-9.]+ sec
Start 2: setupBoth
2/5 Test #2: setupBoth +\.+ +Passed +[0-9.]+ sec
Start 1: one
3/5 Test #1: one +\.+ +Passed +[0-9.]+ sec
Start 6: two
4/5 Test #6: two +\.+ +Passed +[0-9.]+ sec
Start 7: cleanupBar
5/5 Test #7: cleanupBar +\.+ +Passed +[0-9.]+ sec
+
100% tests passed, 0 tests failed out of 5
+
Total Test time \(real\) = +[0-9.]+ sec$
Test project .*/Tests/RunCMake/ctest_fixtures/exclude_setup_bar-build
Start 3: setupFoo
1/6 Test #3: setupFoo +\.+ +Passed +[0-9.]+ sec
Start 2: setupBoth
2/6 Test #2: setupBoth +\.+ +Passed +[0-9.]+ sec
Start 1: one
3/6 Test #1: one +\.+ +Passed +[0-9.]+ sec
Start 5: cleanupFoo
4/6 Test #5: cleanupFoo +\.+ +Passed +[0-9.]+ sec
Start 6: two
5/6 Test #6: two +\.+ +Passed +[0-9.]+ sec
Start 7: cleanupBar
6/6 Test #7: cleanupBar +\.+ +Passed +[0-9.]+ sec
+
100% tests passed, 0 tests failed out of 6
+
Total Test time \(real\) = +[0-9.]+ sec$
Test project .*/Tests/RunCMake/ctest_fixtures/exclude_setup_foo-build
Start 2: setupBoth
1/5 Test #2: setupBoth +\.+ +Passed +[0-9.]+ sec
Start 1: one
2/5 Test #1: one +\.+ +Passed +[0-9.]+ sec
Start 5: cleanupFoo
3/5 Test #5: cleanupFoo +\.+ +Passed +[0-9.]+ sec
Start 6: two
4/5 Test #6: two +\.+ +Passed +[0-9.]+ sec
Start 7: cleanupBar
5/5 Test #7: cleanupBar +\.+ +Passed +[0-9.]+ sec
+
100% tests passed, 0 tests failed out of 5
+
Total Test time \(real\) = +[0-9.]+ sec$
cmake_minimum_required(VERSION 3.6.2)
cmake_minimum_required(VERSION 3.8.0)
set(CTEST_SITE "test-site")
set(CTEST_BUILD_NAME "test-build-name")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment