Commit 16bb46ea authored by Kyle Edwards's avatar Kyle Edwards Committed by Kitware Robot

Merge topic 'skip_regular_expression'

5dbd9c85 Vim: Add SKIP_REGULAR_EXPRESSION test property to vim syntax file
407dd1a9 Help: Add documentation for SKIP_REGULAR_EXPRESSION test property
d7955d4e Tests: Create test for SKIP_REGULAR_EXPRESSION test property
4f1dec86 CTest: Add SKIP_REGULAR_EXPRESSION test property
Acked-by: Kitware Robot's avatarKitware Robot <kwrobot@kitware.com>
Merge-request: !3570
parents 573f1fc1 5dbd9c85
Pipeline #141720 passed with stage
in 0 seconds
......@@ -288,6 +288,7 @@ syn keyword cmakeProperty contained
\ SKIP_AUTORCC
\ SKIP_AUTOUIC
\ SKIP_BUILD_RPATH
\ SKIP_REGULAR_EXPRESSION
\ SKIP_RETURN_CODE
\ SOURCES
\ SOURCE_DIR
......@@ -1772,6 +1773,7 @@ syn keyword cmakeKWadd_test contained
\ NAME
\ OFF
\ PASS_REGULAR_EXPRESSION
\ SKIP_REGULAR_EXPRESSION
\ TARGET_FILE
\ WILL_FAIL
\ WORKING_DIRECTORY
......
......@@ -38,8 +38,9 @@ The given test command is expected to exit with code ``0`` to pass and
non-zero to fail, or vice-versa if the :prop_test:`WILL_FAIL` test
property is set. Any output written to stdout or stderr will be
captured by :manual:`ctest(1)` but does not affect the pass/fail status
unless the :prop_test:`PASS_REGULAR_EXPRESSION` or
:prop_test:`FAIL_REGULAR_EXPRESSION` test property is used.
unless the :prop_test:`PASS_REGULAR_EXPRESSION`,
:prop_test:`FAIL_REGULAR_EXPRESSION` or
:prop_test:`SKIP_REGULAR_EXPRESSION` test property is used.
The ``COMMAND`` and ``WORKING_DIRECTORY`` options may use "generator
expressions" with the syntax ``$<...>``. See the
......
......@@ -407,6 +407,7 @@ Properties on Tests
/prop_test/REQUIRED_FILES
/prop_test/RESOURCE_LOCK
/prop_test/RUN_SERIAL
/prop_test/SKIP_REGULAR_EXPRESSION
/prop_test/SKIP_RETURN_CODE
/prop_test/TIMEOUT
/prop_test/TIMEOUT_AFTER_MATCH
......
SKIP_REGULAR_EXPRESSION
-----------------------
If the output matches this regular expression the test will be marked as skipped.
If set, if the output matches one of specified regular expressions,
the test will be marked as skipped. Example:
.. code-block:: cmake
set_property(TEST mytest PROPERTY
SKIP_REGULAR_EXPRESSION "[^a-z]Skip" "SKIP" "Skipped"
)
``SKIP_REGULAR_EXPRESSION`` expects a list of regular expressions.
See also the :prop_test:`SKIP_RETURN_CODE` property.
......@@ -8,3 +8,5 @@ test are met. If such a situation should not be considered a hard failure
a return code of the process can be specified that will mark the test as
``Not Run`` if it is encountered. Valid values are in the range of
0 to 255, inclusive.
See also the :prop_test:`SKIP_REGULAR_EXPRESSION` property.
add_skip_regular_expression_test_property
-----------------------------------------
* A new test property, :prop_test:`SKIP_REGULAR_EXPRESSION`, has been added.
This property is similar to :prop_test:`FAIL_REGULAR_EXPRESSION` and
:prop_test:`PASS_REGULAR_EXPRESSION`, but with the same meaning as
:prop_test:`SKIP_RETURN_CODE`. This is useful, for example, in cases where
the user has no control over the return code of the test. For example, in
Catch2, the return value is the number of assertion failed, therefore it is
impossible to use it for :prop_test:`SKIP_RETURN_CODE`.
......@@ -823,6 +823,11 @@ static Json::Value DumpCTestProperties(
"FAIL_REGULAR_EXPRESSION",
DumpRegExToJsonArray(testProperties.ErrorRegularExpressions)));
}
if (!testProperties.SkipRegularExpressions.empty()) {
properties.append(DumpCTestProperty(
"SKIP_REGULAR_EXPRESSION",
DumpRegExToJsonArray(testProperties.SkipRegularExpressions)));
}
if (!testProperties.FixturesCleanup.empty()) {
properties.append(DumpCTestProperty(
"FIXTURES_CLEANUP", DumpToJsonArray(testProperties.FixturesCleanup)));
......
......@@ -77,6 +77,7 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started)
}
std::int64_t retVal = this->TestProcess->GetExitValue();
bool forceFail = false;
bool forceSkip = false;
bool skipped = false;
bool outputTestErrorsToConsole = false;
if (!this->TestProperties->RequiredRegularExpressions.empty() &&
......@@ -116,16 +117,34 @@ bool cmCTestRunTest::EndTest(size_t completed, size_t total, bool started)
}
}
}
if (!this->TestProperties->SkipRegularExpressions.empty() &&
this->FailedDependencies.empty()) {
for (auto& skip : this->TestProperties->SkipRegularExpressions) {
if (skip.first.find(this->ProcessOutput)) {
reason = "Skip regular expression found in output.";
reason += " Regex=[";
reason += skip.second;
reason += "]";
forceSkip = true;
break;
}
}
}
std::ostringstream outputStream;
if (res == cmProcess::State::Exited) {
bool success = !forceFail &&
(retVal == 0 ||
!this->TestProperties->RequiredRegularExpressions.empty());
if (this->TestProperties->SkipReturnCode >= 0 &&
this->TestProperties->SkipReturnCode == retVal) {
if ((this->TestProperties->SkipReturnCode >= 0 &&
this->TestProperties->SkipReturnCode == retVal) ||
forceSkip) {
this->TestResult.Status = cmCTestTestHandler::NOT_RUN;
std::ostringstream s;
s << "SKIP_RETURN_CODE=" << this->TestProperties->SkipReturnCode;
if (forceSkip) {
s << "SKIP_REGULAR_EXPRESSION_MATCHED";
} else {
s << "SKIP_RETURN_CODE=" << this->TestProperties->SkipReturnCode;
}
this->TestResult.CompletionStatus = s.str();
cmCTestLog(this->CTest, HANDLER_OUTPUT, "***Skipped ");
skipped = true;
......
......@@ -526,7 +526,7 @@ int cmCTestTestHandler::ProcessHandler()
std::vector<cmCTestTestHandler::cmCTestTestResult> disabledTests;
for (cmCTestTestResult const& ft : resultsSet) {
if (cmHasLiteralPrefix(ft.CompletionStatus, "SKIP_RETURN_CODE=") ||
if (cmHasLiteralPrefix(ft.CompletionStatus, "SKIP_") ||
ft.CompletionStatus == "Disabled") {
disabledTests.push_back(ft);
}
......@@ -599,7 +599,7 @@ int cmCTestTestHandler::ProcessHandler()
for (cmCTestTestResult const& ft : resultsSet) {
if (ft.Status != cmCTestTestHandler::COMPLETED &&
!cmHasLiteralPrefix(ft.CompletionStatus, "SKIP_RETURN_CODE=") &&
!cmHasLiteralPrefix(ft.CompletionStatus, "SKIP_") &&
ft.CompletionStatus != "Disabled") {
ofs << ft.TestCount << ":" << ft.Name << std::endl;
auto testColor = cmCTest::Color::RED;
......@@ -2229,6 +2229,13 @@ bool cmCTestTestHandler::SetTestsProperties(
rt.ErrorRegularExpressions.emplace_back(cr, cr);
}
}
if (key == "SKIP_REGULAR_EXPRESSION") {
std::vector<std::string> lval;
cmSystemTools::ExpandListArgument(val, lval);
for (std::string const& cr : lval) {
rt.SkipRegularExpressions.emplace_back(cr, cr);
}
}
if (key == "PROCESSORS") {
rt.Processors = atoi(val.c_str());
if (rt.Processors < 1) {
......
......@@ -117,6 +117,8 @@ public:
ErrorRegularExpressions;
std::vector<std::pair<cmsys::RegularExpression, std::string>>
RequiredRegularExpressions;
std::vector<std::pair<cmsys::RegularExpression, std::string>>
SkipRegularExpressions;
std::vector<std::pair<cmsys::RegularExpression, std::string>>
TimeoutRegularExpressions;
std::map<std::string, std::string> Measurements;
......
......@@ -120,6 +120,20 @@ set_tests_properties(test1 PROPERTIES FAIL_REGULAR_EXPRESSION \"foo;test1;bar\"
endfunction()
run_FailRegexFoundTest()
function(run_SkipRegexFoundTest)
set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/SkipRegexFound)
set(RunCMake_TEST_NO_CLEAN 1)
file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
file(WRITE "${RunCMake_TEST_BINARY_DIR}/CTestTestfile.cmake" "
add_test(test1 \"${CMAKE_COMMAND}\" -E echo \"test1\")
set_tests_properties(test1 PROPERTIES SKIP_REGULAR_EXPRESSION \"test1\")
")
run_cmake_command(SkipRegexFound ${CMAKE_CTEST_COMMAND} -V)
endfunction()
run_SkipRegexFoundTest()
function(run_SerialFailed)
set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/SerialFailed)
set(RunCMake_TEST_NO_CLEAN 1)
......
set(last_test_log "${RunCMake_TEST_BINARY_DIR}/Testing/Temporary/LastTest.log")
if(EXISTS "${last_test_log}")
file(READ "${last_test_log}" last_test_log_content)
string(REGEX REPLACE "\n+$" "" last_test_log_content "${last_test_log_content}")
if(NOT last_test_log_content MATCHES "
Test Pass Reason:
Skip regular expression found in output. Regex=[[]test1]")
string(REPLACE "\n" "\n " last_test_log_content " ${last_test_log_content}")
set(RunCMake_TEST_FAILED "LastTest.log does not have expected content:\n${last_test_log_content}")
endif()
else()
set(RunCMake_TEST_FAILED "LastTest.log missing:\n ${last_test_log}")
endif()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment