Merge topic 'ctest_runtime_labels'

bd38749f ctest: allow test output to add labels

Acked-by: Kitware Robot <kwrobot@kitware.com>
Merge-request: !6234

Help/command/ctest_test.rst

@@ -270,3 +270,18 @@ The following example demonstrates how to specify a custom value for the
 
    std::cout <<
      "<CTestDetails>My Custom Details Value</CTestDetails>" << std::endl;
+
+Additional Labels
+"""""""""""""""""
+
+The following example demonstrates how to add additional labels to a test
+at runtime.
+
+.. code-block:: c++
+
+   std::cout <<
+     "<CTestLabel>Custom Label 1</CTestLabel>\n" <<
+     "<CTestLabel>Custom Label 2</CTestLabel>"   << std::endl;
+
+Use the :prop_test:`LABELS` test property instead for labels that can be
+determined at configure time.

Help/release/dev/ctest-runtime-labels.rst

+ctest-runtime-labels
+--------------------
+
+* :manual:`ctest(1)` learned to recognize labels attached to a test at run time.
+  Previously it was only possible to attach labels to tests at configure time
+  by using the :prop_test:`LABELS` test property.
+  See :ref:`Additional Test Measurements` for more information.

Source/CTest/cmCTestRunTest.cxx

@@ -2,6 +2,7 @@
    file Copyright.txt or https://cmake.org/licensing for details.  */
 #include "cmCTestRunTest.h"
 
+#include <algorithm>
 #include <chrono>
 #include <cstddef> // IWYU pragma: keep
 #include <cstdint>
@@ -44,7 +45,9 @@ void cmCTestRunTest::CheckOutput(std::string const& line)
   // Check for special CTest XML tags in this line of output.
   // If any are found, this line is excluded from ProcessOutput.
   if (!line.empty() && line.find("<CTest") != std::string::npos) {
+    bool ctest_tag_found = false;
     if (this->TestHandler->CustomCompletionStatusRegex.find(line)) {
+      ctest_tag_found = true;
       this->TestResult.CustomCompletionStatus =
         this->TestHandler->CustomCompletionStatusRegex.match(1);
       cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
@@ -52,6 +55,20 @@ void cmCTestRunTest::CheckOutput(std::string const& line)
                                   << "Test Details changed to '"
                                   << this->TestResult.CustomCompletionStatus
                                   << "'" << std::endl);
+    } else if (this->TestHandler->CustomLabelRegex.find(line)) {
+      ctest_tag_found = true;
+      auto label = this->TestHandler->CustomLabelRegex.match(1);
+      auto& labels = this->TestProperties->Labels;
+      if (std::find(labels.begin(), labels.end(), label) == labels.end()) {
+        labels.push_back(label);
+        std::sort(labels.begin(), labels.end());
+        cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
+                   this->GetIndex()
+                     << ": "
+                     << "Test Label added: '" << label << "'" << std::endl);
+      }
+    }
+    if (ctest_tag_found) {
       return;
     }
   }

Source/CTest/cmCTestTestHandler.cxx

@@ -312,6 +312,8 @@ cmCTestTestHandler::cmCTestTestHandler()
   // regex to detect <CTestDetails>...</CTestDetails>
   this->CustomCompletionStatusRegex.compile(
     "<CTestDetails>(.*)</CTestDetails>");
+  // regex to detect <CTestLabel>...</CTestLabel>
+  this->CustomLabelRegex.compile("<CTestLabel>(.*)</CTestLabel>");
 }
 
 void cmCTestTestHandler::Initialize()

Source/CTest/cmCTestTestHandler.h

@@ -360,6 +360,7 @@ private:
   size_t TotalNumberOfTests;
   cmsys::RegularExpression DartStuff;
   cmsys::RegularExpression CustomCompletionStatusRegex;
+  cmsys::RegularExpression CustomLabelRegex;
 
   std::ostream* LogFile;
 

Tests/RunCMake/ctest_test/RunCMakeTest.cmake

@@ -181,3 +181,16 @@ add_test(
   run_ctest(TestCompletionStatus)
 endfunction()
 run_completion_status()
+
+# Verify that test output can add additional labels
+function(run_extra_labels)
+  set(CASE_CMAKELISTS_SUFFIX_CODE [[
+add_test(
+  NAME custom_labels
+  COMMAND ${CMAKE_COMMAND} -E
+  echo before\n<CTestLabel>label2</CTestLabel>\n<CTestLabel>label1</CTestLabel>\n<CTestLabel>label3</CTestLabel>\n<CTestLabel>label2</CTestLabel>\nafter)
+set_tests_properties(custom_labels PROPERTIES LABELS "label1")
+  ]])
+  run_ctest(TestExtraLabels)
+endfunction()
+run_extra_labels()

Tests/RunCMake/ctest_test/TestExtraLabels-check.cmake

+file(READ "${RunCMake_TEST_BINARY_DIR}/Testing/TAG" _tag)
+string(REGEX REPLACE "^([^\n]*)\n.*$" "\\1" _date "${_tag}")
+file(READ "${RunCMake_TEST_BINARY_DIR}/Testing/${_date}/Test.xml" _test_contents)
+
+# Check labels.
+STRING(REGEX MATCHALL [[<Label>label1</Label>]] matches "${_test_contents}")
+list(LENGTH matches n_matches)
+if(NOT n_matches EQUAL 1)
+  string(APPEND RunCMake_TEST_FAILED  "expected 1 match for label1, found ${n_matches}")
+endif()
+STRING(REGEX MATCHALL [[<Label>label2</Label>]] matches "${_test_contents}")
+list(LENGTH matches n_matches)
+if(NOT n_matches EQUAL 1)
+  string(APPEND RunCMake_TEST_FAILED  "expected 1 match for label2, found ${n_matches}")
+endif()
+STRING(REGEX MATCHALL [[<Label>label3</Label>]] matches "${_test_contents}")
+list(LENGTH matches n_matches)
+if(NOT n_matches EQUAL 1)
+  string(APPEND RunCMake_TEST_FAILED  "expected 1 match for label3, found ${n_matches}")
+endif()
+
+# Check test output.
+if(NOT _test_contents MATCHES "before")
+  string(APPEND RunCMake_TEST_FAILED "Could not find expected string 'before' in Test.xml")
+endif()
+if(NOT _test_contents MATCHES "after")
+  string(APPEND RunCMake_TEST_FAILED "Could not find expected string 'after' in Test.xml")
+endif()