cmCTestTestHandler.cxx 78.3 KB
Newer Older
1 2
/* Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
   file Copyright.txt or https://cmake.org/licensing for details.  */
Ken Martin's avatar
Ken Martin committed
3
#include "cmCTestTestHandler.h"
4

5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
#include <algorithm>
#include <chrono>
#include <cmath>
#include <cstddef>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <functional>
#include <iomanip>
#include <iterator>
#include <set>
#include <sstream>
#include <utility>

20
#include <cm/memory>
Ken Martin's avatar
Ken Martin committed
21

22 23 24 25 26 27 28
#include "cmsys/FStream.hxx"
#include <cmsys/Base64.h>
#include <cmsys/Directory.hxx>
#include <cmsys/RegularExpression.hxx>

#include "cm_utf8.h"

wahikihiki's avatar
wahikihiki committed
29
#include "cmAlgorithms.h"
Daniel Pfeifer's avatar
Daniel Pfeifer committed
30 31
#include "cmCTest.h"
#include "cmCTestMultiProcessHandler.h"
32
#include "cmCTestResourceGroupsLexerHelper.h"
Wouter Klouwen's avatar
Wouter Klouwen committed
33
#include "cmDuration.h"
34
#include "cmExecutionStatus.h"
Daniel Pfeifer's avatar
Daniel Pfeifer committed
35 36 37 38 39
#include "cmGeneratedFileStream.h"
#include "cmGlobalGenerator.h"
#include "cmMakefile.h"
#include "cmState.h"
#include "cmStateSnapshot.h"
40
#include "cmStringAlgorithms.h"
Daniel Pfeifer's avatar
Daniel Pfeifer committed
41
#include "cmSystemTools.h"
42
#include "cmWorkingDirectory.h"
Daniel Pfeifer's avatar
Daniel Pfeifer committed
43 44
#include "cmXMLWriter.h"
#include "cmake.h"
45

46
namespace {
47

48
class cmCTestCommand
49 50
{
public:
51 52
  cmCTestCommand(cmCTestTestHandler* testHandler)
    : TestHandler(testHandler)
53 54
  {
  }
55

56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72
  virtual ~cmCTestCommand() = default;

  bool operator()(std::vector<cmListFileArgument> const& args,
                  cmExecutionStatus& status)
  {
    cmMakefile& mf = status.GetMakefile();
    std::vector<std::string> expandedArguments;
    if (!mf.ExpandArguments(args, expandedArguments)) {
      // There was an error expanding arguments.  It was already
      // reported, so we can skip this command without error.
      return true;
    }
    return this->InitialPass(expandedArguments, status);
  }

  virtual bool InitialPass(std::vector<std::string> const& args,
                           cmExecutionStatus& status) = 0;
73 74 75 76

  cmCTestTestHandler* TestHandler;
};

77 78
bool cmCTestSubdirCommand(std::vector<std::string> const& args,
                          cmExecutionStatus& status)
79
{
80
  if (args.empty()) {
81
    status.SetError("called with incorrect number of arguments");
82
    return false;
83
  }
84
  std::string cwd = cmSystemTools::GetCurrentWorkingDirectory();
85
  for (std::string const& arg : args) {
86
    std::string fname;
87

88
    if (cmSystemTools::FileIsFullPath(arg)) {
89
      fname = arg;
90
    } else {
91
      fname = cmStrCat(cwd, '/', arg);
92
    }
93

94
    if (!cmSystemTools::FileIsDirectory(fname)) {
95 96
      // No subdirectory? So what...
      continue;
97
    }
98 99 100
    bool readit = false;
    {
      cmWorkingDirectory workdir(fname);
101
      if (workdir.Failed()) {
102 103
        status.SetError("Failed to change directory to " + fname + " : " +
                        std::strerror(workdir.GetLastResult()));
104 105
        return false;
      }
106 107 108 109 110 111 112 113 114 115 116 117 118
      const char* testFilename;
      if (cmSystemTools::FileExists("CTestTestfile.cmake")) {
        // does the CTestTestfile.cmake exist ?
        testFilename = "CTestTestfile.cmake";
      } else if (cmSystemTools::FileExists("DartTestfile.txt")) {
        // does the DartTestfile.txt exist ?
        testFilename = "DartTestfile.txt";
      } else {
        // No CTestTestfile? Who cares...
        continue;
      }
      fname += "/";
      fname += testFilename;
119
      readit = status.GetMakefile().ReadDependentFile(fname);
120 121
    }
    if (!readit) {
122
      status.SetError(cmStrCat("Could not find include file: ", fname));
123 124
      return false;
    }
125
  }
126 127 128
  return true;
}

129 130
bool cmCTestAddSubdirectoryCommand(std::vector<std::string> const& args,
                                   cmExecutionStatus& status)
131
{
132
  if (args.empty()) {
133
    status.SetError("called with incorrect number of arguments");
134
    return false;
135
  }
136

137 138
  std::string fname =
    cmStrCat(cmSystemTools::GetCurrentWorkingDirectory(), '/', args[0]);
Andy Cedilnik's avatar
Andy Cedilnik committed
139

140
  if (!cmSystemTools::FileExists(fname)) {
141 142
    // No subdirectory? So what...
    return true;
143
  }
144 145 146 147 148 149 150 151 152 153 154 155 156 157 158
  bool readit = false;
  {
    const char* testFilename;
    if (cmSystemTools::FileExists("CTestTestfile.cmake")) {
      // does the CTestTestfile.cmake exist ?
      testFilename = "CTestTestfile.cmake";
    } else if (cmSystemTools::FileExists("DartTestfile.txt")) {
      // does the DartTestfile.txt exist ?
      testFilename = "DartTestfile.txt";
    } else {
      // No CTestTestfile? Who cares...
      return true;
    }
    fname += "/";
    fname += testFilename;
159
    readit = status.GetMakefile().ReadDependentFile(fname);
160 161
  }
  if (!readit) {
162
    status.SetError(cmStrCat("Could not find include file: ", fname));
163
    return false;
164
  }
165 166 167
  return true;
}

168
class cmCTestAddTestCommand : public cmCTestCommand
169 170
{
public:
171
  using cmCTestCommand::cmCTestCommand;
172 173 174 175 176

  /**
   * This is called when the command is first encountered in
   * the CMakeLists.txt file.
   */
177
  bool InitialPass(std::vector<std::string> const& /*args*/,
178
                   cmExecutionStatus& /*unused*/) override;
179 180
};

181
bool cmCTestAddTestCommand::InitialPass(std::vector<std::string> const& args,
182
                                        cmExecutionStatus& status)
183
{
184
  if (args.size() < 2) {
185
    status.SetError("called with incorrect number of arguments");
186
    return false;
187
  }
188
  return this->TestHandler->AddTest(args);
189 190
}

191
class cmCTestSetTestsPropertiesCommand : public cmCTestCommand
192 193
{
public:
194
  using cmCTestCommand::cmCTestCommand;
195 196 197 198

  /**
   * This is called when the command is first encountered in
   * the CMakeLists.txt file.
199
   */
200
  bool InitialPass(std::vector<std::string> const& /*args*/,
201
                   cmExecutionStatus& /*unused*/) override;
202 203
};

204
bool cmCTestSetTestsPropertiesCommand::InitialPass(
205
  std::vector<std::string> const& args, cmExecutionStatus& /*unused*/)
206
{
Andy Cedilnik's avatar
Andy Cedilnik committed
207
  return this->TestHandler->SetTestsProperties(args);
208 209
}

210
class cmCTestSetDirectoryPropertiesCommand : public cmCTestCommand
211 212
{
public:
213
  using cmCTestCommand::cmCTestCommand;
214 215 216 217

  /**
   * This is called when the command is first encountered in
   * the CMakeLists.txt file.
218
   */
219
  bool InitialPass(std::vector<std::string> const& /*unused*/,
220
                   cmExecutionStatus& /*unused*/) override;
221 222 223 224 225 226 227 228
};

bool cmCTestSetDirectoryPropertiesCommand::InitialPass(
  std::vector<std::string> const& args, cmExecutionStatus&)
{
  return this->TestHandler->SetDirectoryProperties(args);
}

Ken Martin's avatar
Ken Martin committed
229 230
// get the next number in a string with numbers separated by ,
// pos is the start of the search and pos2 is the end of the search
Andy Cedilnik's avatar
Andy Cedilnik committed
231
// pos becomes pos2 after a call to GetNextNumber.
Ken Martin's avatar
Ken Martin committed
232
// -1 is returned at the end of the list.
233
inline int GetNextNumber(std::string const& in, int& val,
Ken Martin's avatar
Ken Martin committed
234 235 236 237
                         std::string::size_type& pos,
                         std::string::size_type& pos2)
{
  pos2 = in.find(',', pos);
238
  if (pos2 != std::string::npos) {
239
    if (pos2 - pos == 0) {
Ken Martin's avatar
Ken Martin committed
240
      val = -1;
241 242 243 244
    } else {
      val = atoi(in.substr(pos, pos2 - pos).c_str());
    }
    pos = pos2 + 1;
Ken Martin's avatar
Ken Martin committed
245
    return 1;
246 247 248
  }
  if (in.size() - pos == 0) {
    val = -1;
249
  } else {
250
    val = atoi(in.substr(pos, in.size() - pos).c_str());
251
  }
252
  return 0;
Ken Martin's avatar
Ken Martin committed
253 254 255 256
}

// get the next number in a string with numbers separated by ,
// pos is the start of the search and pos2 is the end of the search
Andy Cedilnik's avatar
Andy Cedilnik committed
257
// pos becomes pos2 after a call to GetNextNumber.
Ken Martin's avatar
Ken Martin committed
258
// -1 is returned at the end of the list.
259
inline int GetNextRealNumber(std::string const& in, double& val,
Ken Martin's avatar
Ken Martin committed
260 261 262 263
                             std::string::size_type& pos,
                             std::string::size_type& pos2)
{
  pos2 = in.find(',', pos);
264
  if (pos2 != std::string::npos) {
265
    if (pos2 - pos == 0) {
Ken Martin's avatar
Ken Martin committed
266
      val = -1;
267 268 269 270
    } else {
      val = atof(in.substr(pos, pos2 - pos).c_str());
    }
    pos = pos2 + 1;
Ken Martin's avatar
Ken Martin committed
271
    return 1;
272 273 274
  }
  if (in.size() - pos == 0) {
    val = -1;
275
  } else {
276
    val = atof(in.substr(pos, in.size() - pos).c_str());
277
  }
278
  return 0;
Ken Martin's avatar
Ken Martin committed
279 280
}

281 282
} // namespace

Ken Martin's avatar
Ken Martin committed
283 284
cmCTestTestHandler::cmCTestTestHandler()
{
Andy Cedilnik's avatar
Andy Cedilnik committed
285
  this->UseUnion = false;
Andy Cedilnik's avatar
Andy Cedilnik committed
286

287 288 289 290 291
  this->UseIncludeLabelRegExpFlag = false;
  this->UseExcludeLabelRegExpFlag = false;
  this->UseIncludeRegExpFlag = false;
  this->UseExcludeRegExpFlag = false;
  this->UseExcludeRegExpFirst = false;
292
  this->UseHardwareSpec = false;
293

Andy Cedilnik's avatar
Andy Cedilnik committed
294 295
  this->CustomMaximumPassedTestOutputSize = 1 * 1024;
  this->CustomMaximumFailedTestOutputSize = 300 * 1024;
Andy Cedilnik's avatar
Andy Cedilnik committed
296

Andy Cedilnik's avatar
Andy Cedilnik committed
297
  this->MemCheck = false;
298

Daniel Pfeifer's avatar
Daniel Pfeifer committed
299
  this->LogFile = nullptr;
Andy Cedilnik's avatar
Andy Cedilnik committed
300

Bill Hoffman's avatar
Bill Hoffman committed
301
  // regex to detect <DartMeasurement>...</DartMeasurement>
302
  this->DartStuff.compile("(<DartMeasurement.*/DartMeasurement[a-zA-Z]*>)");
Bill Hoffman's avatar
Bill Hoffman committed
303 304 305
  // regex to detect each individual <DartMeasurement>...</DartMeasurement>
  this->DartStuff1.compile(
    "(<DartMeasurement[^<]*</DartMeasurement[a-zA-Z]*>)");
Ken Martin's avatar
Ken Martin committed
306 307
}

308 309
void cmCTestTestHandler::Initialize()
{
310 311
  this->Superclass::Initialize();

Wouter Klouwen's avatar
Wouter Klouwen committed
312
  this->ElapsedTestingTime = cmDuration();
313

Andy Cedilnik's avatar
Andy Cedilnik committed
314
  this->TestResults.clear();
315

Andy Cedilnik's avatar
Andy Cedilnik committed
316
  this->CustomTestsIgnore.clear();
317 318
  this->StartTest.clear();
  this->EndTest.clear();
319

Andy Cedilnik's avatar
Andy Cedilnik committed
320 321 322 323
  this->CustomPreTest.clear();
  this->CustomPostTest.clear();
  this->CustomMaximumPassedTestOutputSize = 1 * 1024;
  this->CustomMaximumFailedTestOutputSize = 300 * 1024;
324

Andy Cedilnik's avatar
Andy Cedilnik committed
325
  this->TestsToRun.clear();
326

327 328
  this->UseIncludeLabelRegExpFlag = false;
  this->UseExcludeLabelRegExpFlag = false;
Andy Cedilnik's avatar
Andy Cedilnik committed
329 330 331
  this->UseIncludeRegExpFlag = false;
  this->UseExcludeRegExpFlag = false;
  this->UseExcludeRegExpFirst = false;
332 333
  this->IncludeLabelRegularExpression = "";
  this->ExcludeLabelRegularExpression = "";
334 335
  this->IncludeRegExp.clear();
  this->ExcludeRegExp.clear();
336 337 338
  this->ExcludeFixtureRegExp.clear();
  this->ExcludeFixtureSetupRegExp.clear();
  this->ExcludeFixtureCleanupRegExp.clear();
339

340
  TestsToRunString.clear();
Andy Cedilnik's avatar
Andy Cedilnik committed
341 342
  this->UseUnion = false;
  this->TestList.clear();
343 344
}

345
void cmCTestTestHandler::PopulateCustomVectors(cmMakefile* mf)
Ken Martin's avatar
Ken Martin committed
346
{
347
  this->CTest->PopulateCustomVector(mf, "CTEST_CUSTOM_PRE_TEST",
348
                                    this->CustomPreTest);
349
  this->CTest->PopulateCustomVector(mf, "CTEST_CUSTOM_POST_TEST",
350 351 352 353 354 355 356 357 358
                                    this->CustomPostTest);
  this->CTest->PopulateCustomVector(mf, "CTEST_CUSTOM_TESTS_IGNORE",
                                    this->CustomTestsIgnore);
  this->CTest->PopulateCustomInteger(
    mf, "CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE",
    this->CustomMaximumPassedTestOutputSize);
  this->CTest->PopulateCustomInteger(
    mf, "CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE",
    this->CustomMaximumFailedTestOutputSize);
Ken Martin's avatar
Ken Martin committed
359 360
}

361
int cmCTestTestHandler::PreProcessHandler()
Ken Martin's avatar
Ken Martin committed
362
{
363
  if (!this->ExecuteCommands(this->CustomPreTest)) {
Andy Cedilnik's avatar
Andy Cedilnik committed
364
    cmCTestLog(this->CTest, ERROR_MESSAGE,
365
               "Problem executing pre-test command(s)." << std::endl);
366
    return 0;
367
  }
368 369
  return 1;
}
Ken Martin's avatar
Ken Martin committed
370

371 372
int cmCTestTestHandler::PostProcessHandler()
{
373
  if (!this->ExecuteCommands(this->CustomPostTest)) {
Andy Cedilnik's avatar
Andy Cedilnik committed
374
    cmCTestLog(this->CTest, ERROR_MESSAGE,
375
               "Problem executing post-test command(s)." << std::endl);
376
    return 0;
377
  }
378 379 380 381 382
  return 1;
}

int cmCTestTestHandler::ProcessHandler()
{
383 384
  if (!this->ProcessOptions()) {
    return -1;
385
  }
Brad King's avatar
Brad King committed
386

Andy Cedilnik's avatar
Andy Cedilnik committed
387
  this->TestResults.clear();
388

389 390 391 392 393 394
  cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT,
                     (this->MemCheck ? "Memory check" : "Test")
                       << " project "
                       << cmSystemTools::GetCurrentWorkingDirectory()
                       << std::endl,
                     this->Quiet);
395
  if (!this->PreProcessHandler()) {
396
    return -1;
397
  }
Ken Martin's avatar
Ken Martin committed
398

Andy Cedilnik's avatar
Andy Cedilnik committed
399
  cmGeneratedFileStream mLogFile;
400
  this->StartLogFile((this->MemCheck ? "DynamicAnalysis" : "Test"), mLogFile);
Andy Cedilnik's avatar
Andy Cedilnik committed
401
  this->LogFile = &mLogFile;
Andy Cedilnik's avatar
Andy Cedilnik committed
402

403 404
  std::vector<std::string> passed;
  std::vector<std::string> failed;
405

406
  // start the real time clock
407
  auto clock_start = std::chrono::steady_clock::now();
408

409
  this->ProcessDirectory(passed, failed);
Ken Martin's avatar
Ken Martin committed
410

411
  auto clock_finish = std::chrono::steady_clock::now();
412

413
  if (passed.size() + failed.size() == 0) {
414
    if (!this->CTest->GetShowOnly() && !this->CTest->ShouldPrintLabels()) {
415 416
      cmCTestLog(this->CTest, ERROR_MESSAGE,
                 "No tests were found!!!" << std::endl);
Ken Martin's avatar
Ken Martin committed
417
    }
418
  } else {
419
    if (this->HandlerVerbose && !passed.empty() &&
420
        (this->UseIncludeRegExpFlag || this->UseExcludeRegExpFlag)) {
421 422
      cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
                         std::endl
423 424
                           << "The following tests passed:" << std::endl,
                         this->Quiet);
425
      for (std::string const& j : passed) {
426
        cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
427
                           "\t" << j << std::endl, this->Quiet);
Ken Martin's avatar
Ken Martin committed
428
      }
429
    }
Ken Martin's avatar
Ken Martin committed
430

Betsy McPhail's avatar
Betsy McPhail committed
431 432 433
    SetOfTests resultsSet(this->TestResults.begin(), this->TestResults.end());
    std::vector<cmCTestTestHandler::cmCTestTestResult> disabledTests;

434
    for (cmCTestTestResult const& ft : resultsSet) {
435
      if (cmHasLiteralPrefix(ft.CompletionStatus, "SKIP_") ||
436 437
          ft.CompletionStatus == "Disabled") {
        disabledTests.push_back(ft);
Betsy McPhail's avatar
Betsy McPhail committed
438 439 440
      }
    }

441 442
    cmDuration durationInSecs = clock_finish - clock_start;
    this->LogTestSummary(passed, failed, durationInSecs);
443

444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469
    this->LogDisabledTests(disabledTests);

    this->LogFailedTests(failed, resultsSet);
  }

  if (!this->GenerateXML()) {
    return 1;
  }

  if (!this->PostProcessHandler()) {
    this->LogFile = nullptr;
    return -1;
  }

  if (!failed.empty()) {
    this->LogFile = nullptr;
    return -1;
  }
  this->LogFile = nullptr;
  return 0;
}

bool cmCTestTestHandler::ProcessOptions()
{
  // Update internal data structure from generic one
  this->SetTestsToRunInformation(this->GetOption("TestsToRunInformation"));
470 471
  this->SetUseUnion(cmIsOn(this->GetOption("UseUnion")));
  if (cmIsOn(this->GetOption("ScheduleRandom"))) {
472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510
    this->CTest->SetScheduleType("Random");
  }
  if (this->GetOption("ParallelLevel")) {
    this->CTest->SetParallelLevel(atoi(this->GetOption("ParallelLevel")));
  }

  const char* val;
  val = this->GetOption("LabelRegularExpression");
  if (val) {
    this->UseIncludeLabelRegExpFlag = true;
    this->IncludeLabelRegExp = val;
  }
  val = this->GetOption("ExcludeLabelRegularExpression");
  if (val) {
    this->UseExcludeLabelRegExpFlag = true;
    this->ExcludeLabelRegExp = val;
  }
  val = this->GetOption("IncludeRegularExpression");
  if (val) {
    this->UseIncludeRegExp();
    this->SetIncludeRegExp(val);
  }
  val = this->GetOption("ExcludeRegularExpression");
  if (val) {
    this->UseExcludeRegExp();
    this->SetExcludeRegExp(val);
  }
  val = this->GetOption("ExcludeFixtureRegularExpression");
  if (val) {
    this->ExcludeFixtureRegExp = val;
  }
  val = this->GetOption("ExcludeFixtureSetupRegularExpression");
  if (val) {
    this->ExcludeFixtureSetupRegExp = val;
  }
  val = this->GetOption("ExcludeFixtureCleanupRegularExpression");
  if (val) {
    this->ExcludeFixtureCleanupRegExp = val;
  }
511
  this->SetRerunFailed(cmIsOn(this->GetOption("RerunFailed")));
512

513
  val = this->GetOption("ResourceSpecFile");
514 515 516 517
  if (val) {
    this->UseHardwareSpec = true;
    if (!this->HardwareSpec.ReadFromJSONFile(val)) {
      cmCTestLog(this->CTest, ERROR_MESSAGE,
518
                 "Could not read resource spec file: " << val << std::endl);
519 520 521 522
      return false;
    }
  }

523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569
  return true;
}

void cmCTestTestHandler::LogTestSummary(const std::vector<std::string>& passed,
                                        const std::vector<std::string>& failed,
                                        const cmDuration& durationInSecs)
{
  std::size_t total = passed.size() + failed.size();

  float percent = float(passed.size()) * 100.0f / float(total);
  if (!failed.empty() && percent > 99) {
    percent = 99;
  }

  std::string passColorCode;
  std::string failedColorCode;
  if (failed.empty()) {
    passColorCode = this->CTest->GetColorCode(cmCTest::Color::GREEN);
  } else {
    failedColorCode = this->CTest->GetColorCode(cmCTest::Color::RED);
  }
  cmCTestLog(this->CTest, HANDLER_OUTPUT,
             std::endl
               << passColorCode << std::lround(percent) << "% tests passed"
               << this->CTest->GetColorCode(cmCTest::Color::CLEAR_COLOR)
               << ", " << failedColorCode << failed.size() << " tests failed"
               << this->CTest->GetColorCode(cmCTest::Color::CLEAR_COLOR)
               << " out of " << total << std::endl);
  if ((!this->CTest->GetLabelsForSubprojects().empty() &&
       this->CTest->GetSubprojectSummary())) {
    this->PrintLabelOrSubprojectSummary(true);
  }
  if (this->CTest->GetLabelSummary()) {
    this->PrintLabelOrSubprojectSummary(false);
  }
  char realBuf[1024];
  sprintf(realBuf, "%6.2f sec", durationInSecs.count());
  cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT,
                     "\nTotal Test time (real) = " << realBuf << "\n",
                     this->Quiet);
}

void cmCTestTestHandler::LogDisabledTests(
  const std::vector<cmCTestTestResult>& disabledTests)
{
  if (!disabledTests.empty()) {
    cmGeneratedFileStream ofs;
570 571
    cmCTestLog(this->CTest, HANDLER_OUTPUT,
               std::endl
572 573
                 << "The following tests did not run:" << std::endl);
    this->StartLogFile("TestsDisabled", ofs);
Betsy McPhail's avatar
Betsy McPhail committed
574

575 576 577 578 579 580 581 582 583
    const char* disabled_reason;
    cmCTestLog(this->CTest, HANDLER_OUTPUT,
               this->CTest->GetColorCode(cmCTest::Color::BLUE));
    for (cmCTestTestResult const& dt : disabledTests) {
      ofs << dt.TestCount << ":" << dt.Name << std::endl;
      if (dt.CompletionStatus == "Disabled") {
        disabled_reason = "Disabled";
      } else {
        disabled_reason = "Skipped";
Betsy McPhail's avatar
Betsy McPhail committed
584
      }
585
      cmCTestLog(this->CTest, HANDLER_OUTPUT,
586 587
                 "\t" << std::setw(3) << dt.TestCount << " - " << dt.Name
                      << " (" << disabled_reason << ")" << std::endl);
Betsy McPhail's avatar
Betsy McPhail committed
588
    }
589 590 591 592
    cmCTestLog(this->CTest, HANDLER_OUTPUT,
               this->CTest->GetColorCode(cmCTest::Color::CLEAR_COLOR));
  }
}
Betsy McPhail's avatar
Betsy McPhail committed
593

594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611
void cmCTestTestHandler::LogFailedTests(const std::vector<std::string>& failed,
                                        const SetOfTests& resultsSet)
{
  if (!failed.empty()) {
    cmGeneratedFileStream ofs;
    cmCTestLog(this->CTest, HANDLER_OUTPUT,
               std::endl
                 << "The following tests FAILED:" << std::endl);
    this->StartLogFile("TestsFailed", ofs);

    for (cmCTestTestResult const& ft : resultsSet) {
      if (ft.Status != cmCTestTestHandler::COMPLETED &&
          !cmHasLiteralPrefix(ft.CompletionStatus, "SKIP_") &&
          ft.CompletionStatus != "Disabled") {
        ofs << ft.TestCount << ":" << ft.Name << std::endl;
        auto testColor = cmCTest::Color::RED;
        if (this->GetTestStatus(ft) == "Not Run") {
          testColor = cmCTest::Color::YELLOW;
Ken Martin's avatar
Ken Martin committed
612
        }
613 614 615 616 617 618 619
        cmCTestLog(
          this->CTest, HANDLER_OUTPUT,
          "\t" << this->CTest->GetColorCode(testColor) << std::setw(3)
               << ft.TestCount << " - " << ft.Name << " ("
               << this->GetTestStatus(ft) << ")"
               << this->CTest->GetColorCode(cmCTest::Color::CLEAR_COLOR)
               << std::endl);
Ken Martin's avatar
Ken Martin committed
620 621
      }
    }
622
  }
623
}
Ken Martin's avatar
Ken Martin committed
624

625 626
bool cmCTestTestHandler::GenerateXML()
{
627
  if (this->CTest->GetProduceXML()) {
628
    cmGeneratedFileStream xmlfile;
629
    if (!this->StartResultingXML(
630
          (this->MemCheck ? cmCTest::PartMemCheck : cmCTest::PartTest),
631
          (this->MemCheck ? "DynamicAnalysis" : "Test"), xmlfile)) {
632 633
      cmCTestLog(this->CTest, ERROR_MESSAGE,
                 "Cannot create "
634 635
                   << (this->MemCheck ? "memory check" : "testing")
                   << " XML file" << std::endl);
Daniel Pfeifer's avatar
Daniel Pfeifer committed
636
      this->LogFile = nullptr;
637
      return false;
638
    }
639 640
    cmXMLWriter xml(xmlfile);
    this->GenerateDartOutput(xml);
641
  }
Ken Martin's avatar
Ken Martin committed
642

643
  return true;
Ken Martin's avatar
Ken Martin committed
644 645
}

646
void cmCTestTestHandler::PrintLabelOrSubprojectSummary(bool doSubProject)
647
{
648
  // collect subproject labels
649 650 651 652 653 654
  std::vector<std::string> subprojects =
    this->CTest->GetLabelsForSubprojects();
  std::map<std::string, double> labelTimes;
  std::map<std::string, int> labelCounts;
  std::set<std::string> labels;
  std::string::size_type maxlen = 0;
655
  // initialize maps
656 657
  for (cmCTestTestProperties& p : this->TestList) {
    for (std::string const& l : p.Labels) {
658 659
      // first check to see if the current label is a subproject label
      bool isSubprojectLabel = false;
wahikihiki's avatar
wahikihiki committed
660
      auto subproject = std::find(subprojects.begin(), subprojects.end(), l);
661
      if (subproject != subprojects.end()) {
662 663 664 665
        isSubprojectLabel = true;
      }
      // if we are doing sub projects and this label is one, then use it
      // if we are not doing sub projects and the label is not one use it
666
      if (doSubProject == isSubprojectLabel) {
667 668
        if (l.size() > maxlen) {
          maxlen = l.size();
669
        }
670 671 672
        labels.insert(l);
        labelTimes[l] = 0;
        labelCounts[l] = 0;
673 674 675 676
      }
    }
  }
  // fill maps
677
  for (cmCTestTestResult& result : this->TestResults) {
678
    cmCTestTestProperties& p = *result.Properties;
679
    for (std::string const& l : p.Labels) {
680
      // only use labels found in labels
wahikihiki's avatar
wahikihiki committed
681
      if (cmContains(labels, l)) {
682
        labelTimes[l] +=
683
          result.ExecutionTime.count() * result.Properties->Processors;
684
        ++labelCounts[l];
685 686 687
      }
    }
  }
688 689 690 691
  // if no labels are found return and print nothing
  if (labels.empty()) {
    return;
  }
692
  // now print times
693
  if (doSubProject) {
694 695
    cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT,
                       "\nSubproject Time Summary:", this->Quiet);
696
  } else {
697 698
    cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT,
                       "\nLabel Time Summary:", this->Quiet);
699
  }
700 701
  for (std::string const& i : labels) {
    std::string label = i;
702 703 704
    label.resize(maxlen + 3, ' ');

    char buf[1024];
705
    sprintf(buf, "%6.2f sec*proc", labelTimes[i]);
706 707

    std::ostringstream labelCountStr;
708 709
    labelCountStr << "(" << labelCounts[i] << " test";
    if (labelCounts[i] > 1) {
710 711 712
      labelCountStr << "s";
    }
    labelCountStr << ")";
713 714
    cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT,
                       "\n"
715 716 717 718
                         << label << " = " << buf << " "
                         << labelCountStr.str(),
                       this->Quiet);
    if (this->LogFile) {
719
      *this->LogFile << "\n" << i << " = " << buf << "\n";
720 721
    }
  }
722 723
  if (this->LogFile) {
    *this->LogFile << "\n";
724
  }
725
  cmCTestOptionalLog(this->CTest, HANDLER_OUTPUT, "\n", this->Quiet);
726
}
727

728 729 730
void cmCTestTestHandler::CheckLabelFilterInclude(cmCTestTestProperties& it)
{
  // if not using Labels to filter then return
731
  if (!this->UseIncludeLabelRegExpFlag) {
732
    return;
733
  }
734 735
  // if there are no labels and we are filtering by labels
  // then exclude the test as it does not have the label
736
  if (it.Labels.empty()) {
737 738
    it.IsInBasedOnREOptions = false;
    return;
739
  }
740
  // check to see if the label regular expression matches
741
  bool found = false; // assume it does not match
742
  // loop over all labels and look for match
743 744
  for (std::string const& l : it.Labels) {
    if (this->IncludeLabelRegularExpression.find(l)) {
745 746
      found = true;
    }
747
  }
748
  // if no match was found, exclude the test
749
  if (!found) {
750
    it.IsInBasedOnREOptions = false;
751
  }
752 753 754 755 756
}

void cmCTestTestHandler::CheckLabelFilterExclude(cmCTestTestProperties& it)
{
  // if not using Labels to filter then return
757
  if (!this->UseExcludeLabelRegExpFlag) {
758
    return;
759
  }
760 761
  // if there are no labels and we are excluding by labels
  // then do nothing as a no label can not be a match
762
  if (it.Labels.empty()) {
763
    return;
764
  }
765
  // check to see if the label regular expression matches
766
  bool found = false; // assume it does not match
767
  // loop over all labels and look for match
768 769
  for (std::string const& l : it.Labels) {
    if (this->ExcludeLabelRegularExpression.find(l)) {
770 771
      found = true;
    }
772
  }
773
  // if match was found, exclude the test
774
  if (found) {
775
    it.IsInBasedOnREOptions = false;
776
  }
777 778 779 780 781 782 783 784
}

void cmCTestTestHandler::CheckLabelFilter(cmCTestTestProperties& it)
{
  this->CheckLabelFilterInclude(it);
  this->CheckLabelFilterExclude(it);
}

785
void cmCTestTestHandler::ComputeTestList()
Ken Martin's avatar
Ken Martin committed
786
{
787
  this->TestList.clear(); // clear list of test
788
  this->GetListOfTests();
789

790
  if (this->RerunFailed) {
791 792
    this->ComputeTestListForRerunFailed();
    return;
793
  }
794

Andy Cedilnik's avatar
Andy Cedilnik committed
795
  cmCTestTestHandler::ListOfTests::size_type tmsize = this->TestList.size();
796 797
  // how many tests are in based on RegExp?
  int inREcnt = 0;
798 799 800
  for (cmCTestTestProperties& tp : this->TestList) {
    this->CheckLabelFilter(tp);
    if (tp.IsInBasedOnREOptions) {
801
      inREcnt++;
802
    }
803
  }
804
  // expand the test list based on the union flag
805
  if (this->UseUnion) {
Daniel Pfeifer's avatar
Daniel Pfeifer committed
806
    this->ExpandTestsToRunInformation(static_cast<int>(tmsize));
807
  } else {