cmCTestMultiProcessHandler.cxx 40.2 KB
Newer Older
1 2
/* Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
   file Copyright.txt or https://cmake.org/licensing for details.  */
3
#include "cmCTestMultiProcessHandler.h"
Brad King's avatar
Brad King committed
4

5
#include <algorithm>
6
#include <cassert>
7
#include <chrono>
8
#include <cmath>
9
#include <cstddef>
10
#include <cstdlib>
11
#include <cstring>
12
#include <iomanip>
13
#include <iostream>
14
#include <list>
15
#include <memory>
16
#include <sstream>
17
#include <stack>
18
#include <unordered_map>
19
#include <utility>
20
#include <vector>
21

22 23 24 25 26 27 28 29 30 31
#include "cmsys/FStream.hxx"
#include "cmsys/SystemInformation.hxx"

#include "cm_jsoncpp_value.h"
#include "cm_jsoncpp_writer.h"
#include "cm_uv.h"

#include "cmAffinity.h"
#include "cmAlgorithms.h"
#include "cmCTest.h"
32
#include "cmCTestBinPacker.h"
33 34 35 36 37 38 39 40 41 42
#include "cmCTestRunTest.h"
#include "cmCTestTestHandler.h"
#include "cmDuration.h"
#include "cmListFileCache.h"
#include "cmRange.h"
#include "cmStringAlgorithms.h"
#include "cmSystemTools.h"
#include "cmUVSignalHackRAII.h" // IWYU pragma: keep
#include "cmWorkingDirectory.h"

43 44 45 46
namespace cmsys {
class RegularExpression;
}

Zach's avatar
Zach committed
47 48 49
class TestComparator
{
public:
50 51 52 53
  TestComparator(cmCTestMultiProcessHandler* handler)
    : Handler(handler)
  {
  }
Zach's avatar
Zach committed
54 55

  // Sorts tests in descending order of cost
56 57
  bool operator()(int index1, int index2) const
  {
Zach's avatar
Zach committed
58 59
    return Handler->Properties[index1]->Cost >
      Handler->Properties[index2]->Cost;
60
  }
Zach's avatar
Zach committed
61 62 63 64 65

private:
  cmCTestMultiProcessHandler* Handler;
};

66 67 68
cmCTestMultiProcessHandler::cmCTestMultiProcessHandler()
{
  this->ParallelLevel = 1;
69
  this->TestLoad = 0;
Brad King's avatar
Brad King committed
70
  this->FakeLoadForTesting = 0;
71
  this->Completed = 0;
72
  this->RunningCount = 0;
73 74
  this->ProcessorsAvailable = cmAffinity::GetProcessorsAvailable();
  this->HaveAffinity = this->ProcessorsAvailable.size();
Nils Gladitz's avatar
Nils Gladitz committed
75
  this->HasCycles = false;
76
  this->SerialTestRunning = false;
77
}
78

wahikihiki's avatar
wahikihiki committed
79
cmCTestMultiProcessHandler::~cmCTestMultiProcessHandler() = default;
80

81 82 83
// Set the tests
void cmCTestMultiProcessHandler::SetTests(TestMap& tests,
                                          PropertiesMap& properties)
84
{
85 86 87
  this->Tests = tests;
  this->Properties = properties;
  this->Total = this->Tests.size();
88
  // set test run map to false for all
89 90 91
  for (auto const& t : this->Tests) {
    this->TestRunningMap[t.first] = false;
    this->TestFinishMap[t.first] = false;
92 93
  }
  if (!this->CTest->GetShowOnly()) {
94
    this->ReadCostData();
Nils Gladitz's avatar
Nils Gladitz committed
95
    this->HasCycles = !this->CheckCycles();
96
    if (this->HasCycles) {
Nils Gladitz's avatar
Nils Gladitz committed
97
      return;
98
    }
99 100
    this->CreateTestCostList();
  }
101
}
102

103
// Set the max number of tests that can be run at the same time.
104
void cmCTestMultiProcessHandler::SetParallelLevel(size_t level)
105
{
106
  this->ParallelLevel = level < 1 ? 1 : level;
107 108
}

109 110 111
void cmCTestMultiProcessHandler::SetTestLoad(unsigned long load)
{
  this->TestLoad = load;
Brad King's avatar
Brad King committed
112 113 114 115

  std::string fake_load_value;
  if (cmSystemTools::GetEnv("__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING",
                            fake_load_value)) {
116
    if (!cmStrToULong(fake_load_value, &this->FakeLoadForTesting)) {
117 118
      cmSystemTools::Error("Failed to parse fake load value: " +
                           fake_load_value);
Brad King's avatar
Brad King committed
119 120
    }
  }
121 122
}

123 124
void cmCTestMultiProcessHandler::RunTests()
{
125
  this->CheckResume();
126
  if (this->HasCycles) {
127
    return;
128
  }
129 130 131
#ifdef CMAKE_UV_SIGNAL_HACK
  cmUVSignalHackRAII hackRAII;
#endif
132
  this->TestHandler->SetMaxIndex(this->FindMaxIndex());
133 134

  uv_loop_init(&this->Loop);
135
  this->StartNextTests();
136 137 138
  uv_run(&this->Loop, UV_RUN_DEFAULT);
  uv_loop_close(&this->Loop);

139 140 141 142
  if (!this->StopTimePassed) {
    assert(this->Completed == this->Total);
    assert(this->Tests.empty());
  }
143
  assert(this->AllResourcesAvailable());
144

145
  this->MarkFinished();
146
  this->UpdateCostData();
147 148
}

149
bool cmCTestMultiProcessHandler::StartTestProcess(int test)
150
{
151 152 153 154 155 156 157 158 159 160 161 162 163 164 165
  if (this->HaveAffinity && this->Properties[test]->WantAffinity) {
    size_t needProcessors = this->GetProcessorsUsed(test);
    if (needProcessors > this->ProcessorsAvailable.size()) {
      return false;
    }
    std::vector<size_t> affinity;
    affinity.reserve(needProcessors);
    for (size_t i = 0; i < needProcessors; ++i) {
      auto p = this->ProcessorsAvailable.begin();
      affinity.push_back(*p);
      this->ProcessorsAvailable.erase(p);
    }
    this->Properties[test]->Affinity = std::move(affinity);
  }

Zack Galbreath's avatar
Zack Galbreath committed
166
  cmCTestOptionalLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
167
                     "test " << test << "\n", this->Quiet);
168 169
  this->TestRunningMap[test] = true; // mark the test as running
  // now remove the test itself
170
  this->EraseTest(test);
171
  this->RunningCount += GetProcessorsUsed(test);
172

173
  cmCTestRunTest* testRun = new cmCTestRunTest(*this);
174
  if (this->CTest->GetRepeatUntilFail()) {
175 176
    testRun->SetRunUntilFailOn();
    testRun->SetNumberOfRuns(this->CTest->GetTestRepeat());
177
  }
178 179
  testRun->SetIndex(test);
  testRun->SetTestProperties(this->Properties[test]);
180 181 182
  if (this->TestHandler->UseResourceSpec) {
    testRun->SetUseAllocatedResources(true);
    testRun->SetAllocatedResources(this->AllocatedResources[test]);
183
  }
184

185 186
  // Find any failed dependencies for this test. We assume the more common
  // scenario has no failed tests, so make it the outer loop.
187
  for (std::string const& f : *this->Failed) {
wahikihiki's avatar
wahikihiki committed
188
    if (cmContains(this->Properties[test]->RequireSuccessDepends, f)) {
189
      testRun->AddFailedDependency(f);
190 191 192
    }
  }

193 194
  // Always lock the resources we'll be using, even if we fail to set the
  // working directory because FinishTestProcess() will try to unlock them
195
  this->LockResources(test);
196

197
  if (!this->TestsHaveSufficientResources[test]) {
198
    testRun->StartFailure("Insufficient resources");
199 200 201
    this->FinishTestProcess(testRun, false);
    return false;
  }
Zach's avatar
Zach committed
202

203 204 205 206 207 208
  cmWorkingDirectory workdir(this->Properties[test]->Directory);
  if (workdir.Failed()) {
    testRun->StartFailure("Failed to change working directory to " +
                          this->Properties[test]->Directory + " : " +
                          std::strerror(workdir.GetLastResult()));
  } else {
209
    if (testRun->StartTest(this->Completed, this->Total)) {
210 211
      // Ownership of 'testRun' has moved to another structure.
      // When the test finishes, FinishTestProcess will be called.
212 213
      return true;
    }
214
  }
215

216
  // Pass ownership of 'testRun'.
217
  this->FinishTestProcess(testRun, false);
218
  return false;
219 220
}

221
bool cmCTestMultiProcessHandler::AllocateResources(int index)
222
{
223
  if (!this->TestHandler->UseResourceSpec) {
224 225 226 227
    return true;
  }

  std::map<std::string, std::vector<cmCTestBinPackerAllocation>> allocations;
228
  if (!this->TryAllocateResources(index, allocations)) {
229 230 231
    return false;
  }

232 233
  auto& allocatedResources = this->AllocatedResources[index];
  allocatedResources.resize(this->Properties[index]->ResourceGroups.size());
234 235
  for (auto const& it : allocations) {
    for (auto const& alloc : it.second) {
236
      bool result = this->ResourceAllocator.AllocateResource(
237 238 239
        it.first, alloc.Id, alloc.SlotsNeeded);
      (void)result;
      assert(result);
240
      allocatedResources[alloc.ProcessIndex][it.first].push_back(
241 242 243 244 245 246 247
        { alloc.Id, static_cast<unsigned int>(alloc.SlotsNeeded) });
    }
  }

  return true;
}

248
bool cmCTestMultiProcessHandler::TryAllocateResources(
249 250 251 252 253 254
  int index,
  std::map<std::string, std::vector<cmCTestBinPackerAllocation>>& allocations)
{
  allocations.clear();

  std::size_t processIndex = 0;
255
  for (auto const& process : this->Properties[index]->ResourceGroups) {
256 257 258 259 260 261 262 263 264
    for (auto const& requirement : process) {
      for (int i = 0; i < requirement.UnitsNeeded; ++i) {
        allocations[requirement.ResourceType].push_back(
          { processIndex, requirement.SlotsNeeded, "" });
      }
    }
    ++processIndex;
  }

265
  auto const& availableResources = this->ResourceAllocator.GetResources();
266
  for (auto& it : allocations) {
267
    if (!availableResources.count(it.first)) {
268 269
      return false;
    }
270 271
    if (!cmAllocateCTestResourcesRoundRobin(availableResources.at(it.first),
                                            it.second)) {
272 273 274 275 276 277 278
      return false;
    }
  }

  return true;
}

279
void cmCTestMultiProcessHandler::DeallocateResources(int index)
280
{
281
  if (!this->TestHandler->UseResourceSpec) {
282 283 284 285
    return;
  }

  {
286 287
    auto& allocatedResources = this->AllocatedResources[index];
    for (auto const& processAlloc : allocatedResources) {
288 289 290
      for (auto const& it : processAlloc) {
        auto resourceType = it.first;
        for (auto const& it2 : it.second) {
291
          bool success = this->ResourceAllocator.DeallocateResource(
292 293 294 295 296 297 298
            resourceType, it2.Id, it2.Slots);
          (void)success;
          assert(success);
        }
      }
    }
  }
299
  this->AllocatedResources.erase(index);
300 301
}

302
bool cmCTestMultiProcessHandler::AllResourcesAvailable()
303
{
304
  for (auto const& it : this->ResourceAllocator.GetResources()) {
305 306 307 308 309 310 311 312 313 314
    for (auto const& it2 : it.second) {
      if (it2.second.Locked != 0) {
        return false;
      }
    }
  }

  return true;
}

315
void cmCTestMultiProcessHandler::CheckResourcesAvailable()
316 317 318
{
  for (auto test : this->SortedTests) {
    std::map<std::string, std::vector<cmCTestBinPackerAllocation>> allocations;
319 320 321
    this->TestsHaveSufficientResources[test] =
      !this->TestHandler->UseResourceSpec ||
      this->TryAllocateResources(test, allocations);
322 323 324
  }
}

325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348
bool cmCTestMultiProcessHandler::CheckStopTimePassed()
{
  if (!this->StopTimePassed) {
    std::chrono::system_clock::time_point stop_time =
      this->CTest->GetStopTime();
    if (stop_time != std::chrono::system_clock::time_point() &&
        stop_time <= std::chrono::system_clock::now()) {
      this->SetStopTimePassed();
    }
  }
  return this->StopTimePassed;
}

void cmCTestMultiProcessHandler::SetStopTimePassed()
{
  if (!this->StopTimePassed) {
    cmCTestLog(this->CTest, ERROR_MESSAGE,
               "The stop time has been passed. "
               "Stopping all tests."
                 << std::endl);
    this->StopTimePassed = true;
  }
}

349
void cmCTestMultiProcessHandler::LockResources(int index)
Zach's avatar
Zach committed
350
{
351
  this->LockedResources.insert(
352 353
    this->Properties[index]->LockedResources.begin(),
    this->Properties[index]->LockedResources.end());
354

355
  if (this->Properties[index]->RunSerial) {
356
    this->SerialTestRunning = true;
357
  }
Zach's avatar
Zach committed
358 359
}

360
void cmCTestMultiProcessHandler::UnlockResources(int index)
Zach's avatar
Zach committed
361
{
362 363
  for (std::string const& i : this->Properties[index]->LockedResources) {
    this->LockedResources.erase(i);
364 365
  }
  if (this->Properties[index]->RunSerial) {
366
    this->SerialTestRunning = false;
367
  }
Zach's avatar
Zach committed
368 369
}

370 371 372
void cmCTestMultiProcessHandler::EraseTest(int test)
{
  this->Tests.erase(test);
Zach's avatar
Zach committed
373 374
  this->SortedTests.erase(
    std::find(this->SortedTests.begin(), this->SortedTests.end(), test));
375 376
}

377 378
inline size_t cmCTestMultiProcessHandler::GetProcessorsUsed(int test)
{
379 380 381 382
  size_t processors = static_cast<int>(this->Properties[test]->Processors);
  // If processors setting is set higher than the -j
  // setting, we default to using all of the process slots.
  if (processors > this->ParallelLevel) {
383
    processors = this->ParallelLevel;
384
  }
385 386 387 388 389
  // Cap tests that want affinity to the maximum affinity available.
  if (this->HaveAffinity && processors > this->HaveAffinity &&
      this->Properties[test]->WantAffinity) {
    processors = this->HaveAffinity;
  }
390 391 392
  return processors;
}

393 394 395 396 397
std::string cmCTestMultiProcessHandler::GetName(int test)
{
  return this->Properties[test]->Name;
}

398 399
bool cmCTestMultiProcessHandler::StartTest(int test)
{
400
  // Check for locked resources
401
  for (std::string const& i : this->Properties[test]->LockedResources) {
wahikihiki's avatar
wahikihiki committed
402
    if (cmContains(this->LockedResources, i)) {
Zach's avatar
Zach committed
403 404
      return false;
    }
405
  }
Zach's avatar
Zach committed
406

407 408 409 410
  // Allocate resources
  if (this->TestsHaveSufficientResources[test] &&
      !this->AllocateResources(test)) {
    this->DeallocateResources(test);
411 412 413
    return false;
  }

414
  // if there are no depends left then run this test
415
  if (this->Tests[test].empty()) {
416
    return this->StartTestProcess(test);
417
  }
418
  // This test was not able to start because it is waiting
419
  // on depends to run
420
  this->DeallocateResources(test);
421
  return false;
422 423 424 425
}

void cmCTestMultiProcessHandler::StartNextTests()
{
Brad King's avatar
Brad King committed
426 427 428 429 430
  if (this->TestLoadRetryTimer.get() != nullptr) {
    // This timer may be waiting to call StartNextTests again.
    // Since we have been called it is no longer needed.
    uv_timer_stop(this->TestLoadRetryTimer);
  }
431 432

  if (this->Tests.empty()) {
Brad King's avatar
Brad King committed
433
    this->TestLoadRetryTimer.reset();
434 435 436
    return;
  }

437 438 439 440
  if (this->CheckStopTimePassed()) {
    return;
  }

Brad King's avatar
Brad King committed
441 442
  size_t numToStart = 0;

443
  if (this->RunningCount < this->ParallelLevel) {
444
    numToStart = this->ParallelLevel - this->RunningCount;
445
  }
446

447
  if (numToStart == 0) {
448
    return;
449
  }
450

451 452
  // Don't start any new tests if one with the RUN_SERIAL property
  // is already running.
453
  if (this->SerialTestRunning) {
454
    return;
455
  }
456

457 458
  bool allTestsFailedTestLoadCheck = false;
  size_t minProcessorsRequired = this->ParallelLevel;
459
  std::string testWithMinProcessors;
460 461 462 463 464

  cmsys::SystemInformation info;

  unsigned long systemLoad = 0;
  size_t spareLoad = 0;
465
  if (this->TestLoad > 0) {
466 467 468 469
    // Activate possible wait.
    allTestsFailedTestLoadCheck = true;

    // Check for a fake load average value used in testing.
Brad King's avatar
Brad King committed
470 471 472 473 474
    if (this->FakeLoadForTesting > 0) {
      systemLoad = this->FakeLoadForTesting;
      // Drop the fake load for the next iteration to a value low enough
      // that the next iteration will start tests.
      this->FakeLoadForTesting = 1;
475
    }
476
    // If it's not set, look up the true load average.
477
    else {
478
      systemLoad = static_cast<unsigned long>(ceil(info.GetLoadAverage()));
479 480 481
    }
    spareLoad =
      (this->TestLoad > systemLoad ? this->TestLoad - systemLoad : 0);
482 483

    // Don't start more tests than the spare load can support.
484
    if (numToStart > spareLoad) {
485 486
      numToStart = spareLoad;
    }
487
  }
488

Zach's avatar
Zach committed
489
  TestList copy = this->SortedTests;
490
  for (auto const& test : copy) {
491
    // Take a nap if we're currently performing a RUN_SERIAL test.
492
    if (this->SerialTestRunning) {
493
      break;
494
    }
495
    // We can only start a RUN_SERIAL test if no other tests are also running.
496
    if (this->Properties[test]->RunSerial && this->RunningCount > 0) {
497
      continue;
498
    }
499

500
    size_t processors = GetProcessorsUsed(test);
501
    bool testLoadOk = true;
502 503
    if (this->TestLoad > 0) {
      if (processors <= spareLoad) {
504 505 506 507
        cmCTestLog(this->CTest, DEBUG,
                   "OK to run " << GetName(test) << ", it requires "
                                << processors << " procs & system load is: "
                                << systemLoad << std::endl);
508
        allTestsFailedTestLoadCheck = false;
509
      } else {
510 511
        testLoadOk = false;
      }
512
    }
513

514
    if (processors <= minProcessorsRequired) {
515
      minProcessorsRequired = processors;
516
      testWithMinProcessors = GetName(test);
517
    }
518

519
    if (testLoadOk && processors <= numToStart && this->StartTest(test)) {
520
      numToStart -= processors;
521
    } else if (numToStart == 0) {
522 523
      break;
    }
524
  }
525

526
  if (allTestsFailedTestLoadCheck) {
527 528 529 530 531 532 533 534
    // Find out whether there are any non RUN_SERIAL tests left, so that the
    // correct warning may be displayed.
    bool onlyRunSerialTestsLeft = true;
    for (auto const& test : copy) {
      if (!this->Properties[test]->RunSerial) {
        onlyRunSerialTestsLeft = false;
      }
    }
535
    cmCTestLog(this->CTest, HANDLER_OUTPUT, "***** WAITING, ");
536

537
    if (this->SerialTestRunning) {
538 539
      cmCTestLog(this->CTest, HANDLER_OUTPUT,
                 "Waiting for RUN_SERIAL test to finish.");
540 541 542
    } else if (onlyRunSerialTestsLeft) {
      cmCTestLog(this->CTest, HANDLER_OUTPUT,
                 "Only RUN_SERIAL tests remain, awaiting available slot.");
543
    } else {
544
      /* clang-format off */
545 546 547 548 549
      cmCTestLog(this->CTest, HANDLER_OUTPUT,
                 "System Load: " << systemLoad << ", "
                 "Max Allowed Load: " << this->TestLoad << ", "
                 "Smallest test " << testWithMinProcessors <<
                 " requires " << minProcessorsRequired);
550
      /* clang-format on */
551
    }
552 553
    cmCTestLog(this->CTest, HANDLER_OUTPUT, "*****" << std::endl);

Brad King's avatar
Brad King committed
554 555 556 557 558 559 560
    // Wait between 1 and 5 seconds before trying again.
    unsigned int milliseconds = (cmSystemTools::RandomSeed() % 5 + 1) * 1000;
    if (this->FakeLoadForTesting) {
      milliseconds = 10;
    }
    if (this->TestLoadRetryTimer.get() == nullptr) {
      this->TestLoadRetryTimer.init(this->Loop, this);
561
    }
Brad King's avatar
Brad King committed
562 563
    this->TestLoadRetryTimer.start(
      &cmCTestMultiProcessHandler::OnTestLoadRetryCB, milliseconds, 0);
564
  }
565 566
}

Brad King's avatar
Brad King committed
567 568 569 570 571 572
void cmCTestMultiProcessHandler::OnTestLoadRetryCB(uv_timer_t* timer)
{
  auto self = static_cast<cmCTestMultiProcessHandler*>(timer->data);
  self->StartNextTests();
}

573 574 575 576 577 578 579
void cmCTestMultiProcessHandler::FinishTestProcess(cmCTestRunTest* runner,
                                                   bool started)
{
  this->Completed++;

  int test = runner->GetIndex();
  auto properties = runner->GetTestProperties();
580

581
  bool testResult = runner->EndTest(this->Completed, this->Total, started);
582 583 584
  if (runner->TimedOutForStopTime()) {
    this->SetStopTimePassed();
  }
585
  if (started) {
586
    if (!this->StopTimePassed && runner->StartAgain(this->Completed)) {
587
      this->Completed--; // remove the completed test because run again
588
      return;
589 590
    }
  }
591 592 593 594 595 596 597 598 599 600 601 602 603 604

  if (testResult) {
    this->Passed->push_back(properties->Name);
  } else if (!properties->Disabled) {
    this->Failed->push_back(properties->Name);
  }

  for (auto& t : this->Tests) {
    t.second.erase(test);
  }

  this->TestFinishMap[test] = true;
  this->TestRunningMap[test] = false;
  this->WriteCheckpoint(test);
605
  this->DeallocateResources(test);
606
  this->UnlockResources(test);
607 608
  this->RunningCount -= GetProcessorsUsed(test);

609 610 611 612 613
  for (auto p : properties->Affinity) {
    this->ProcessorsAvailable.insert(p);
  }
  properties->Affinity.clear();

614
  delete runner;
615 616 617
  if (started) {
    this->StartNextTests();
  }
618 619
}

620
void cmCTestMultiProcessHandler::UpdateCostData()
621
{
622
  std::string fname = this->CTest->GetCostDataFile();
623
  std::string tmpout = fname + ".tmp";
624 625
  cmsys::ofstream fout;
  fout.open(tmpout.c_str());
626 627

  PropertiesMap temp = this->Properties;
628

629
  if (cmSystemTools::FileExists(fname)) {
630
    cmsys::ifstream fin;
631
    fin.open(fname.c_str());
632

633
    std::string line;
634
    while (std::getline(fin, line)) {
635
      if (line == "---") {
636
        break;
637
      }
638
      std::vector<std::string> parts = cmSystemTools::SplitString(line, ' ');
639
      // Format: <name> <previous_runs> <avg_cost>
640
      if (parts.size() < 3) {
641
        break;
642
      }
643 644 645 646 647 648

      std::string name = parts[0];
      int prev = atoi(parts[1].c_str());
      float cost = static_cast<float>(atof(parts[2].c_str()));

      int index = this->SearchByName(name);
649
      if (index == -1) {
650 651
        // This test is not in memory. We just rewrite the entry
        fout << name << " " << prev << " " << cost << "\n";
652
      } else {
653 654
        // Update with our new average cost
        fout << name << " " << this->Properties[index]->PreviousRuns << " "
655
             << this->Properties[index]->Cost << "\n";
656 657
        temp.erase(index);
      }
658
    }
659
    fin.close();
660
    cmSystemTools::RemoveFile(fname);
661
  }
662 663

  // Add all tests not previously listed in the file
664 665 666
  for (auto const& i : temp) {
    fout << i.second->Name << " " << i.second->PreviousRuns << " "
         << i.second->Cost << "\n";
667
  }
668 669 670

  // Write list of failed tests
  fout << "---\n";
671 672
  for (std::string const& f : *this->Failed) {
    fout << f << "\n";
673
  }
674
  fout.close();
675
  cmSystemTools::RenameFile(tmpout, fname);
676
}
677

678 679
void cmCTestMultiProcessHandler::ReadCostData()
{
680 681
  std::string fname = this->CTest->GetCostDataFile();

682
  if (cmSystemTools::FileExists(fname, true)) {
683
    cmsys::ifstream fin;
684 685
    fin.open(fname.c_str());
    std::string line;
686
    while (std::getline(fin, line)) {
687
      if (line == "---") {
688
        break;
689
      }
690

691
      std::vector<std::string> parts = cmSystemTools::SplitString(line, ' ');
692 693

      // Probably an older version of the file, will be fixed next run
694
      if (parts.size() < 3) {
Zach's avatar
Zach committed
695 696
        fin.close();
        return;
697
      }
698 699 700 701 702 703

      std::string name = parts[0];
      int prev = atoi(parts[1].c_str());
      float cost = static_cast<float>(atof(parts[2].c_str()));

      int index = this->SearchByName(name);
704
      if (index == -1) {
705
        continue;
706
      }
707 708

      this->Properties[index]->PreviousRuns = prev;
709
      // When not running in parallel mode, don't use cost data
710 711
      if (this->ParallelLevel > 1 && this->Properties[index] &&
          this->Properties[index]->Cost == 0) {
712 713
        this->Properties[index]->Cost = cost;
      }
714
    }
715
    // Next part of the file is the failed tests
716
    while (std::getline(fin, line)) {
717
      if (!line.empty()) {
718 719
        this->LastTestsFailed.push_back(line);
      }
720
    }
721 722
    fin.close();
  }
723 724
}

725
int cmCTestMultiProcessHandler::SearchByName(std::string const& name)
726
{
727 728
  int index = -1;

729 730 731
  for (auto const& p : this->Properties) {
    if (p.second->Name == name) {
      index = p.first;
732
    }
733
  }
734
  return index;
735 736
}

737
void cmCTestMultiProcessHandler::CreateTestCostList()
738
{
739
  if (this->ParallelLevel > 1) {
740
    CreateParallelTestCostList();
741
  } else {
742
    CreateSerialTestCostList();
743
  }
744 745 746
}

void cmCTestMultiProcessHandler::CreateParallelTestCostList()
747
{
748 749
  TestSet alreadySortedTests;

750
  std::list<TestSet> priorityStack;
751
  priorityStack.emplace_back();
752
  TestSet& topLevel = priorityStack.back();
753

754 755
  // In parallel test runs add previously failed tests to the front
  // of the cost list and queue other tests for further sorting
756
  for (auto const& t : this->Tests) {
wahikihiki's avatar
wahikihiki committed
757
    if (cmContains(this->LastTestsFailed, this->Properties[t.first]->Name)) {
758
      // If the test failed last time, it should be run first.
759 760
      this->SortedTests.push_back(t.first);
      a