Commit c544cb66 authored by Craig Scott's avatar Craig Scott Committed by Brad King

CTest: Rename hardware -> resources for source code

parent 6ce27d3a
......@@ -919,8 +919,8 @@ set(CTEST_SRCS cmCTest.cxx
CTest/cmCTestEmptyBinaryDirectoryCommand.cxx
CTest/cmCTestGenericHandler.cxx
CTest/cmCTestHandlerCommand.cxx
CTest/cmCTestHardwareAllocator.cxx
CTest/cmCTestHardwareSpec.cxx
CTest/cmCTestResourceAllocator.cxx
CTest/cmCTestResourceSpec.cxx
CTest/cmCTestLaunch.cxx
CTest/cmCTestMemCheckCommand.cxx
CTest/cmCTestMemCheckHandler.cxx
......
......@@ -23,7 +23,7 @@ namespace {
/*
* The following algorithm is used to do two things:
*
* 1) Determine if a test's hardware requirements can fit within the hardware
* 1) Determine if a test's resource requirements can fit within the resources
* present on the system, and
* 2) Do the actual allocation
*
......@@ -34,46 +34,46 @@ namespace {
* more combinations can be tried.
*/
template <typename AllocationStrategy>
static bool AllocateCTestHardware(
const std::map<std::string, cmCTestHardwareAllocator::Resource>& hardware,
const std::vector<std::string>& hardwareSorted, std::size_t currentIndex,
static bool AllocateCTestResources(
const std::map<std::string, cmCTestResourceAllocator::Resource>& resources,
const std::vector<std::string>& resourcesSorted, std::size_t currentIndex,
std::vector<cmCTestBinPackerAllocation*>& allocations)
{
// Iterate through all large enough resources until we find a solution
std::size_t hardwareIndex = 0;
while (hardwareIndex < hardwareSorted.size()) {
auto const& resource = hardware.at(hardwareSorted[hardwareIndex]);
std::size_t resourceIndex = 0;
while (resourceIndex < resourcesSorted.size()) {
auto const& resource = resources.at(resourcesSorted[resourceIndex]);
if (resource.Free() >=
static_cast<unsigned int>(allocations[currentIndex]->SlotsNeeded)) {
// Preemptively allocate the resource
allocations[currentIndex]->Id = hardwareSorted[hardwareIndex];
allocations[currentIndex]->Id = resourcesSorted[resourceIndex];
if (currentIndex + 1 >= allocations.size()) {
// We have a solution
return true;
}
// Move the resource up the list until it is sorted again
auto hardware2 = hardware;
auto hardwareSorted2 = hardwareSorted;
hardware2[hardwareSorted2[hardwareIndex]].Locked +=
auto resources2 = resources;
auto resourcesSorted2 = resourcesSorted;
resources2[resourcesSorted2[resourceIndex]].Locked +=
allocations[currentIndex]->SlotsNeeded;
AllocationStrategy::IncrementalSort(hardware2, hardwareSorted2,
hardwareIndex);
AllocationStrategy::IncrementalSort(resources2, resourcesSorted2,
resourceIndex);
// Recurse one level deeper
if (AllocateCTestHardware<AllocationStrategy>(
hardware2, hardwareSorted2, currentIndex + 1, allocations)) {
if (AllocateCTestResources<AllocationStrategy>(
resources2, resourcesSorted2, currentIndex + 1, allocations)) {
return true;
}
}
// No solution found here, deallocate the resource and try the next one
allocations[currentIndex]->Id.clear();
auto freeSlots = hardware.at(hardwareSorted.at(hardwareIndex)).Free();
auto freeSlots = resources.at(resourcesSorted.at(resourceIndex)).Free();
do {
++hardwareIndex;
} while (hardwareIndex < hardwareSorted.size() &&
hardware.at(hardwareSorted.at(hardwareIndex)).Free() ==
++resourceIndex;
} while (resourceIndex < resourcesSorted.size() &&
resources.at(resourcesSorted.at(resourceIndex)).Free() ==
freeSlots);
}
......@@ -82,8 +82,8 @@ static bool AllocateCTestHardware(
}
template <typename AllocationStrategy>
static bool AllocateCTestHardware(
const std::map<std::string, cmCTestHardwareAllocator::Resource>& hardware,
static bool AllocateCTestResources(
const std::map<std::string, cmCTestResourceAllocator::Resource>& resources,
std::vector<cmCTestBinPackerAllocation>& allocations)
{
// Sort the resource requirements in descending order by slots needed
......@@ -99,103 +99,105 @@ static bool AllocateCTestHardware(
});
// Sort the resources according to sort strategy
std::vector<std::string> hardwareSorted;
hardwareSorted.reserve(hardware.size());
for (auto const& hw : hardware) {
hardwareSorted.push_back(hw.first);
std::vector<std::string> resourcesSorted;
resourcesSorted.reserve(resources.size());
for (auto const& res : resources) {
resourcesSorted.push_back(res.first);
}
AllocationStrategy::InitialSort(hardware, hardwareSorted);
AllocationStrategy::InitialSort(resources, resourcesSorted);
// Do the actual allocation
return AllocateCTestHardware<AllocationStrategy>(
hardware, hardwareSorted, std::size_t(0), allocationsPtr);
return AllocateCTestResources<AllocationStrategy>(
resources, resourcesSorted, std::size_t(0), allocationsPtr);
}
class RoundRobinAllocationStrategy
{
public:
static void InitialSort(
const std::map<std::string, cmCTestHardwareAllocator::Resource>& hardware,
std::vector<std::string>& hardwareSorted);
const std::map<std::string, cmCTestResourceAllocator::Resource>& resources,
std::vector<std::string>& resourcesSorted);
static void IncrementalSort(
const std::map<std::string, cmCTestHardwareAllocator::Resource>& hardware,
std::vector<std::string>& hardwareSorted, std::size_t lastAllocatedIndex);
const std::map<std::string, cmCTestResourceAllocator::Resource>& resources,
std::vector<std::string>& resourcesSorted, std::size_t lastAllocatedIndex);
};
void RoundRobinAllocationStrategy::InitialSort(
const std::map<std::string, cmCTestHardwareAllocator::Resource>& hardware,
std::vector<std::string>& hardwareSorted)
const std::map<std::string, cmCTestResourceAllocator::Resource>& resources,
std::vector<std::string>& resourcesSorted)
{
std::stable_sort(
hardwareSorted.rbegin(), hardwareSorted.rend(),
[&hardware](const std::string& id1, const std::string& id2) {
return hardware.at(id1).Free() < hardware.at(id2).Free();
resourcesSorted.rbegin(), resourcesSorted.rend(),
[&resources](const std::string& id1, const std::string& id2) {
return resources.at(id1).Free() < resources.at(id2).Free();
});
}
void RoundRobinAllocationStrategy::IncrementalSort(
const std::map<std::string, cmCTestHardwareAllocator::Resource>& hardware,
std::vector<std::string>& hardwareSorted, std::size_t lastAllocatedIndex)
const std::map<std::string, cmCTestResourceAllocator::Resource>& resources,
std::vector<std::string>& resourcesSorted, std::size_t lastAllocatedIndex)
{
auto tmp = hardwareSorted[lastAllocatedIndex];
auto tmp = resourcesSorted[lastAllocatedIndex];
std::size_t i = lastAllocatedIndex;
while (i < hardwareSorted.size() - 1 &&
hardware.at(hardwareSorted[i + 1]).Free() > hardware.at(tmp).Free()) {
hardwareSorted[i] = hardwareSorted[i + 1];
while (i < resourcesSorted.size() - 1 &&
resources.at(resourcesSorted[i + 1]).Free() >
resources.at(tmp).Free()) {
resourcesSorted[i] = resourcesSorted[i + 1];
++i;
}
hardwareSorted[i] = tmp;
resourcesSorted[i] = tmp;
}
class BlockAllocationStrategy
{
public:
static void InitialSort(
const std::map<std::string, cmCTestHardwareAllocator::Resource>& hardware,
std::vector<std::string>& hardwareSorted);
const std::map<std::string, cmCTestResourceAllocator::Resource>& resources,
std::vector<std::string>& resourcesSorted);
static void IncrementalSort(
const std::map<std::string, cmCTestHardwareAllocator::Resource>& hardware,
std::vector<std::string>& hardwareSorted, std::size_t lastAllocatedIndex);
const std::map<std::string, cmCTestResourceAllocator::Resource>& resources,
std::vector<std::string>& resourcesSorted, std::size_t lastAllocatedIndex);
};
void BlockAllocationStrategy::InitialSort(
const std::map<std::string, cmCTestHardwareAllocator::Resource>& hardware,
std::vector<std::string>& hardwareSorted)
const std::map<std::string, cmCTestResourceAllocator::Resource>& resources,
std::vector<std::string>& resourcesSorted)
{
std::stable_sort(
hardwareSorted.rbegin(), hardwareSorted.rend(),
[&hardware](const std::string& id1, const std::string& id2) {
return hardware.at(id1).Free() < hardware.at(id2).Free();
resourcesSorted.rbegin(), resourcesSorted.rend(),
[&resources](const std::string& id1, const std::string& id2) {
return resources.at(id1).Free() < resources.at(id2).Free();
});
}
void BlockAllocationStrategy::IncrementalSort(
const std::map<std::string, cmCTestHardwareAllocator::Resource>&,
std::vector<std::string>& hardwareSorted, std::size_t lastAllocatedIndex)
const std::map<std::string, cmCTestResourceAllocator::Resource>&,
std::vector<std::string>& resourcesSorted, std::size_t lastAllocatedIndex)
{
auto tmp = hardwareSorted[lastAllocatedIndex];
auto tmp = resourcesSorted[lastAllocatedIndex];
std::size_t i = lastAllocatedIndex;
while (i > 0) {
hardwareSorted[i] = hardwareSorted[i - 1];
resourcesSorted[i] = resourcesSorted[i - 1];
--i;
}
hardwareSorted[i] = tmp;
resourcesSorted[i] = tmp;
}
}
bool cmAllocateCTestHardwareRoundRobin(
const std::map<std::string, cmCTestHardwareAllocator::Resource>& hardware,
bool cmAllocateCTestResourcesRoundRobin(
const std::map<std::string, cmCTestResourceAllocator::Resource>& resources,
std::vector<cmCTestBinPackerAllocation>& allocations)
{
return AllocateCTestHardware<RoundRobinAllocationStrategy>(hardware,
allocations);
return AllocateCTestResources<RoundRobinAllocationStrategy>(resources,
allocations);
}
bool cmAllocateCTestHardwareBlock(
const std::map<std::string, cmCTestHardwareAllocator::Resource>& hardware,
bool cmAllocateCTestResourcesBlock(
const std::map<std::string, cmCTestResourceAllocator::Resource>& resources,
std::vector<cmCTestBinPackerAllocation>& allocations)
{
return AllocateCTestHardware<BlockAllocationStrategy>(hardware, allocations);
return AllocateCTestResources<BlockAllocationStrategy>(resources,
allocations);
}
......@@ -8,7 +8,7 @@
#include <string>
#include <vector>
#include "cmCTestHardwareAllocator.h"
#include "cmCTestResourceAllocator.h"
struct cmCTestBinPackerAllocation
{
......@@ -20,12 +20,12 @@ struct cmCTestBinPackerAllocation
bool operator!=(const cmCTestBinPackerAllocation& other) const;
};
bool cmAllocateCTestHardwareRoundRobin(
const std::map<std::string, cmCTestHardwareAllocator::Resource>& hardware,
bool cmAllocateCTestResourcesRoundRobin(
const std::map<std::string, cmCTestResourceAllocator::Resource>& resources,
std::vector<cmCTestBinPackerAllocation>& allocations);
bool cmAllocateCTestHardwareBlock(
const std::map<std::string, cmCTestHardwareAllocator::Resource>& hardware,
bool cmAllocateCTestResourcesBlock(
const std::map<std::string, cmCTestResourceAllocator::Resource>& resources,
std::vector<cmCTestBinPackerAllocation>& allocations);
#endif
......@@ -140,7 +140,7 @@ void cmCTestMultiProcessHandler::RunTests()
assert(this->Completed == this->Total);
assert(this->Tests.empty());
}
assert(this->AllHardwareAvailable());
assert(this->AllResourcesAvailable());
this->MarkFinished();
this->UpdateCostData();
......@@ -177,9 +177,9 @@ bool cmCTestMultiProcessHandler::StartTestProcess(int test)
}
testRun->SetIndex(test);
testRun->SetTestProperties(this->Properties[test]);
if (this->TestHandler->UseHardwareSpec) {
testRun->SetUseAllocatedHardware(true);
testRun->SetAllocatedHardware(this->AllocatedHardware[test]);
if (this->TestHandler->UseResourceSpec) {
testRun->SetUseAllocatedResources(true);
testRun->SetAllocatedResources(this->AllocatedResources[test]);
}
// Find any failed dependencies for this test. We assume the more common
......@@ -194,7 +194,7 @@ bool cmCTestMultiProcessHandler::StartTestProcess(int test)
// working directory because FinishTestProcess() will try to unlock them
this->LockResources(test);
if (!this->TestsHaveSufficientHardware[test]) {
if (!this->TestsHaveSufficientResources[test]) {
testRun->StartFailure("Insufficient resources");
this->FinishTestProcess(testRun, false);
return false;
......@@ -218,26 +218,26 @@ bool cmCTestMultiProcessHandler::StartTestProcess(int test)
return false;
}
bool cmCTestMultiProcessHandler::AllocateHardware(int index)
bool cmCTestMultiProcessHandler::AllocateResources(int index)
{
if (!this->TestHandler->UseHardwareSpec) {
if (!this->TestHandler->UseResourceSpec) {
return true;
}
std::map<std::string, std::vector<cmCTestBinPackerAllocation>> allocations;
if (!this->TryAllocateHardware(index, allocations)) {
if (!this->TryAllocateResources(index, allocations)) {
return false;
}
auto& allocatedHardware = this->AllocatedHardware[index];
allocatedHardware.resize(this->Properties[index]->ResourceGroups.size());
auto& allocatedResources = this->AllocatedResources[index];
allocatedResources.resize(this->Properties[index]->ResourceGroups.size());
for (auto const& it : allocations) {
for (auto const& alloc : it.second) {
bool result = this->HardwareAllocator.AllocateResource(
bool result = this->ResourceAllocator.AllocateResource(
it.first, alloc.Id, alloc.SlotsNeeded);
(void)result;
assert(result);
allocatedHardware[alloc.ProcessIndex][it.first].push_back(
allocatedResources[alloc.ProcessIndex][it.first].push_back(
{ alloc.Id, static_cast<unsigned int>(alloc.SlotsNeeded) });
}
}
......@@ -245,7 +245,7 @@ bool cmCTestMultiProcessHandler::AllocateHardware(int index)
return true;
}
bool cmCTestMultiProcessHandler::TryAllocateHardware(
bool cmCTestMultiProcessHandler::TryAllocateResources(
int index,
std::map<std::string, std::vector<cmCTestBinPackerAllocation>>& allocations)
{
......@@ -262,13 +262,13 @@ bool cmCTestMultiProcessHandler::TryAllocateHardware(
++processIndex;
}
auto const& availableHardware = this->HardwareAllocator.GetResources();
auto const& availableResources = this->ResourceAllocator.GetResources();
for (auto& it : allocations) {
if (!availableHardware.count(it.first)) {
if (!availableResources.count(it.first)) {
return false;
}
if (!cmAllocateCTestHardwareRoundRobin(availableHardware.at(it.first),
it.second)) {
if (!cmAllocateCTestResourcesRoundRobin(availableResources.at(it.first),
it.second)) {
return false;
}
}
......@@ -276,19 +276,19 @@ bool cmCTestMultiProcessHandler::TryAllocateHardware(
return true;
}
void cmCTestMultiProcessHandler::DeallocateHardware(int index)
void cmCTestMultiProcessHandler::DeallocateResources(int index)
{
if (!this->TestHandler->UseHardwareSpec) {
if (!this->TestHandler->UseResourceSpec) {
return;
}
{
auto& allocatedHardware = this->AllocatedHardware[index];
for (auto const& processAlloc : allocatedHardware) {
auto& allocatedResources = this->AllocatedResources[index];
for (auto const& processAlloc : allocatedResources) {
for (auto const& it : processAlloc) {
auto resourceType = it.first;
for (auto const& it2 : it.second) {
bool success = this->HardwareAllocator.DeallocateResource(
bool success = this->ResourceAllocator.DeallocateResource(
resourceType, it2.Id, it2.Slots);
(void)success;
assert(success);
......@@ -296,12 +296,12 @@ void cmCTestMultiProcessHandler::DeallocateHardware(int index)
}
}
}
this->AllocatedHardware.erase(index);
this->AllocatedResources.erase(index);
}
bool cmCTestMultiProcessHandler::AllHardwareAvailable()
bool cmCTestMultiProcessHandler::AllResourcesAvailable()
{
for (auto const& it : this->HardwareAllocator.GetResources()) {
for (auto const& it : this->ResourceAllocator.GetResources()) {
for (auto const& it2 : it.second) {
if (it2.second.Locked != 0) {
return false;
......@@ -312,13 +312,13 @@ bool cmCTestMultiProcessHandler::AllHardwareAvailable()
return true;
}
void cmCTestMultiProcessHandler::CheckHardwareAvailable()
void cmCTestMultiProcessHandler::CheckResourcesAvailable()
{
for (auto test : this->SortedTests) {
std::map<std::string, std::vector<cmCTestBinPackerAllocation>> allocations;
this->TestsHaveSufficientHardware[test] =
!this->TestHandler->UseHardwareSpec ||
this->TryAllocateHardware(test, allocations);
this->TestsHaveSufficientResources[test] =
!this->TestHandler->UseResourceSpec ||
this->TryAllocateResources(test, allocations);
}
}
......@@ -404,10 +404,10 @@ bool cmCTestMultiProcessHandler::StartTest(int test)
}
}
// Allocate hardware
if (this->TestsHaveSufficientHardware[test] &&
!this->AllocateHardware(test)) {
this->DeallocateHardware(test);
// Allocate resources
if (this->TestsHaveSufficientResources[test] &&
!this->AllocateResources(test)) {
this->DeallocateResources(test);
return false;
}
......@@ -417,7 +417,7 @@ bool cmCTestMultiProcessHandler::StartTest(int test)
}
// This test was not able to start because it is waiting
// on depends to run
this->DeallocateHardware(test);
this->DeallocateResources(test);
return false;
}
......@@ -602,7 +602,7 @@ void cmCTestMultiProcessHandler::FinishTestProcess(cmCTestRunTest* runner,
this->TestFinishMap[test] = true;
this->TestRunningMap[test] = false;
this->WriteCheckpoint(test);
this->DeallocateHardware(test);
this->DeallocateResources(test);
this->UnlockResources(test);
this->RunningCount -= GetProcessorsUsed(test);
......
......@@ -14,13 +14,13 @@
#include "cm_uv.h"
#include "cmCTestHardwareAllocator.h"
#include "cmCTestResourceAllocator.h"
#include "cmCTestTestHandler.h"
#include "cmUVHandlePtr.h"
class cmCTest;
struct cmCTestBinPackerAllocation;
class cmCTestHardwareSpec;
class cmCTestResourceSpec;
class cmCTestRunTest;
/** \class cmCTestMultiProcessHandler
......@@ -47,7 +47,7 @@ public:
: public std::map<int, cmCTestTestHandler::cmCTestTestProperties*>
{
};
struct HardwareAllocation
struct ResourceAllocation
{
std::string Id;
unsigned int Slots;
......@@ -87,12 +87,12 @@ public:
void SetQuiet(bool b) { this->Quiet = b; }
void InitHardwareAllocator(const cmCTestHardwareSpec& spec)
void InitResourceAllocator(const cmCTestResourceSpec& spec)
{
this->HardwareAllocator.InitializeFromHardwareSpec(spec);
this->ResourceAllocator.InitializeFromResourceSpec(spec);
}
void CheckHardwareAvailable();
void CheckResourcesAvailable();
protected:
// Start the next test or tests as many as are allowed by
......@@ -137,13 +137,13 @@ protected:
void LockResources(int index);
void UnlockResources(int index);
bool AllocateHardware(int index);
bool TryAllocateHardware(
bool AllocateResources(int index);
bool TryAllocateResources(
int index,
std::map<std::string, std::vector<cmCTestBinPackerAllocation>>&
allocations);
void DeallocateHardware(int index);
bool AllHardwareAvailable();
void DeallocateResources(int index);
bool AllResourcesAvailable();
// map from test number to set of depend tests
TestMap Tests;
......@@ -166,10 +166,10 @@ protected:
std::vector<std::string> LastTestsFailed;
std::set<std::string> LockedResources;
std::map<int,
std::vector<std::map<std::string, std::vector<HardwareAllocation>>>>
AllocatedHardware;
std::map<int, bool> TestsHaveSufficientHardware;
cmCTestHardwareAllocator HardwareAllocator;
std::vector<std::map<std::string, std::vector<ResourceAllocation>>>>
AllocatedResources;
std::map<int, bool> TestsHaveSufficientResources;
cmCTestResourceAllocator ResourceAllocator;
std::vector<cmCTestTestHandler::cmCTestTestResult>* TestResults;
size_t ParallelLevel; // max number of process that can be run at once
unsigned long TestLoad;
......
/* Distributed under the OSI-approved BSD 3-Clause License. See accompanying
file Copyright.txt or https://cmake.org/licensing for details. */
#include "cmCTestHardwareAllocator.h"
#include "cmCTestResourceAllocator.h"
#include <utility>
#include <vector>
#include "cmCTestHardwareSpec.h"
#include "cmCTestResourceSpec.h"
void cmCTestHardwareAllocator::InitializeFromHardwareSpec(
const cmCTestHardwareSpec& spec)
void cmCTestResourceAllocator::InitializeFromResourceSpec(
const cmCTestResourceSpec& spec)
{
this->Resources.clear();
......@@ -23,13 +23,13 @@ void cmCTestHardwareAllocator::InitializeFromHardwareSpec(
}
const std::map<std::string,
std::map<std::string, cmCTestHardwareAllocator::Resource>>&
cmCTestHardwareAllocator::GetResources() const
std::map<std::string, cmCTestResourceAllocator::Resource>>&
cmCTestResourceAllocator::GetResources() const
{
return this->Resources;
}
bool cmCTestHardwareAllocator::AllocateResource(const std::string& name,
bool cmCTestResourceAllocator::AllocateResource(const std::string& name,
const std::string& id,
unsigned int slots)
{
......@@ -51,7 +51,7 @@ bool cmCTestHardwareAllocator::AllocateResource(const std::string& name,
return true;
}
bool cmCTestHardwareAllocator::DeallocateResource(const std::string& name,
bool cmCTestResourceAllocator::DeallocateResource(const std::string& name,
const std::string& id,
unsigned int slots)
{
......@@ -73,13 +73,13 @@ bool cmCTestHardwareAllocator::DeallocateResource(const std::string& name,
return true;
}
bool cmCTestHardwareAllocator::Resource::operator==(
bool cmCTestResourceAllocator::Resource::operator==(
const Resource& other) const
{
return this->Total == other.Total && this->Locked == other.Locked;
}
bool cmCTestHardwareAllocator::Resource::operator!=(