Commit 838cb433 authored by Robert Maynard's avatar Robert Maynard
Browse files

Redesign RuntimeDeviceTracker and RuntimeDeviceInformation

The RuntimeDeviceTracker had grown organically to handle multiple
different roles inside VTK-m. Now that we have device tags
that can be passed around at runtime, large portions of
the RuntimeDeviceTracker API aren't needed.

Additionally the RuntimeDeviceTracker had a dependency on knowing
the names of each device, and this wasn't possible
as that information was part of its self. Now we have moved that
information into RuntimeDeviceInformation and have broken
the recursion.
parent 256e0c3c
# Redesign Runtime Device Tracking
The device tracking infrastructure in VTK-m has been redesigned to
remove multiple redundant codes paths and to simplify reasoning
about around what an instance of RuntimeDeviceTracker will modify.
......@@ -90,7 +90,7 @@ struct VtkmArg : public opt::Arg
return false;
}
auto tracker = vtkm::cont::GetRuntimeDeviceTracker();
auto& tracker = vtkm::cont::GetRuntimeDeviceTracker();
bool result = false;
try
{
......@@ -180,7 +180,7 @@ InitializeResult Initialize(int& argc, char* argv[], InitializeOptions opts)
if (options[DEVICE])
{
auto id = vtkm::cont::make_DeviceAdapterId(options[DEVICE].arg);
auto tracker = vtkm::cont::GetRuntimeDeviceTracker();
auto& tracker = vtkm::cont::GetRuntimeDeviceTracker();
tracker.ForceDevice(id);
config.Device = id;
}
......
......@@ -22,6 +22,7 @@
#include <vtkm/ListTag.h>
#include <vtkm/cont/DeviceAdapter.h>
#include <vtkm/cont/DeviceAdapterListTag.h>
#include <vtkm/cont/internal/DeviceAdapterTag.h>
//Bring in each device adapters runtime class
#include <vtkm/cont/cuda/internal/DeviceAdapterRuntimeDetectorCuda.h>
......@@ -30,17 +31,49 @@
#include <vtkm/cont/serial/internal/DeviceAdapterRuntimeDetectorSerial.h>
#include <vtkm/cont/tbb/internal/DeviceAdapterRuntimeDetectorTBB.h>
namespace vtkm
{
namespace cont
#include <cctype> //for tolower
namespace
{
namespace detail
struct VTKM_NEVER_EXPORT InitializeDeviceNames
{
struct RuntimeDeviceInformationFunctor
vtkm::cont::DeviceAdapterNameType* Names;
vtkm::cont::DeviceAdapterNameType* LowerCaseNames;
VTKM_CONT
InitializeDeviceNames(vtkm::cont::DeviceAdapterNameType* names,
vtkm::cont::DeviceAdapterNameType* lower)
: Names(names)
, LowerCaseNames(lower)
{
std::fill_n(this->Names, VTKM_MAX_DEVICE_ADAPTER_ID, "InvalidDeviceId");
std::fill_n(this->LowerCaseNames, VTKM_MAX_DEVICE_ADAPTER_ID, "invaliddeviceid");
}
template <typename Device>
VTKM_CONT void operator()(Device device)
{
auto lowerCaseFunc = [](char c) {
return static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
};
auto id = device.GetValue();
if (id > 0 && id < VTKM_MAX_DEVICE_ADAPTER_ID)
{
auto name = vtkm::cont::DeviceAdapterTraits<Device>::GetName();
this->Names[id] = name;
std::transform(name.begin(), name.end(), name.begin(), lowerCaseFunc);
this->LowerCaseNames[id] = name;
}
}
};
struct VTKM_NEVER_EXPORT RuntimeDeviceInformationFunctor
{
bool Exists = false;
template <typename DeviceAdapter>
VTKM_CONT void operator()(DeviceAdapter, DeviceAdapterId device)
VTKM_CONT void operator()(DeviceAdapter, vtkm::cont::DeviceAdapterId device)
{
if (DeviceAdapter() == device)
{
......@@ -50,10 +83,95 @@ struct RuntimeDeviceInformationFunctor
};
}
namespace vtkm
{
namespace cont
{
namespace detail
{
struct RuntimeDeviceNames
{
RuntimeDeviceNames()
{
InitializeDeviceNames functor(DeviceNames, LowerCaseDeviceNames);
vtkm::ListForEach(functor, VTKM_DEFAULT_DEVICE_ADAPTER_LIST_TAG());
}
DeviceAdapterNameType DeviceNames[VTKM_MAX_DEVICE_ADAPTER_ID];
DeviceAdapterNameType LowerCaseDeviceNames[VTKM_MAX_DEVICE_ADAPTER_ID];
};
thread_local RuntimeDeviceNames RuntimeNames;
}
VTKM_CONT
DeviceAdapterNameType RuntimeDeviceInformation::GetName(DeviceAdapterId device) const
{
const auto id = device.GetValue();
if (device.IsValueValid())
{
return detail::RuntimeNames.DeviceNames[id];
}
else if (id == VTKM_DEVICE_ADAPTER_ERROR)
{
return vtkm::cont::DeviceAdapterTraits<vtkm::cont::DeviceAdapterTagError>::GetName();
}
else if (id == VTKM_DEVICE_ADAPTER_UNDEFINED)
{
return vtkm::cont::DeviceAdapterTraits<vtkm::cont::DeviceAdapterTagUndefined>::GetName();
}
else if (id == VTKM_DEVICE_ADAPTER_ANY)
{
return vtkm::cont::DeviceAdapterTraits<vtkm::cont::DeviceAdapterTagAny>::GetName();
}
// Deviceis invalid:
return detail::RuntimeNames.DeviceNames[0];
}
VTKM_CONT
DeviceAdapterId RuntimeDeviceInformation::GetId(DeviceAdapterNameType name) const
{
// The GetDeviceAdapterId call is case-insensitive so transform the name to be lower case
// as that is how we cache the case-insensitive version.
auto lowerCaseFunc = [](char c) {
return static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
};
std::transform(name.begin(), name.end(), name.begin(), lowerCaseFunc);
//lower-case the name here
if (name == "any")
{
return vtkm::cont::DeviceAdapterTagAny{};
}
else if (name == "error")
{
return vtkm::cont::DeviceAdapterTagError{};
}
else if (name == "undefined")
{
return vtkm::cont::DeviceAdapterTagUndefined{};
}
for (vtkm::Int8 id = 0; id < VTKM_MAX_DEVICE_ADAPTER_ID; ++id)
{
if (name == detail::RuntimeNames.LowerCaseDeviceNames[id])
{
return vtkm::cont::make_DeviceAdapterId(id);
}
}
return vtkm::cont::DeviceAdapterTagUndefined{};
}
VTKM_CONT
bool RuntimeDeviceInformation::Exists(DeviceAdapterId id) const
{
detail::RuntimeDeviceInformationFunctor functor;
RuntimeDeviceInformationFunctor functor;
vtkm::ListForEach(functor, VTKM_DEFAULT_DEVICE_ADAPTER_LIST_TAG(), id);
return functor.Exists;
}
......
......@@ -30,16 +30,23 @@ namespace cont
/// A class that can be used to determine if a given device adapter
/// is supported on the current machine at runtime. This is very important
/// for device adapters that a physical hardware requirements such as a GPU
/// or a Accelerator Card.
/// for device adapters where a physical hardware requirements such as a GPU
/// or a Accelerator Card is needed for support to exist.
///
///
class VTKM_CONT_EXPORT RuntimeDeviceInformation
{
public:
/// Returns the name corresponding to the device adapter id. If @a id is
/// not recognized, `InvalidDeviceId` is returned. Queries for a
/// name are all case-insensitive.
VTKM_CONT
DeviceAdapterNameType GetName(DeviceAdapterId id) const;
/// Returns the id corresponding to the device adapter name. If @a name is
/// not recognized, DeviceAdapterTagUndefined is returned.
VTKM_CONT
RuntimeDeviceInformation() {}
DeviceAdapterId GetId(DeviceAdapterNameType name) const;
/// Returns true if the given device adapter is supported on the current
/// machine.
......
......@@ -38,55 +38,11 @@
#include <algorithm>
#include <cctype> //for tolower
#include <map>
#include <mutex>
#include <sstream>
#include <thread>
namespace
{
struct VTKM_NEVER_EXPORT GetDeviceNameFunctor
{
vtkm::cont::DeviceAdapterNameType* Names;
vtkm::cont::DeviceAdapterNameType* LowerCaseNames;
VTKM_CONT
GetDeviceNameFunctor(vtkm::cont::DeviceAdapterNameType* names,
vtkm::cont::DeviceAdapterNameType* lower)
: Names(names)
, LowerCaseNames(lower)
{
std::fill_n(this->Names, VTKM_MAX_DEVICE_ADAPTER_ID, "InvalidDeviceId");
std::fill_n(this->LowerCaseNames, VTKM_MAX_DEVICE_ADAPTER_ID, "invaliddeviceid");
}
template <typename Device>
VTKM_CONT void operator()(Device device)
{
auto lowerCaseFunc = [](char c) {
return static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
};
auto id = device.GetValue();
if (id > 0 && id < VTKM_MAX_DEVICE_ADAPTER_ID)
{
auto name = vtkm::cont::DeviceAdapterTraits<Device>::GetName();
this->Names[id] = name;
std::transform(name.begin(), name.end(), name.begin(), lowerCaseFunc);
this->LowerCaseNames[id] = name;
}
}
};
#if !(defined(VTKM_CLANG) && (__apple_build_version__ < 8000000))
thread_local static vtkm::cont::RuntimeDeviceTracker runtimeDeviceTracker;
#endif
} // end anon namespace
namespace vtkm
{
namespace cont
......@@ -97,15 +53,13 @@ namespace detail
struct RuntimeDeviceTrackerInternals
{
bool RuntimeValid[VTKM_MAX_DEVICE_ADAPTER_ID];
DeviceAdapterNameType DeviceNames[VTKM_MAX_DEVICE_ADAPTER_ID];
DeviceAdapterNameType LowerCaseDeviceNames[VTKM_MAX_DEVICE_ADAPTER_ID];
bool RuntimeAllowed[VTKM_MAX_DEVICE_ADAPTER_ID];
};
struct RuntimeDeviceTrackerFunctor
{
template <typename DeviceAdapter>
VTKM_CONT void operator()(DeviceAdapter, DeviceAdapterId id, RuntimeDeviceTracker* rdt)
VTKM_CONT void operator()(DeviceAdapter, DeviceAdapterId id, RuntimeDeviceTracker* rdt) const
{
vtkm::cont::RuntimeDeviceInformation runtimeDevice;
if (DeviceAdapter() == id)
......@@ -120,9 +74,6 @@ VTKM_CONT
RuntimeDeviceTracker::RuntimeDeviceTracker()
: Internals(std::make_shared<detail::RuntimeDeviceTrackerInternals>())
{
GetDeviceNameFunctor functor(this->Internals->DeviceNames, this->Internals->LowerCaseDeviceNames);
vtkm::ListForEach(functor, VTKM_DEFAULT_DEVICE_ADAPTER_LIST_TAG());
this->Reset();
}
......@@ -147,7 +98,7 @@ VTKM_CONT
bool RuntimeDeviceTracker::CanRunOnImpl(vtkm::cont::DeviceAdapterId deviceId) const
{
this->CheckDevice(deviceId);
return this->Internals->RuntimeValid[deviceId.GetValue()];
return this->Internals->RuntimeAllowed[deviceId.GetValue()];
}
VTKM_CONT
......@@ -157,7 +108,7 @@ void RuntimeDeviceTracker::SetDeviceState(vtkm::cont::DeviceAdapterId deviceId,
VTKM_LOG_S(vtkm::cont::LogLevel::Info,
"Setting device '" << deviceId.GetName() << "' to " << state);
this->Internals->RuntimeValid[deviceId.GetValue()] = state;
this->Internals->RuntimeAllowed[deviceId.GetValue()] = state;
}
namespace
......@@ -165,18 +116,16 @@ namespace
struct VTKM_NEVER_EXPORT RuntimeDeviceTrackerResetFunctor
{
vtkm::cont::RuntimeDeviceTracker Tracker;
VTKM_CONT
RuntimeDeviceTrackerResetFunctor(const vtkm::cont::RuntimeDeviceTracker tracker)
: Tracker(tracker)
{
}
template <typename Device>
VTKM_CONT void operator()(Device device)
VTKM_CONT void operator()(Device device, bool runtimeAllowed[VTKM_MAX_DEVICE_ADAPTER_ID]) const
{
this->Tracker.ResetDevice(device);
if (device.IsValueValid())
{
const bool state = vtkm::cont::DeviceAdapterRuntimeDetector<Device>().Exists();
runtimeAllowed[device.GetValue()] = state;
VTKM_LOG_S(vtkm::cont::LogLevel::Info,
"Reset device '" << device.GetName() << "' to " << state);
}
}
};
}
......@@ -184,10 +133,17 @@ struct VTKM_NEVER_EXPORT RuntimeDeviceTrackerResetFunctor
VTKM_CONT
void RuntimeDeviceTracker::Reset()
{
std::fill_n(this->Internals->RuntimeValid, VTKM_MAX_DEVICE_ADAPTER_ID, false);
std::fill_n(this->Internals->RuntimeAllowed, VTKM_MAX_DEVICE_ADAPTER_ID, false);
RuntimeDeviceTrackerResetFunctor functor(*this);
vtkm::ListForEach(functor, VTKM_DEFAULT_DEVICE_ADAPTER_LIST_TAG());
// We use this instead of calling CheckDevice/SetDeviceState so that
// when we use logging we get better messages stating we are reseting
// the devices.
//
// 1. We can't log anything as this can be called during startup
// and
RuntimeDeviceTrackerResetFunctor functor;
vtkm::ListForEach(
functor, VTKM_DEFAULT_DEVICE_ADAPTER_LIST_TAG(), std::ref(this->Internals->RuntimeAllowed));
}
VTKM_CONT
......@@ -197,10 +153,10 @@ vtkm::cont::RuntimeDeviceTracker RuntimeDeviceTracker::DeepCopy() const
}
VTKM_CONT
void RuntimeDeviceTracker::DeepCopy(const vtkm::cont::RuntimeDeviceTracker src)
void RuntimeDeviceTracker::DeepCopy(const vtkm::cont::RuntimeDeviceTracker& src)
{
std::copy_n(
src.Internals->RuntimeValid, VTKM_MAX_DEVICE_ADAPTER_ID, this->Internals->RuntimeValid);
src.Internals->RuntimeAllowed, VTKM_MAX_DEVICE_ADAPTER_ID, this->Internals->RuntimeAllowed);
}
VTKM_CONT
......@@ -208,11 +164,8 @@ RuntimeDeviceTracker::RuntimeDeviceTracker(
const std::shared_ptr<detail::RuntimeDeviceTrackerInternals>& internals)
: Internals(std::make_shared<detail::RuntimeDeviceTrackerInternals>())
{
std::copy_n(internals->RuntimeValid, VTKM_MAX_DEVICE_ADAPTER_ID, this->Internals->RuntimeValid);
std::copy_n(internals->DeviceNames, VTKM_MAX_DEVICE_ADAPTER_ID, this->Internals->DeviceNames);
std::copy_n(internals->LowerCaseDeviceNames,
VTKM_MAX_DEVICE_ADAPTER_ID,
this->Internals->LowerCaseDeviceNames);
std::copy_n(
internals->RuntimeAllowed, VTKM_MAX_DEVICE_ADAPTER_ID, this->Internals->RuntimeAllowed);
}
VTKM_CONT
......@@ -230,110 +183,40 @@ void RuntimeDeviceTracker::ForceDeviceImpl(vtkm::cont::DeviceAdapterId deviceId,
VTKM_LOG_S(vtkm::cont::LogLevel::Info,
"Forcing execution to occur on device '" << deviceId.GetName() << "'");
std::fill_n(this->Internals->RuntimeValid, VTKM_MAX_DEVICE_ADAPTER_ID, false);
std::fill_n(this->Internals->RuntimeAllowed, VTKM_MAX_DEVICE_ADAPTER_ID, false);
this->Internals->RuntimeValid[deviceId.GetValue()] = runtimeExists;
this->Internals->RuntimeAllowed[deviceId.GetValue()] = runtimeExists;
}
VTKM_CONT
void RuntimeDeviceTracker::ForceDevice(DeviceAdapterId id)
void RuntimeDeviceTracker::ForceDevice(DeviceAdapterId deviceId)
{
detail::RuntimeDeviceTrackerFunctor functor;
vtkm::ListForEach(functor, VTKM_DEFAULT_DEVICE_ADAPTER_LIST_TAG(), id, this);
}
VTKM_CONT
DeviceAdapterNameType RuntimeDeviceTracker::GetDeviceName(DeviceAdapterId device) const
{
auto id = device.GetValue();
if (id < 0)
{
switch (id)
{
case VTKM_DEVICE_ADAPTER_ERROR:
return vtkm::cont::DeviceAdapterTraits<vtkm::cont::DeviceAdapterTagError>::GetName();
case VTKM_DEVICE_ADAPTER_UNDEFINED:
return vtkm::cont::DeviceAdapterTraits<vtkm::cont::DeviceAdapterTagUndefined>::GetName();
default:
break;
}
}
else if (id >= VTKM_MAX_DEVICE_ADAPTER_ID)
{
switch (id)
{
case VTKM_DEVICE_ADAPTER_ANY:
return vtkm::cont::DeviceAdapterTraits<vtkm::cont::DeviceAdapterTagAny>::GetName();
default:
break;
}
}
else // id is valid:
{
return this->Internals->DeviceNames[id];
}
// Device 0 is invalid:
return this->Internals->DeviceNames[0];
}
VTKM_CONT
DeviceAdapterId RuntimeDeviceTracker::GetDeviceAdapterId(DeviceAdapterNameType name) const
{
// The GetDeviceAdapterId call is case-insensitive so transform the name to be lower case
// as that is how we cache the case-insensitive version.
auto lowerCaseFunc = [](char c) {
return static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
};
std::transform(name.begin(), name.end(), name.begin(), lowerCaseFunc);
//lower-case the name here
if (name == "any")
{
return vtkm::cont::DeviceAdapterTagAny{};
}
else if (name == "error")
{
return vtkm::cont::DeviceAdapterTagError{};
}
else if (name == "undefined")
{
return vtkm::cont::DeviceAdapterTagUndefined{};
}
for (vtkm::Int8 id = 0; id < VTKM_MAX_DEVICE_ADAPTER_ID; ++id)
{
if (name == this->Internals->LowerCaseDeviceNames[id])
{
return vtkm::cont::make_DeviceAdapterId(id);
}
}
return vtkm::cont::DeviceAdapterTagUndefined{};
vtkm::cont::RuntimeDeviceInformation runtimeDevice;
this->ForceDeviceImpl(deviceId, runtimeDevice.Exists(deviceId));
}
VTKM_CONT
vtkm::cont::RuntimeDeviceTracker GetRuntimeDeviceTracker()
vtkm::cont::RuntimeDeviceTracker& GetRuntimeDeviceTracker()
{
#if defined(VTKM_CLANG) && (__apple_build_version__ < 8000000)
#if defined(VTKM_CLANG) && defined(__apple_build_version__) && (__apple_build_version__ < 8000000)
static std::mutex mtx;
static std::map<std::thread::id, vtkm::cont::RuntimeDeviceTracker> globalTrackers;
static std::map<std::thread::id, vtkm::cont::RuntimeDeviceTracker*> globalTrackers;
std::thread::id this_id = std::this_thread::get_id();
std::unique_lock<std::mutex> lock(mtx);
auto iter = globalTrackers.find(this_id);
if (iter != globalTrackers.end())
{
return iter->second;
return *iter->second;
}
else
{
vtkm::cont::RuntimeDeviceTracker tracker;
vtkm::cont::RuntimeDeviceTracker* tracker = new vtkm::cont::RuntimeDeviceTracker();
globalTrackers[this_id] = tracker;
return tracker;
return *tracker;
}
#else
static thread_local vtkm::cont::RuntimeDeviceTracker runtimeDeviceTracker;
return runtimeDeviceTracker;
#endif
}
......
......@@ -34,8 +34,6 @@ namespace vtkm
{
namespace cont
{
class RuntimeDeviceTracker;
namespace detail
{
......@@ -50,38 +48,19 @@ struct RuntimeDeviceTrackerFunctor;
/// of valid runtime devices based on that information.
///
///
class VTKM_ALWAYS_EXPORT RuntimeDeviceTracker
class VTKM_CONT_EXPORT RuntimeDeviceTracker
{
friend struct detail::RuntimeDeviceTrackerFunctor;
friend VTKM_CONT_EXPORT vtkm::cont::RuntimeDeviceTracker& GetRuntimeDeviceTracker();
public:
VTKM_CONT_EXPORT
VTKM_CONT
RuntimeDeviceTracker();
VTKM_CONT_EXPORT
VTKM_CONT
~RuntimeDeviceTracker();
/// Returns true if the given device adapter is supported on the current
/// machine.
///
template <typename DeviceAdapterTag>
VTKM_CONT bool CanRunOn(DeviceAdapterTag device) const
{
return this->CanRunOnImpl(device);
}
/// Report a failure to allocate memory on a device, this will flag the
/// device as being unusable for all future invocations of the instance of
/// the filter.
///
template <typename DeviceAdapterTag>
VTKM_CONT void ReportAllocationFailure(DeviceAdapterTag device,
const vtkm::cont::ErrorBadAllocation&)
{
this->SetDeviceState(device, false);
}
VTKM_CONT bool CanRunOn(DeviceAdapterId device) const { return this->CanRunOnImpl(device); }
/// Report a failure to allocate memory on a device, this will flag the
/// device as being unusable for all future invocations of the instance of
......@@ -93,35 +72,26 @@ public:
this->SetDeviceState(deviceId, false);
}
//@{
/// Report a ErrorBadDevice failure and flag the device as unusable.
template <typename DeviceAdapterTag>
VTKM_CONT void ReportBadDeviceFailure(DeviceAdapterTag device, const vtkm::cont::ErrorBadDevice&)
{
this->SetDeviceState(device, false);
}
/// Report a ErrorBadDevice failure and flag the device as unusable.
VTKM_CONT void ReportBadDeviceFailure(vtkm::cont::DeviceAdapterId deviceId,
const vtkm::cont::ErrorBadDevice&)
{
this->SetDeviceState(deviceId, false);
}
//@}
/// Reset the tracker for the given device. This will discard any updates
/// caused by reported failures
///
template <typename DeviceAdapterTag>
VTKM_CONT void ResetDevice(DeviceAdapterTag device)
VTKM_CONT void ResetDevice(vtkm::cont::DeviceAdapterId device)
{
vtkm::cont::RuntimeDeviceInformation runtimeDevice;
this->SetDeviceState(device, runtimeDevice.Exists(DeviceAdapterTag()));
this->SetDeviceState(device, runtimeDevice.Exists(device));
}
/// Reset the tracker to its default state for default devices.
/// Will discard any updates caused by reported failures.
///
VTKM_CONT_EXPORT
VTKM_CONT
void Reset();
......@@ -142,7 +112,6 @@ public:
/// This version of \c DeepCopy creates a whole new \c RuntimeDeviceTracker
/// with a state that is not shared with any other object.
///
VTKM_CONT_EXPORT
VTKM_CONT
vtkm::cont::RuntimeDeviceTracker DeepCopy() const;
......@@ -165,9 +134,8 @@ public:
/// state with this object will also get updated. This method is good for
/// restoring a state that was previously saved.
///
VTKM_CONT_EXPORT
VTKM_CONT
void DeepCopy(const vtkm::cont::RuntimeDeviceTracker src);
void DeepCopy(const vtkm::cont::RuntimeDeviceTracker& src);