Commit 84a772c3 authored by Robert Maynard's avatar Robert Maynard Committed by Kitware Robot

Merge topic 'vtkm_storage_can_use_cuda_uvm_memory'

c1237969 VTK-m ArrayHandle can now take ownership of a user allocated memory location
707970f4 VTK-m StorageBasic is now able to give/take ownership of user allocated memory.
Acked-by: Kitware Robot's avatarKitware Robot <kwrobot@kitware.com>
Acked-by: Kenneth Moreland's avatarKenneth Moreland <kmorel@sandia.gov>
Merge-request: !1137
parents a579b2c5 c1237969
......@@ -303,6 +303,13 @@ public:
///
ArrayHandle(const StorageType& storage);
/// Special constructor for subclass specializations that need to set the
/// initial state of the control array. When this constructor is used, it
/// is assumed that the control array is valid.
///
ArrayHandle(StorageType&& storage);
/// Destructs an empty ArrayHandle.
///
/// Implemented so that it is defined exclusively in the control environment.
......
......@@ -52,6 +52,15 @@ ArrayHandle<T, S>::ArrayHandle(const typename ArrayHandle<T, S>::StorageType& st
this->Internals->ExecutionArrayValid = false;
}
template <typename T, typename S>
ArrayHandle<T, S>::ArrayHandle(typename ArrayHandle<T, S>::StorageType&& storage)
: Internals(new InternalStruct)
{
this->Internals->ControlArray = std::move(storage);
this->Internals->ControlArrayValid = true;
this->Internals->ExecutionArrayValid = false;
}
template <typename T, typename S>
ArrayHandle<T, S>::~ArrayHandle()
{
......
......@@ -52,6 +52,19 @@ namespace cont
namespace internal
{
void free_memory(void* mem)
{
#if defined(VTKM_MEMALIGN_POSIX)
free(mem);
#elif defined(VTKM_MEMALIGN_WIN)
_aligned_free(mem);
#elif defined(VTKM_MEMALIGN_SSE)
_mm_free(mem);
#else
free(mem);
#endif
}
void* StorageBasicAllocator::allocate(size_t size, size_t align)
{
#if defined(VTKM_MEMALIGN_POSIX)
......@@ -70,25 +83,11 @@ void* StorageBasicAllocator::allocate(size_t size, size_t align)
return mem;
}
void StorageBasicAllocator::free_memory(void* mem)
{
#if defined(VTKM_MEMALIGN_POSIX)
free(mem);
#elif defined(VTKM_MEMALIGN_WIN)
_aligned_free(mem);
#elif defined(VTKM_MEMALIGN_SSE)
_mm_free(mem);
#else
free(mem);
#endif
}
StorageBasicBase::StorageBasicBase()
: Array(nullptr)
, AllocatedByteSize(0)
, NumberOfValues(0)
, DeallocateOnRelease(true)
, DeleteFunction(internal::free_memory)
{
}
......@@ -98,7 +97,18 @@ StorageBasicBase::StorageBasicBase(const void* array,
: Array(const_cast<void*>(array))
, AllocatedByteSize(static_cast<vtkm::UInt64>(numberOfValues) * sizeOfValue)
, NumberOfValues(numberOfValues)
, DeallocateOnRelease(array == nullptr ? true : false)
, DeleteFunction(array == nullptr ? internal::free_memory : nullptr)
{
}
StorageBasicBase::StorageBasicBase(const void* array,
vtkm::Id numberOfValues,
vtkm::UInt64 sizeOfValue,
void (*deleteFunction)(void*))
: Array(const_cast<void*>(array))
, AllocatedByteSize(static_cast<vtkm::UInt64>(numberOfValues) * sizeOfValue)
, NumberOfValues(numberOfValues)
, DeleteFunction(deleteFunction)
{
}
......@@ -107,14 +117,27 @@ StorageBasicBase::~StorageBasicBase()
this->ReleaseResources();
}
StorageBasicBase::StorageBasicBase(StorageBasicBase&& src)
: Array(src.Array)
, AllocatedByteSize(src.AllocatedByteSize)
, NumberOfValues(src.NumberOfValues)
, DeleteFunction(src.DeleteFunction)
{
src.Array = nullptr;
src.AllocatedByteSize = 0;
src.NumberOfValues = 0;
src.DeleteFunction = nullptr;
}
StorageBasicBase::StorageBasicBase(const StorageBasicBase& src)
: Array(src.Array)
, AllocatedByteSize(src.AllocatedByteSize)
, NumberOfValues(src.NumberOfValues)
, DeallocateOnRelease(src.DeallocateOnRelease)
, DeleteFunction(src.DeleteFunction)
{
if (src.DeallocateOnRelease)
if (src.DeleteFunction)
{
throw vtkm::cont::ErrorBadValue(
"Attempted to copy a storage array that needs deallocation. "
......@@ -122,9 +145,24 @@ StorageBasicBase::StorageBasicBase(const StorageBasicBase& src)
}
}
StorageBasicBase StorageBasicBase::operator=(StorageBasicBase&& src)
{
this->ReleaseResources();
this->Array = src.Array;
this->AllocatedByteSize = src.AllocatedByteSize;
this->NumberOfValues = src.NumberOfValues;
this->DeleteFunction = src.DeleteFunction;
src.Array = nullptr;
src.AllocatedByteSize = 0;
src.NumberOfValues = 0;
src.DeleteFunction = nullptr;
return *this;
}
StorageBasicBase StorageBasicBase::operator=(const StorageBasicBase& src)
{
if (src.DeallocateOnRelease)
if (src.DeleteFunction)
{
throw vtkm::cont::ErrorBadValue(
"Attempted to copy a storage array that needs deallocation. "
......@@ -135,29 +173,10 @@ StorageBasicBase StorageBasicBase::operator=(const StorageBasicBase& src)
this->Array = src.Array;
this->AllocatedByteSize = src.AllocatedByteSize;
this->NumberOfValues = src.NumberOfValues;
this->DeallocateOnRelease = src.DeallocateOnRelease;
this->DeleteFunction = src.DeleteFunction;
return *this;
}
void StorageBasicBase::ReleaseResources()
{
if (this->AllocatedByteSize > 0)
{
VTKM_ASSERT(this->Array != nullptr);
if (this->DeallocateOnRelease)
{
AllocatorType{}.free_memory(this->Array);
}
this->Array = nullptr;
this->AllocatedByteSize = 0;
this->NumberOfValues = 0;
}
else
{
VTKM_ASSERT(this->Array == nullptr);
}
}
void StorageBasicBase::AllocateValues(vtkm::Id numberOfValues, vtkm::UInt64 sizeOfValue)
{
if (numberOfValues < 0)
......@@ -181,7 +200,7 @@ void StorageBasicBase::AllocateValues(vtkm::Id numberOfValues, vtkm::UInt64 size
return;
}
if (!this->DeallocateOnRelease)
if (!this->DeleteFunction)
{
throw vtkm::cont::ErrorBadValue("User allocated arrays cannot be reallocated.");
}
......@@ -193,9 +212,10 @@ void StorageBasicBase::AllocateValues(vtkm::Id numberOfValues, vtkm::UInt64 size
this->Array = AllocatorType{}.allocate(allocsize, VTKM_ALLOCATION_ALIGNMENT);
this->AllocatedByteSize = allocsize;
this->NumberOfValues = numberOfValues;
this->DeleteFunction = internal::free_memory;
if (this->Array == nullptr)
{
// Make sureour state is OK.
// Make sure our state is OK.
this->AllocatedByteSize = 0;
this->NumberOfValues = 0;
throw vtkm::cont::ErrorBadAllocation("Could not allocate basic control array.");
......@@ -207,7 +227,6 @@ void StorageBasicBase::AllocateValues(vtkm::Id numberOfValues, vtkm::UInt64 size
VTKM_ASSERT(this->NumberOfValues == 0);
VTKM_ASSERT(this->AllocatedByteSize == 0);
}
this->DeallocateOnRelease = true;
}
void StorageBasicBase::Shrink(vtkm::Id numberOfValues)
......@@ -220,6 +239,37 @@ void StorageBasicBase::Shrink(vtkm::Id numberOfValues)
this->NumberOfValues = numberOfValues;
}
void StorageBasicBase::ReleaseResources()
{
if (this->AllocatedByteSize > 0)
{
VTKM_ASSERT(this->Array != nullptr);
if (this->DeleteFunction)
{
this->DeleteFunction(this->Array);
}
this->Array = nullptr;
this->AllocatedByteSize = 0;
this->NumberOfValues = 0;
}
else
{
VTKM_ASSERT(this->Array == nullptr);
}
}
void StorageBasicBase::SetBasePointer(const void* ptr,
vtkm::Id numberOfValues,
vtkm::UInt64 sizeOfValue,
void (*deleteFunction)(void*))
{
this->ReleaseResources();
this->Array = const_cast<void*>(ptr);
this->AllocatedByteSize = static_cast<vtkm::UInt64>(numberOfValues) * sizeOfValue;
this->NumberOfValues = numberOfValues;
this->DeleteFunction = deleteFunction;
}
void* StorageBasicBase::GetBasePointer() const
{
return this->Array;
......
......@@ -41,6 +41,12 @@ struct VTKM_ALWAYS_EXPORT StorageTagBasic
namespace internal
{
/// Function that does all of VTK-m de-allocations for storage basic.
/// This is exists so that stolen arrays can call the correct free
/// function ( _aligned_malloc / cuda_free ).
VTKM_CONT_EXPORT void free_memory(void* ptr);
/// Class that does all of VTK-m allocations
/// for storage basic. This is exists so that
/// stolen arrays can call the correct free
......@@ -48,14 +54,14 @@ namespace internal
struct VTKM_CONT_EXPORT StorageBasicAllocator
{
void* allocate(size_t size, size_t align);
void free_memory(void* p);
template <typename T>
void deallocate(T* p)
inline void deallocate(T* p)
{
this->free_memory(static_cast<void*>(p));
internal::free_memory(static_cast<void*>(p));
}
};
/// Base class for basic storage classes. This allow us to implement
/// vtkm::cont::Storage<T, StorageTagBasic > for any T type with no overhead
/// as all heavy logic is provide by a type-agnostic API including allocations, etc.
......@@ -64,10 +70,22 @@ class VTKM_CONT_EXPORT StorageBasicBase
public:
using AllocatorType = StorageBasicAllocator;
VTKM_CONT StorageBasicBase();
/// A non owning view of already allocated memory
VTKM_CONT StorageBasicBase(const void* array, vtkm::Id size, vtkm::UInt64 sizeOfValue);
/// Transfer the ownership of already allocated memory to VTK-m
VTKM_CONT StorageBasicBase(const void* array,
vtkm::Id size,
vtkm::UInt64 sizeOfValue,
void (*deleteFunction)(void*));
VTKM_CONT ~StorageBasicBase();
VTKM_CONT StorageBasicBase(StorageBasicBase&& src);
VTKM_CONT StorageBasicBase(const StorageBasicBase& src);
VTKM_CONT StorageBasicBase operator=(StorageBasicBase&& src);
VTKM_CONT StorageBasicBase operator=(const StorageBasicBase& src);
/// \brief Return the number of bytes allocated for this storage object(Capacity).
......@@ -106,7 +124,23 @@ public:
/// \brief Returns if vtkm will deallocate this memory. VTK-m StorageBasic
/// is designed that VTK-m will not deallocate user passed memory, or
/// instances that have been stolen (\c StealArray)
VTKM_CONT bool WillDeallocate() const { return this->DeallocateOnRelease; }
VTKM_CONT bool WillDeallocate() const { return this->DeleteFunction != nullptr; }
/// \brief Return the free function that will be used to free this memory.
///
/// Get the function that VTK-m will call to deallocate the memory. This
/// is useful when stealing memory from VTK-m so that you call the correct
/// free/delete function when releasing the memory
using DeleteFunctionSignature = void (*)(void*);
DeleteFunctionSignature GetDeleteFunction() const { return this->DeleteFunction; }
/// \brief Change the Change the pointer that this class is using. Should only be used
/// by ExecutionArrayInterface sublcasses.
/// Note: This will release any previous allocated memory!
VTKM_CONT void SetBasePointer(const void* ptr,
vtkm::Id numberOfValues,
vtkm::UInt64 sizeOfValue,
void (*deleteFunction)(void*));
/// Return the memory location of the first element of the array data.
VTKM_CONT void* GetBasePointer() const;
......@@ -121,7 +155,7 @@ protected:
void* Array;
vtkm::UInt64 AllocatedByteSize;
vtkm::Id NumberOfValues;
bool DeallocateOnRelease;
void (*DeleteFunction)(void*);
};
/// A basic implementation of an Storage object.
......@@ -146,7 +180,11 @@ public:
VTKM_CONT Storage();
/// \brief construct storage that VTK-m is not responsible for
VTKM_CONT Storage(const ValueType* array, vtkm::Id numberOfValues = 0);
VTKM_CONT Storage(const ValueType* array, vtkm::Id numberOfValues);
/// \brief construct storage that was previously allocated and now VTK-m is
/// responsible for
VTKM_CONT Storage(const ValueType* array, vtkm::Id numberOfValues, void (*deleteFunction)(void*));
VTKM_CONT void Allocate(vtkm::Id numberOfValues);
......@@ -171,7 +209,8 @@ public:
/// Storage will never deallocate the array or be able to reallocate it. This
/// is helpful for taking a reference for an array created internally by
/// VTK-m and not having to keep a VTK-m object around. Obviously the caller
/// becomes responsible for destroying the memory.
/// becomes responsible for destroying the memory, and should before hand
/// grab the correct DeleteFunction by calling \c GetDeleteFunction
///
VTKM_CONT ValueType* StealArray();
};
......
......@@ -41,6 +41,14 @@ Storage<T, vtkm::cont::StorageTagBasic>::Storage(const T* array, vtkm::Id number
{
}
template <typename T>
Storage<T, vtkm::cont::StorageTagBasic>::Storage(const T* array,
vtkm::Id numberOfValues,
void (*deleteFunction)(void*))
: StorageBasicBase(const_cast<T*>(array), numberOfValues, sizeof(T), deleteFunction)
{
}
template <typename T>
void Storage<T, vtkm::cont::StorageTagBasic>::Allocate(vtkm::Id numberOfValues)
{
......@@ -78,7 +86,7 @@ const T* Storage<T, vtkm::cont::StorageTagBasic>::GetArray() const
template <typename T>
T* Storage<T, vtkm::cont::StorageTagBasic>::StealArray()
{
this->DeallocateOnRelease = false;
this->DeleteFunction = nullptr;
return static_cast<T*>(this->Array);
}
......
......@@ -98,6 +98,15 @@ void ExecutionArrayInterfaceBasic<DeviceAdapterTagCuda>::Allocate(TypelessExecut
err << "Failed to allocate " << numBytes << " bytes on device: " << error.what();
throw vtkm::cont::ErrorBadAllocation(err.str());
}
// If we just allocated managed cuda memory and don't a host memory pointer
// we can share out managed memory. This allows for the use case of where we
// first allocate on CUDA and than want to use it on the host
if (CudaAllocator::IsManagedPointer(execArray.Array) && execArray.ArrayControl == nullptr)
{
this->ControlStorage.SetBasePointer(
execArray.Array, numberOfValues, sizeOfValue, [](void* ptr) { CudaAllocator::Free(ptr); });
}
}
void ExecutionArrayInterfaceBasic<DeviceAdapterTagCuda>::Free(
......
......@@ -19,6 +19,7 @@
##============================================================================
set(unit_tests
UnitTestCudaArrayHandle.cu
UnitTestCudaArrayHandleFancy.cu
UnitTestCudaArrayHandleVirtualCoordinates.cu
UnitTestCudaCellLocatorTwoLevelUniformGrid.cu
......
//============================================================================
// Copyright (c) Kitware, Inc.
// All rights reserved.
// See LICENSE.txt for details.
// This software is distributed WITHOUT ANY WARRANTY; without even
// the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
// PURPOSE. See the above copyright notice for more information.
//
// Copyright 2014 National Technology & Engineering Solutions of Sandia, LLC (NTESS).
// Copyright 2014 UT-Battelle, LLC.
// Copyright 2014 Los Alamos National Security.
//
// Under the terms of Contract DE-NA0003525 with NTESS,
// the U.S. Government retains certain rights in this software.
//
// Under the terms of Contract DE-AC52-06NA25396 with Los Alamos National
// Laboratory (LANL), the U.S. Government retains certain rights in
// this software.
//============================================================================
#define VTKM_DEVICE_ADAPTER VTKM_DEVICE_ADAPTER_ERROR
#include <vtkm/cont/cuda/DeviceAdapterCuda.h>
#include <vtkm/cont/testing/TestingArrayHandles.h>
int UnitTestCudaArrayHandle(int, char* [])
{
return vtkm::cont::testing::TestingArrayHandles<vtkm::cont::DeviceAdapterTagCuda>::Run();
}
......@@ -156,6 +156,20 @@ struct VTKM_CONT_EXPORT ArrayHandleImpl
{
}
VTKM_CONT
template <typename T>
explicit ArrayHandleImpl(vtkm::cont::internal::Storage<T, vtkm::cont::StorageTagBasic>&& storage)
: ControlArrayValid(true)
, ControlArray(
new vtkm::cont::internal::Storage<T, vtkm::cont::StorageTagBasic>(std::move(storage)))
, ExecutionInterface(nullptr)
, ExecutionArrayValid(false)
, ExecutionArray(nullptr)
, ExecutionArrayEnd(nullptr)
, ExecutionArrayCapacity(nullptr)
{
}
VTKM_CONT ~ArrayHandleImpl();
VTKM_CONT ArrayHandleImpl(const ArrayHandleImpl&) = delete;
......@@ -225,8 +239,10 @@ public:
VTKM_CONT ArrayHandle();
VTKM_CONT ArrayHandle(const Thisclass& src);
VTKM_CONT ArrayHandle(const Thisclass&& src);
VTKM_CONT ArrayHandle(Thisclass&& src);
VTKM_CONT ArrayHandle(const StorageType& storage);
VTKM_CONT ArrayHandle(StorageType&& storage);
VTKM_CONT ~ArrayHandle();
......
......@@ -40,7 +40,7 @@ ArrayHandle<T, StorageTagBasic>::ArrayHandle(const Thisclass& src)
}
template <typename T>
ArrayHandle<T, StorageTagBasic>::ArrayHandle(const Thisclass&& src)
ArrayHandle<T, StorageTagBasic>::ArrayHandle(Thisclass&& src)
: Internals(std::move(src.Internals))
{
}
......@@ -51,6 +51,12 @@ ArrayHandle<T, StorageTagBasic>::ArrayHandle(const StorageType& storage)
{
}
template <typename T>
ArrayHandle<T, StorageTagBasic>::ArrayHandle(StorageType&& storage)
: Internals(new internal::ArrayHandleImpl(std::move(storage)))
{
}
template <typename T>
ArrayHandle<T, StorageTagBasic>::~ArrayHandle()
{
......
......@@ -166,7 +166,7 @@ private:
}
};
struct VerifyUserAllocatedHandle
struct VerifyUserOwnedMemory
{
template <typename T>
VTKM_CONT void operator()(T) const
......@@ -241,6 +241,85 @@ private:
}
};
struct VerifyUserTransferredMemory
{
template <typename T>
VTKM_CONT void operator()(T) const
{
T* buffer = new T[ARRAY_SIZE];
for (vtkm::Id index = 0; index < ARRAY_SIZE; index++)
{
buffer[static_cast<std::size_t>(index)] = TestValue(index, T());
}
auto user_free_function = [](void* ptr) { delete[] static_cast<T*>(ptr); };
vtkm::cont::internal::Storage<T, vtkm::cont::StorageTagBasic> storage(
buffer, ARRAY_SIZE, user_free_function);
vtkm::cont::ArrayHandle<T> arrayHandle(std::move(storage));
VTKM_TEST_ASSERT(arrayHandle.GetNumberOfValues() == ARRAY_SIZE,
"ArrayHandle has wrong number of entries.");
std::cout << "Check array with user transferred memory." << std::endl;
array_handle_testing::CheckArray(arrayHandle);
std::cout << "Check out execution array behavior." << std::endl;
{ //as input
typename vtkm::cont::ArrayHandle<T>::template ExecutionTypes<DeviceAdapterTag>::PortalConst
executionPortal;
executionPortal = arrayHandle.PrepareForInput(DeviceAdapterTag());
//use a worklet to verify the input transfer worked properly
vtkm::cont::ArrayHandle<T> result;
DispatcherPassThrough().Invoke(arrayHandle, result);
array_handle_testing::CheckArray(result);
}
std::cout << "Check out inplace." << std::endl;
{ //as inplace
typename vtkm::cont::ArrayHandle<T>::template ExecutionTypes<DeviceAdapterTag>::Portal
executionPortal;
executionPortal = arrayHandle.PrepareForInPlace(DeviceAdapterTag());
//use a worklet to verify the inplace transfer worked properly
vtkm::cont::ArrayHandle<T> result;
DispatcherPassThrough().Invoke(arrayHandle, result);
array_handle_testing::CheckArray(result);
}
std::cout << "Check out output." << std::endl;
{ //as output with same length as user provided. This should work
//as no new memory needs to be allocated
typename vtkm::cont::ArrayHandle<T>::template ExecutionTypes<DeviceAdapterTag>::Portal
executionPortal;
executionPortal = arrayHandle.PrepareForOutput(ARRAY_SIZE, DeviceAdapterTag());
//we can't verify output contents as those aren't fetched, we
//can just make sure the allocation didn't throw an exception
}
{ //as the memory ownership has been transferred to VTK-m this should
//allow VTK-m to free the memory and allocate a new block
bool gotException = false;
try
{
//you should not be able to allocate a size larger than the
//user provided and get the results
arrayHandle.PrepareForOutput(ARRAY_SIZE * 2, DeviceAdapterTag());
arrayHandle.GetPortalControl();
}
catch (vtkm::cont::Error&)
{
gotException = true;
}
VTKM_TEST_ASSERT(!gotException,
"PrepareForOutput shouldn't fail when asked to "
"re-allocate user transferred memory.");
}
}
};
struct VerifyVTKMAllocatedHandle
{
template <typename T>
......@@ -369,7 +448,8 @@ private:
void operator()() const
{
vtkm::testing::Testing::TryTypes(VerifyEmptyArrays());
vtkm::testing::Testing::TryTypes(VerifyUserAllocatedHandle());
vtkm::testing::Testing::TryTypes(VerifyUserOwnedMemory());
vtkm::testing::Testing::TryTypes(VerifyUserTransferredMemory());
vtkm::testing::Testing::TryTypes(VerifyVTKMAllocatedHandle());
vtkm::testing::Testing::TryTypes(VerifyEqualityOperators());
}
......
......@@ -135,11 +135,30 @@ struct TemplatedTests
}
}
void UserFreeFunction()
{
ValueType* temp = new ValueType[ARRAY_SIZE];
StorageType arrayStorage(
temp, ARRAY_SIZE, [](void* ptr) { delete[] static_cast<ValueType*>(ptr); });
VTKM_TEST_ASSERT(temp == arrayStorage.GetArray(),
"improper pointer after telling storage to own user allocated memory");
const ValueType BASIC_ALLOC_VALUE = ValueType(48);
this->SetStorage(arrayStorage, BASIC_ALLOC_VALUE);
VTKM_TEST_ASSERT(this->CheckStorage(arrayStorage, BASIC_ALLOC_VALUE),
"Array not holding value.");
arrayStorage.Allocate(ARRAY_SIZE * 2);
VTKM_TEST_ASSERT(arrayStorage.GetNumberOfValues() == ARRAY_SIZE * 2,
"Array not reallocated correctly.");
}
void operator()()
{
ValueType* stolenArray = StealArray1();
BasicAllocation();
UserFreeFunction();
StealArray2(stolenArray);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment