Commit e8afcfd7 authored by Allison Vacanti's avatar Allison Vacanti Committed by Kitware Robot

Merge topic '405_partitioned_ds'

63050f68 `MultiBlock` renamed to `PartitionedDataSet`
Acked-by: Kitware Robot's avatarKitware Robot <kwrobot@kitware.com>
Acked-by: Utkarsh Ayachit's avatarUtkarsh Ayachit <utkarsh.ayachit@kitware.com>
Merge-request: !1821
parents e143eaf2 63050f68
# `MultiBlock` renamed to `PartitionedDataSet`
The `MultiBlock` class has been renamed to `PartitionedDataSet`, and its API
has been refactored to refer to "partitions", rather than "blocks".
Additionally, the `AddBlocks` method has been changed to `AppendPartitions` to
more accurately reflect the operation performed. The associated
`AssignerMultiBlock` class has also been renamed to
`AssignerPartitionedDataSet`.
This change is motivated towards unifying VTK-m's data model with VTK. VTK has
started to move away from `vtkMultiBlockDataSet`, which is a hierarchical tree
of nested datasets, to `vtkPartitionedDataSet`, which is always a flat vector
of datasets used to assist geometry distribution in multi-process environments.
This simplifies traversal during processing and clarifies the intent of the
container: The component datasets are partitions for distribution, not
organizational groupings (e.g. materials).
......@@ -80,18 +80,18 @@ int main(int argc, char* argv[])
const vtkm::Id num_bins = static_cast<vtkm::Id>(std::atoi(argv[1]));
const vtkm::Id numVals = 1024;
vtkm::cont::MultiBlock mb;
vtkm::cont::PartitionedDataSet pds;
vtkm::cont::DataSet ds;
vtkm::cont::DataSetFieldAdd::AddPointField(ds, "pointvar", CreateArray(-1024, 1024, numVals));
mb.AddBlock(ds);
pds.AppendPartition(ds);
example::HistogramMPI histogram;
histogram.SetActiveField("pointvar");
histogram.SetNumberOfBins(std::max<vtkm::Id>(1, num_bins));
vtkm::cont::MultiBlock result = histogram.Execute(mb);
vtkm::cont::PartitionedDataSet result = histogram.Execute(pds);
vtkm::cont::ArrayHandle<vtkm::Id> bins;
result.GetBlock(0).GetField("histogram").GetData().CopyTo(bins);
result.GetPartition(0).GetField("histogram").GetData().CopyTo(bins);
auto binPortal = bins.GetPortalConstControl();
if (rank == 0)
{
......
......@@ -65,16 +65,16 @@ public:
const vtkm::filter::PolicyBase<DerivedPolicy>& policy);
//@{
/// when operating on vtkm::cont::MultiBlock, we
/// when operating on vtkm::cont::PartitionedDataSet, we
/// want to do processing across ranks as well. Just adding pre/post handles
/// for the same does the trick.
template <typename DerivedPolicy>
VTKM_CONT void PreExecute(const vtkm::cont::MultiBlock& input,
VTKM_CONT void PreExecute(const vtkm::cont::PartitionedDataSet& input,
const vtkm::filter::PolicyBase<DerivedPolicy>& policy);
template <typename DerivedPolicy>
VTKM_CONT void PostExecute(const vtkm::cont::MultiBlock& input,
vtkm::cont::MultiBlock& output,
VTKM_CONT void PostExecute(const vtkm::cont::PartitionedDataSet& input,
vtkm::cont::PartitionedDataSet& output,
const vtkm::filter::PolicyBase<DerivedPolicy>&);
//@}
......
......@@ -12,7 +12,7 @@
#include <vtkm/cont/Algorithm.h>
#include <vtkm/cont/ArrayPortalToIterators.h>
#include <vtkm/cont/AssignerMultiBlock.h>
#include <vtkm/cont/AssignerPartitionedDataSet.h>
#include <vtkm/cont/EnvironmentTracker.h>
#include <vtkm/cont/ErrorFilterExecution.h>
#include <vtkm/cont/FieldRangeGlobalCompute.h>
......@@ -143,7 +143,7 @@ inline VTKM_CONT vtkm::cont::DataSet HistogramMPI::DoExecute(
//-----------------------------------------------------------------------------
template <typename DerivedPolicy>
inline VTKM_CONT void HistogramMPI::PreExecute(const vtkm::cont::MultiBlock& input,
inline VTKM_CONT void HistogramMPI::PreExecute(const vtkm::cont::PartitionedDataSet& input,
const vtkm::filter::PolicyBase<DerivedPolicy>&)
{
if (this->Range.IsNonEmpty())
......@@ -164,15 +164,15 @@ inline VTKM_CONT void HistogramMPI::PreExecute(const vtkm::cont::MultiBlock& inp
//-----------------------------------------------------------------------------
template <typename DerivedPolicy>
inline VTKM_CONT void HistogramMPI::PostExecute(const vtkm::cont::MultiBlock&,
vtkm::cont::MultiBlock& result,
inline VTKM_CONT void HistogramMPI::PostExecute(const vtkm::cont::PartitionedDataSet&,
vtkm::cont::PartitionedDataSet& result,
const vtkm::filter::PolicyBase<DerivedPolicy>&)
{
// iterate and compute HistogramMPI for each local block.
detail::DistributedHistogram helper(result.GetNumberOfBlocks());
for (vtkm::Id cc = 0; cc < result.GetNumberOfBlocks(); ++cc)
detail::DistributedHistogram helper(result.GetNumberOfPartitions());
for (vtkm::Id cc = 0; cc < result.GetNumberOfPartitions(); ++cc)
{
auto& ablock = result.GetBlock(cc);
auto& ablock = result.GetPartition(cc);
helper.SetLocalHistogram(cc, ablock.GetField(this->GetOutputFieldName()));
}
......@@ -182,6 +182,6 @@ inline VTKM_CONT void HistogramMPI::PostExecute(const vtkm::cont::MultiBlock&,
helper.ReduceAll(this->NumberOfBins));
output.AddField(rfield);
result = vtkm::cont::MultiBlock(output);
result = vtkm::cont::PartitionedDataSet(output);
}
} // namespace example
......@@ -51,46 +51,47 @@ vtkm::cont::DataSet make_test3DImageData(vtkm::Id3 dims)
}
//=================================================================
void io_generator(TaskQueue<vtkm::cont::MultiBlock>& queue, std::size_t numberOfTasks)
void io_generator(TaskQueue<vtkm::cont::PartitionedDataSet>& queue, std::size_t numberOfTasks)
{
//Step 1. We want to build an initial set of blocks
//Step 1. We want to build an initial set of partitions
//that vary in size. This way we can generate uneven
//work to show off the vtk-m filter work distribution
vtkm::Id3 small(128, 128, 128);
vtkm::Id3 medium(256, 256, 128);
vtkm::Id3 large(512, 256, 128);
std::vector<vtkm::Id3> block_sizes;
block_sizes.push_back(small);
block_sizes.push_back(medium);
block_sizes.push_back(large);
std::vector<vtkm::Id3> partition_sizes;
partition_sizes.push_back(small);
partition_sizes.push_back(medium);
partition_sizes.push_back(large);
std::mt19937 rng;
//uniform_int_distribution is a closed interval [] so both the min and max
//can be chosen values
std::uniform_int_distribution<vtkm::Id> blockNumGen(6, 32);
std::uniform_int_distribution<std::size_t> blockPicker(0, block_sizes.size() - 1);
std::uniform_int_distribution<vtkm::Id> partitionNumGen(6, 32);
std::uniform_int_distribution<std::size_t> partitionPicker(0, partition_sizes.size() - 1);
for (std::size_t i = 0; i < numberOfTasks; ++i)
{
//Step 2. Construct a random number of blocks
const vtkm::Id numberOfBlocks = blockNumGen(rng);
//Step 2. Construct a random number of partitions
const vtkm::Id numberOfPartitions = partitionNumGen(rng);
//Step 3. Randomly pick the blocks in the dataset
vtkm::cont::MultiBlock mb(numberOfBlocks);
for (vtkm::Id b = 0; b < numberOfBlocks; ++b)
//Step 3. Randomly pick the partitions in the dataset
vtkm::cont::PartitionedDataSet pds(numberOfPartitions);
for (vtkm::Id p = 0; p < numberOfPartitions; ++p)
{
const auto& dims = block_sizes[blockPicker(rng)];
auto block = make_test3DImageData(dims);
mb.AddBlock(block);
const auto& dims = partition_sizes[partitionPicker(rng)];
auto partition = make_test3DImageData(dims);
pds.AppendPartition(partition);
}
std::cout << "adding multi-block with " << mb.GetNumberOfBlocks() << " blocks" << std::endl;
std::cout << "adding partitioned dataset with " << pds.GetNumberOfPartitions() << " partitions"
<< std::endl;
//Step 4. Add the multi-block to the queue. We explicitly
//Step 4. Add the partitioned dataset to the queue. We explicitly
//use std::move to signal that this thread can't use the
//mb object after this call
queue.push(std::move(mb));
//pds object after this call
queue.push(std::move(pds));
//Step 5. Go to sleep for a period of time to replicate
//data stream in
......
......@@ -12,9 +12,9 @@
#include "TaskQueue.h"
#include <vtkm/cont/DataSet.h>
#include <vtkm/cont/MultiBlock.h>
#include <vtkm/cont/PartitionedDataSet.h>
vtkm::cont::DataSet make_test3DImageData(int xdim, int ydim, int zdim);
void io_generator(TaskQueue<vtkm::cont::MultiBlock>& queue, std::size_t numberOfTasks);
void io_generator(TaskQueue<vtkm::cont::PartitionedDataSet>& queue, std::size_t numberOfTasks);
#endif
......@@ -12,7 +12,7 @@
#include <thread>
#include <vtkm/cont/Initialize.h>
#include <vtkm/cont/MultiBlock.h>
#include <vtkm/cont/PartitionedDataSet.h>
#include "IOGenerator.h"
#include "MultiDeviceGradient.h"
......@@ -22,9 +22,9 @@
//
//At a high level we have 2 primary threads, an IO thread and a Worker thread
//The IO thread will generate all data using the vtk-m serial device, and
//will post this data to a worker queue as a vtk-m multiblock.
//The Worker thread will pull down these vtk-m multiblock data and run a
//vtk-m filter on the multiblock.
//will post this data to a worker queue as a vtk-m partitioned dataset.
//The Worker thread will pull down these vtk-m data and run a
//vtk-m filter on the partitions.
//The vtk-m filter it runs will itself have a worker pool which it will
//distribute work too. The number of workers is based on what device adapters
//are enabled but uses the following logic:
......@@ -38,7 +38,7 @@
//and single GPU we should expect that we will have 2 primary 'main loop'
//threads, and 5 threads for heavy 'task' work.
void multiblock_processing(TaskQueue<vtkm::cont::MultiBlock>& queue);
void partition_processing(TaskQueue<vtkm::cont::PartitionedDataSet>& queue);
int main(int argc, char** argv)
{
auto opts =
......@@ -48,9 +48,9 @@ int main(int argc, char** argv)
//Step 1. Construct the two primary 'main loops'. The threads
//share a queue object so we need to explicitly pass it
//by reference (the std::ref call)
TaskQueue<vtkm::cont::MultiBlock> queue;
TaskQueue<vtkm::cont::PartitionedDataSet> queue;
std::thread io(io_generator, std::ref(queue), 6);
std::thread worker(multiblock_processing, std::ref(queue));
std::thread worker(partition_processing, std::ref(queue));
//Step N. Wait for the work to finish
io.join();
......@@ -59,7 +59,7 @@ int main(int argc, char** argv)
}
//=================================================================
void multiblock_processing(TaskQueue<vtkm::cont::MultiBlock>& queue)
void partition_processing(TaskQueue<vtkm::cont::PartitionedDataSet>& queue)
{
//Step 1. Construct the gradient filter outside the work loop
//so that we can reuse the thread pool it constructs
......@@ -67,42 +67,37 @@ void multiblock_processing(TaskQueue<vtkm::cont::MultiBlock>& queue)
gradient.SetComputePointGradient(true);
while (queue.hasTasks())
{
//Step 2. grab the next multi-block skipping any that are empty
//Step 2. grab the next partition skipping any that are empty
//as empty ones can be returned when the queue is about
//to say it has no work
vtkm::cont::MultiBlock mb = queue.pop();
if (mb.GetNumberOfBlocks() == 0)
vtkm::cont::PartitionedDataSet pds = queue.pop();
if (pds.GetNumberOfPartitions() == 0)
{
continue;
}
//Step 3. Get the first field name from the multi-block
std::string fieldName = mb.GetBlock(0).GetField(0).GetName();
//Step 3. Get the first field name from the partition
std::string fieldName = pds.GetPartition(0).GetField(0).GetName();
//Step 4. Run a multi device gradient
gradient.SetActiveField(fieldName);
vtkm::cont::MultiBlock result = gradient.Execute(mb);
std::cout << "finished processing a multi-block" << std::endl;
vtkm::cont::PartitionedDataSet result = gradient.Execute(pds);
std::cout << "finished processing a partitioned dataset" << std::endl;
//Step 5. Verify each block has a "Gradients" field
for (auto&& block : result)
//Step 5. Verify each partition has a "Gradients" field
for (auto&& partition : result)
{
// std::cout << std::endl << std::endl << std::endl;
// std::cout << "block: " << std::endl;
// block.PrintSummary(std::cout);
try
{
const auto& field = block.GetField("Gradients", vtkm::cont::Field::Association::POINTS);
(void)field;
}
catch (const vtkm::cont::ErrorBadValue& error)
// std::cout << "partition: " << std::endl;
// partition.PrintSummary(std::cout);
if (!partition.HasField("Gradients", vtkm::cont::Field::Association::POINTS))
{
std::cerr << "Gradient filter failed!" << std::endl;
std::cerr << error.GetMessage() << std::endl;
std::cerr << "Missing Gradient field on output partition." << std::endl;
break;
}
}
}
std::cout << "multiblock_processing finished" << std::endl;
std::cout << "partition_processing finished" << std::endl;
}
......@@ -13,6 +13,6 @@
#include "MultiDeviceGradient.h"
#include "MultiDeviceGradient.hxx"
template vtkm::cont::MultiBlock MultiDeviceGradient::PrepareForExecution<
vtkm::filter::PolicyDefault>(const vtkm::cont::MultiBlock&,
template vtkm::cont::PartitionedDataSet MultiDeviceGradient::PrepareForExecution<
vtkm::filter::PolicyDefault>(const vtkm::cont::PartitionedDataSet&,
const vtkm::filter::PolicyBase<vtkm::filter::PolicyDefault>&);
......@@ -18,7 +18,7 @@
using RuntimeTaskQueue = TaskQueue<std::function<void()>>;
/// \brief Construct a MultiDeviceGradient for a given multiblock dataset
/// \brief Construct a MultiDeviceGradient for a given partitioned dataset
///
/// The Policy used with MultiDeviceGradient must include the TBB and CUDA
/// backends.
......@@ -45,8 +45,8 @@ public:
/// Will submit each block to a work queue that the threads will
/// pull work from
template <typename DerivedPolicy>
VTKM_CONT vtkm::cont::MultiBlock PrepareForExecution(
const vtkm::cont::MultiBlock&,
VTKM_CONT vtkm::cont::PartitionedDataSet PrepareForExecution(
const vtkm::cont::PartitionedDataSet&,
const vtkm::filter::PolicyBase<DerivedPolicy>&);
private:
......@@ -56,8 +56,8 @@ private:
};
#ifndef vtk_m_examples_multibackend_MultiDeviceGradient_cxx
extern template vtkm::cont::MultiBlock MultiDeviceGradient::PrepareForExecution<
vtkm::filter::PolicyDefault>(const vtkm::cont::MultiBlock&,
extern template vtkm::cont::PartitionedDataSet MultiDeviceGradient::PrepareForExecution<
vtkm::filter::PolicyDefault>(const vtkm::cont::PartitionedDataSet&,
const vtkm::filter::PolicyBase<vtkm::filter::PolicyDefault>&);
#endif
......
......@@ -32,7 +32,7 @@ int determine_cuda_gpu_count()
return count;
}
void process_block_tbb(RuntimeTaskQueue& queue)
void process_partition_tbb(RuntimeTaskQueue& queue)
{
//Step 1. Set the device adapter to this thread to TBB.
//This makes sure that any vtkm::filters used by our
......@@ -56,11 +56,11 @@ void process_block_tbb(RuntimeTaskQueue& queue)
//Step 4. Notify the queue that we finished processing this task
queue.completedTask();
std::cout << "finished a block on tbb (" << std::this_thread::get_id() << ")" << std::endl;
std::cout << "finished a partition on tbb (" << std::this_thread::get_id() << ")" << std::endl;
}
}
void process_block_openMP(RuntimeTaskQueue& queue)
void process_partition_openMP(RuntimeTaskQueue& queue)
{
//Step 1. Set the device adapter to this thread to TBB.
//This makes sure that any vtkm::filters used by our
......@@ -84,11 +84,11 @@ void process_block_openMP(RuntimeTaskQueue& queue)
//Step 4. Notify the queue that we finished processing this task
queue.completedTask();
std::cout << "finished a block on tbb (" << std::this_thread::get_id() << ")" << std::endl;
std::cout << "finished a partition on tbb (" << std::this_thread::get_id() << ")" << std::endl;
}
}
void process_block_cuda(RuntimeTaskQueue& queue, int gpuId)
void process_partition_cuda(RuntimeTaskQueue& queue, int gpuId)
{
//Step 1. Set the device adapter to this thread to cuda.
//This makes sure that any vtkm::filters used by our
......@@ -113,7 +113,7 @@ void process_block_cuda(RuntimeTaskQueue& queue, int gpuId)
//Step 4. Notify the queue that we finished processing this task
queue.completedTask();
std::cout << "finished a block on cuda (" << std::this_thread::get_id() << ")" << std::endl;
std::cout << "finished a partition on cuda (" << std::this_thread::get_id() << ")" << std::endl;
}
}
......@@ -149,10 +149,10 @@ VTKM_CONT MultiDeviceGradient::MultiDeviceGradient()
//The number of workers per GPU is purely arbitrary currently,
//but in general we want multiple of them so we can overlap compute
//and transfer
this->Workers.emplace_back(std::bind(process_block_cuda, std::ref(this->Queue), i));
this->Workers.emplace_back(std::bind(process_block_cuda, std::ref(this->Queue), i));
this->Workers.emplace_back(std::bind(process_block_cuda, std::ref(this->Queue), i));
this->Workers.emplace_back(std::bind(process_block_cuda, std::ref(this->Queue), i));
this->Workers.emplace_back(std::bind(process_partition_cuda, std::ref(this->Queue), i));
this->Workers.emplace_back(std::bind(process_partition_cuda, std::ref(this->Queue), i));
this->Workers.emplace_back(std::bind(process_partition_cuda, std::ref(this->Queue), i));
this->Workers.emplace_back(std::bind(process_partition_cuda, std::ref(this->Queue), i));
}
}
//Step 3. Launch a worker that will use openMP (if enabled).
......@@ -161,7 +161,7 @@ VTKM_CONT MultiDeviceGradient::MultiDeviceGradient()
else if (runOnOpenMP)
{
std::cout << "adding a openMP worker" << std::endl;
this->Workers.emplace_back(std::bind(process_block_openMP, std::ref(this->Queue)));
this->Workers.emplace_back(std::bind(process_partition_openMP, std::ref(this->Queue)));
}
//Step 4. Launch a worker that will use tbb (if enabled).
//The threads share a queue object so we need to explicitly pass it
......@@ -169,7 +169,7 @@ VTKM_CONT MultiDeviceGradient::MultiDeviceGradient()
else if (runOnTbb)
{
std::cout << "adding a tbb worker" << std::endl;
this->Workers.emplace_back(std::bind(process_block_tbb, std::ref(this->Queue)));
this->Workers.emplace_back(std::bind(process_partition_tbb, std::ref(this->Queue)));
}
}
......@@ -187,49 +187,50 @@ VTKM_CONT MultiDeviceGradient::~MultiDeviceGradient()
//-----------------------------------------------------------------------------
template <typename DerivedPolicy>
inline VTKM_CONT vtkm::cont::MultiBlock MultiDeviceGradient::PrepareForExecution(
const vtkm::cont::MultiBlock& mb,
inline VTKM_CONT vtkm::cont::PartitionedDataSet MultiDeviceGradient::PrepareForExecution(
const vtkm::cont::PartitionedDataSet& pds,
const vtkm::filter::PolicyBase<DerivedPolicy>& policy)
{
//Step 1. Say that we have no more to submit for this multi block
//Step 1. Say that we have no more to submit for this PartitionedDataSet
//This is needed to happen for each execute as we want to support
//the same filter being used for multiple inputs
this->Queue.reset();
//Step 2. Construct the multi-block we are going to fill. The size signature
//to MultiBlock just reserves size
vtkm::cont::MultiBlock output;
output.AddBlocks(std::vector<vtkm::cont::DataSet>(static_cast<size_t>(mb.GetNumberOfBlocks())));
vtkm::cont::MultiBlock* outPtr = &output;
//Step 2. Construct the PartitionedDataSet we are going to fill. The size
//signature to PartitionedDataSet just reserves size
vtkm::cont::PartitionedDataSet output;
output.AppendPartitions(
std::vector<vtkm::cont::DataSet>(static_cast<size_t>(pds.GetNumberOfPartitions())));
vtkm::cont::PartitionedDataSet* outPtr = &output;
//Step 3. Construct the filter we want to run on each block
//Step 3. Construct the filter we want to run on each partition
vtkm::filter::Gradient gradient;
gradient.SetComputePointGradient(this->GetComputePointGradient());
gradient.SetActiveField(this->GetActiveFieldName());
//Step 3b. Post 1 block up as work and block intil it is
//Step 3b. Post 1 partition up as work and block until it is
//complete. This is needed as currently constructing the virtual
//Point Coordinates is not thread safe.
auto block = mb.cbegin();
auto partition = pds.cbegin();
{
vtkm::cont::DataSet input = *block;
vtkm::cont::DataSet input = *partition;
this->Queue.push( //build a lambda that is the work to do
[=]() {
vtkm::filter::Gradient perThreadGrad = gradient;
vtkm::cont::DataSet result = perThreadGrad.Execute(input, policy);
outPtr->ReplaceBlock(0, result);
outPtr->ReplacePartition(0, result);
});
this->Queue.waitForAllTasksToComplete();
block++;
partition++;
}
vtkm::Id index = 1;
for (; block != mb.cend(); ++block)
for (; partition != pds.cend(); ++partition)
{
vtkm::cont::DataSet input = *block;
//Step 4. For each input block construct a lambda
vtkm::cont::DataSet input = *partition;
//Step 4. For each input partition construct a lambda
//and add it to the queue for workers to take. This
//will allows us to have multiple works execute in a non
//blocking manner
......@@ -238,7 +239,7 @@ inline VTKM_CONT vtkm::cont::MultiBlock MultiDeviceGradient::PrepareForExecution
vtkm::filter::Gradient perThreadGrad = gradient;
vtkm::cont::DataSet result = perThreadGrad.Execute(input, policy);
outPtr->ReplaceBlock(index, result);
outPtr->ReplacePartition(index, result);
});
index++;
}
......
......@@ -10,7 +10,7 @@
#ifndef vtk_m_examples_multibackend_TaskQueue_h
#define vtk_m_examples_multibackend_TaskQueue_h
#include <vtkm/cont/MultiBlock.h>
#include <vtkm/cont/PartitionedDataSet.h>
#include <condition_variable>
#include <mutex>
......
......@@ -10,7 +10,7 @@
#include <vtkm/ImplicitFunction.h>
#include <vtkm/cont/Algorithm.h>
#include <vtkm/cont/AssignerMultiBlock.h>
#include <vtkm/cont/AssignerPartitionedDataSet.h>
#include <vtkm/cont/BoundsGlobalCompute.h>
#include <vtkm/cont/EnvironmentTracker.h>
#include <vtkm/cont/Serialization.h>
......@@ -204,14 +204,14 @@ public:
~RedistributePoints() {}
template <typename DerivedPolicy>
VTKM_CONT vtkm::cont::MultiBlock PrepareForExecution(
const vtkm::cont::MultiBlock& input,
VTKM_CONT vtkm::cont::PartitionedDataSet PrepareForExecution(
const vtkm::cont::PartitionedDataSet& input,
const vtkm::filter::PolicyBase<DerivedPolicy>& policy);
};
template <typename DerivedPolicy>
inline VTKM_CONT vtkm::cont::MultiBlock RedistributePoints::PrepareForExecution(
const vtkm::cont::MultiBlock& input,
inline VTKM_CONT vtkm::cont::PartitionedDataSet RedistributePoints::PrepareForExecution(
const vtkm::cont::PartitionedDataSet& input,
const vtkm::filter::PolicyBase<DerivedPolicy>& policy)
{
auto comm = vtkm::cont::EnvironmentTracker::GetCommunicator();
......@@ -219,7 +219,7 @@ inline VTKM_CONT vtkm::cont::MultiBlock RedistributePoints::PrepareForExecution(
// let's first get the global bounds of the domain
vtkm::Bounds gbounds = vtkm::cont::BoundsGlobalCompute(input);
vtkm::cont::AssignerMultiBlock assigner(input.GetNumberOfBlocks());
vtkm::cont::AssignerPartitionedDataSet assigner(input.GetNumberOfPartitions());
vtkmdiy::RegularDecomposer<vtkmdiy::ContinuousBounds> decomposer(
/*dim*/ 3, internal::convert(gbounds), assigner.nblocks());
......@@ -230,19 +230,19 @@ inline VTKM_CONT vtkm::cont::MultiBlock RedistributePoints::PrepareForExecution(
[](void* ptr) { delete static_cast<vtkm::cont::DataSet*>(ptr); });
decomposer.decompose(comm.rank(), assigner, master);
assert(static_cast<vtkm::Id>(master.size()) == input.GetNumberOfBlocks());
assert(static_cast<vtkm::Id>(master.size()) == input.GetNumberOfPartitions());
// let's populate local blocks
master.foreach ([&input](vtkm::cont::DataSet* ds, const vtkmdiy::Master::ProxyWithLink& proxy) {
auto lid = proxy.master()->lid(proxy.gid());
*ds = input.GetBlock(lid);
*ds = input.GetPartition(lid);
});
internal::Redistributor<DerivedPolicy> redistributor(decomposer, policy);
vtkmdiy::all_to_all(master, assigner, redistributor, /*k=*/2);
vtkm::cont::MultiBlock result;
vtkm::cont::PartitionedDataSet result;
master.foreach ([&result](vtkm::cont::DataSet* ds, const vtkmdiy::Master::ProxyWithLink&) {
result.AddBlock(*ds);
result.AppendPartition(*ds);
});
return result;
......
......@@ -7,10 +7,10 @@
// the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
// PURPOSE. See the above copyright notice for more information.
//============================================================================
#include <vtkm/cont/AssignerMultiBlock.h>
#include <vtkm/cont/AssignerPartitionedDataSet.h>
#include <vtkm/cont/EnvironmentTracker.h>
#include <vtkm/cont/MultiBlock.h>
#include <vtkm/cont/PartitionedDataSet.h>
#include <vtkm/thirdparty/diy/diy.h>
......@@ -24,55 +24,56 @@ namespace cont
{
VTKM_CONT
AssignerMultiBlock::AssignerMultiBlock(const vtkm::cont::MultiBlock& mb)
: AssignerMultiBlock(mb.GetNumberOfBlocks())
AssignerPartitionedDataSet::AssignerPartitionedDataSet(const vtkm::cont::PartitionedDataSet& pds)
: AssignerPartitionedDataSet(pds.GetNumberOfPartitions())
{
}
VTKM_CONT
AssignerMultiBlock::AssignerMultiBlock(vtkm::Id num_blocks)
AssignerPartitionedDataSet::AssignerPartitionedDataSet(vtkm::Id num_partitions)
: vtkmdiy::StaticAssigner(vtkm::cont::EnvironmentTracker::GetCommunicator().size(), 1)
, IScanBlockCounts()
, IScanPartitionCounts()
{
auto comm = vtkm::cont::EnvironmentTracker::GetCommunicator();
if (comm.size() > 1)
{
vtkm::Id iscan;
vtkmdiy::mpi::scan(comm, num_blocks, iscan, std::plus<vtkm::Id>());
vtkmdiy::mpi::all_gather(comm, iscan, this->IScanBlockCounts);
vtkmdiy::mpi::scan(comm, num_partitions, iscan, std::plus<vtkm::Id>());
vtkmdiy::mpi::all_gather(comm, iscan, this->IScanPartitionCounts);
}
else
{
this->IScanBlockCounts.push_back(num_blocks);
this->IScanPartitionCounts.push_back(num_partitions);
}
this->set_nblocks(static_cast<int>(this->IScanBlockCounts.back()));
this->set_nblocks(static_cast<int>(this->IScanPartitionCounts.back()));
}
VTKM_CONT
void AssignerMultiBlock::local_gids(int my_rank, std::vector<int>& gids) const
void AssignerPartitionedDataSet::local_gids(int my_rank, std::vector<int>& gids) const
{
const size_t s_rank = static_cast<size_t>(my_rank);
if (my_rank == 0)
{
assert(this->IScanBlockCounts.size() > 0);
gids.resize(static_cast<size_t>(this->IScanBlockCounts[s_rank]));
assert(this->IScanPartitionCounts.size() > 0);
gids.resize(static_cast<size_t>(this->IScanPartitionCounts[s_rank]));
std::iota(gids.begin(), gids.end(), 0);
}
else if (my_rank > 0 && s_rank < this->IScanBlockCounts.size())
else if (my_rank > 0 && s_rank < this->IScanPartitionCounts.size())
{
gids.resize(
static_cast<size_t>(this->IScanBlockCounts[s_rank] - this->IScanBlockCounts[s_rank - 1]));
std::iota(gids.begin(), gids.end(), static_cast<int>(this->IScanBlockCounts[s_rank - 1]));
gids.resize(static_cast<size_t>(this->IScanPartitionCounts[s_rank] -
this->IScanPartitionCounts[s_rank - 1]));
std::iota(gids.begin(), gids.end(), static_cast<int>(this->IScanPartitionCounts[s_rank - 1]));
}
}
VTKM_CONT
int AssignerMultiBlock::rank(int gid) const
int AssignerPartitionedDataSet::rank(int gid) const
{
return static_cast<int>(
std::lower_bound(this->IScanBlockCounts.begin(), this->IScanBlockCounts.end(), gid + 1) -
this->IScanBlockCounts.begin());
return static_cast<int>(std::lower_bound(this->IScanPartitionCounts.begin(),
this->IScanPartitionCounts.end(),
gid + 1) -
this->IScanPartitionCounts.begin());
}
......
......@@ -7,8 +7,8 @@
// the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
// PURPOSE. See the above copyright notice for more information.
//============================================================================
#ifndef vtk_m_cont_AssignerMultiBlock_h
#define vtk_m_cont_AssignerMultiBlock_h
#ifndef vtk_m_cont_AssignerPartitionedDataSet_h
#define vtk_m_cont_AssignerPartitionedDataSet_h
#include <vtkm/cont/vtkm_cont_export.h>
......@@ -31,31 +31,32 @@ namespace vtkm
namespace cont
{
class MultiBlock;
class PartitionedDataSet;
/// \brief Assigner for `MultiBlock` blocks.
/// \brief Assigner for PartitionedDataSet partitions.
///
/// `AssignerMultiBlock` is a `vtkmdiy::StaticAssigner` implementation that uses
/// `MultiBlock`'s block distribution to build global-id/rank associations
/// needed for several `diy` operations.
/// It uses a contiguous assignment strategy to map blocks to global ids i.e.
/// blocks on rank 0 come first, then rank 1, etc. Any rank may have 0 blocks.
/// `AssignerPartitionedDataSet` is a `vtkmdiy::StaticAssigner` implementation
/// that uses `PartitionedDataSet`'s partition distribution to build
/// global-id/rank associations needed for several `diy` operations.
/// It uses a contiguous assignment strategy to map partitions to global ids,
/// i.e. partitions on rank 0 come first, then rank 1, etc. Any rank may have 0