diff --git a/Documentation/release/dev/htg-gcg-partitioned.md b/Documentation/release/dev/htg-gcg-partitioned.md
new file mode 100644
index 0000000000000000000000000000000000000000..92c43daa7daf351e217b63d7bff584a6d2457eff
--- /dev/null
+++ b/Documentation/release/dev/htg-gcg-partitioned.md
@@ -0,0 +1,3 @@
+# Support Partitioned Input in HyperTreeGridGhostCellsGenerator
+
+The HTG GhostCellsGenerator now natively supports Partitioned inputs, using the first non null partition found as the HTG to process. Before, partitions were processed one by one, causing issues on multi-partition data.
diff --git a/Filters/Parallel/Testing/Cxx/TestHyperTreeGridGhostCellsGenerator.cxx b/Filters/Parallel/Testing/Cxx/TestHyperTreeGridGhostCellsGenerator.cxx
index 7c8f1a3cc90b383e47f4da3ed34efeaf052763c1..e6f58716ea4e6e62498cf794a899436d1c79c78b 100644
--- a/Filters/Parallel/Testing/Cxx/TestHyperTreeGridGhostCellsGenerator.cxx
+++ b/Filters/Parallel/Testing/Cxx/TestHyperTreeGridGhostCellsGenerator.cxx
@@ -14,6 +14,7 @@
#include "vtkTestUtilities.h"
#include "vtkUnsignedCharArray.h"
#include "vtkXMLHyperTreeGridReader.h"
+#include <vtkPartitionedDataSet.h>
namespace
{
@@ -310,7 +311,6 @@ int TestGhostNullPart(vtkMPIController* controller)
// Create GCG
vtkNew<vtkHyperTreeGridGhostCellsGenerator> generator;
- generator->SetDebug(true);
generator->SetInputConnection(htgSource->GetOutputPort());
vtkSmartPointer<vtkHyperTreeGrid> htg(generator->GetHyperTreeGridOutput());
if (generator->UpdatePiece(myRank, nbRanks, 0) != 1)
@@ -356,8 +356,6 @@ int TestGhostNullPart(vtkMPIController* controller)
*/
int TestGhostSinglePiece(vtkMPIController* controller, const std::string& filename)
{
- int ret = EXIT_SUCCESS;
-
int myRank = controller->GetLocalProcessId();
int nbRanks = controller->GetNumberOfProcesses();
@@ -386,8 +384,106 @@ int TestGhostSinglePiece(vtkMPIController* controller, const std::string& filena
vtkErrorWithObjectMacro(nullptr, << "Wrong number of ghost cells generated for process "
<< myRank << ". Has " << nbCellsAfter << " but expect "
<< nbCellsBefore);
- ret = EXIT_FAILURE;
+ return EXIT_FAILURE;
+ }
+ return EXIT_SUCCESS;
+}
+
+/**
+ * HTG GhostCells should handle properly data separated in multiple partitions inside of a
+ * PartitionedDataSet (PDS). It should not matter whether the PDS has a single partition containing
+ * the data for all ranks, or multiple, that may or may not correspond to the number of ranks. This
+ * can happen for example when you write the PDS in an MPI setting of X ranks, but open it back with
+ * Y ranks.
+ *
+ * `config` sets up different PDS configuration schemes:
+ * - 0 will test the classic case of 1 different partition for each piece
+ * - 1 will test with only 1 partition containing a distributed HTG
+ * - 2 will test a PDS with 2 partitions, containing HTGs on 2 ranks each
+ */
+int TestPartitionedHTG(vtkMPIController* controller, int config)
+{
+ int myRank = controller->GetLocalProcessId();
+ int nbRanks = controller->GetNumberOfProcesses();
+
+ vtkNew<vtkRandomHyperTreeGridSource> htgSource;
+ htgSource->SetSeed(3);
+ htgSource->SetMaxDepth(3);
+ htgSource->SetDimensions(3, 3, 3);
+ htgSource->UpdatePiece(myRank, nbRanks, 0);
+ vtkNew<vtkPartitionedDataSet> pdsSource;
+
+ vtkHyperTreeGrid* inputHTG = htgSource->GetHyperTreeGridOutput();
+
+ // In which partition to place data
+ const std::array<unsigned int, 4> configPartition = { static_cast<unsigned int>(myRank), 0,
+ static_cast<unsigned int>(myRank % 2) };
+ pdsSource->SetPartition(configPartition[config], inputHTG);
+
+ // How many parts total
+ const std::array<unsigned int, 3> configNumberOfParts = { static_cast<unsigned int>(nbRanks), 1,
+ 2 };
+ pdsSource->SetNumberOfPartitions(configNumberOfParts[config]);
+
+ // Create and execute GCG
+ vtkNew<vtkHyperTreeGridGhostCellsGenerator> generator;
+ generator->SetDebug(true);
+ generator->SetInputData(pdsSource);
+ vtkSmartPointer<vtkPartitionedDataSet> outputPDS =
+ vtkPartitionedDataSet::SafeDownCast(generator->GetOutputDataObject(0));
+ if (generator->UpdatePiece(myRank, nbRanks, 0) != 1)
+ {
+ vtkErrorWithObjectMacro(nullptr, << "Fail to update piece for process " << myRank);
+ return EXIT_FAILURE;
+ }
+
+ if (outputPDS->GetNumberOfPartitions() != configNumberOfParts[config])
+ {
+ vtkErrorWithObjectMacro(
+ nullptr, << "Expected 4 partitions in output PartitionedDataSet but got "
+ << outputPDS->GetNumberOfPartitions());
+ return EXIT_FAILURE;
}
+
+ int ret = EXIT_SUCCESS;
+
+ // Only one partition on each rank is expected to be non-null.
+ const std::array<vtkIdType, 4> expectedNbOfCells = { 336, 288, 408, 240 };
+ for (unsigned int partId = 0; partId < outputPDS->GetNumberOfPartitions(); partId++)
+ {
+ vtkHyperTreeGrid* partHTG =
+ vtkHyperTreeGrid::SafeDownCast(outputPDS->GetPartitionAsDataObject(partId));
+ if (partId != configPartition[config])
+ {
+ if (partHTG)
+ {
+ vtkErrorWithObjectMacro(nullptr,
+ << "Partition " << partId << " on rank " << myRank << " should be null, but is not.");
+ ret = EXIT_FAILURE;
+ }
+ }
+ else
+ {
+ if (!partHTG)
+ {
+ vtkErrorWithObjectMacro(
+ nullptr, << "Partition " << partId << " on rank " << myRank << " should not be null.");
+ ret = EXIT_FAILURE;
+ }
+ else
+ {
+ vtkIdType nbCellsAfterGCG = partHTG->GetNumberOfCells();
+ if (expectedNbOfCells[myRank] != nbCellsAfterGCG)
+ {
+ vtkErrorWithObjectMacro(nullptr, << "Wrong number of ghost cells generated for process "
+ << myRank << ". Has " << nbCellsAfterGCG
+ << " but expect " << expectedNbOfCells[myRank]);
+ ret = EXIT_FAILURE;
+ }
+ }
+ }
+ }
+
return ret;
}
}
@@ -426,6 +522,9 @@ int TestHyperTreeGridGhostCellsGenerator(int argc, char* argv[])
ret |= ::TestGhost2D(controller);
ret |= ::TestGhostNullPart(controller);
ret |= ::TestGhostSinglePiece(controller, htgFileName);
+ ret |= ::TestPartitionedHTG(controller, 0);
+ ret |= ::TestPartitionedHTG(controller, 1);
+ ret |= ::TestPartitionedHTG(controller, 2);
controller->Finalize();
return ret;
diff --git a/Filters/Parallel/vtkHyperTreeGridGhostCellsGenerator.cxx b/Filters/Parallel/vtkHyperTreeGridGhostCellsGenerator.cxx
index ded590cec2fd6346afceb27462e4b0ad0d726696..e3baeb6a054c60973bf4141f819c698cea1acbf0 100644
--- a/Filters/Parallel/vtkHyperTreeGridGhostCellsGenerator.cxx
+++ b/Filters/Parallel/vtkHyperTreeGridGhostCellsGenerator.cxx
@@ -11,16 +11,31 @@
#include "vtkInformationVector.h"
#include "vtkMultiProcessController.h"
#include "vtkObjectFactory.h"
+#include "vtkPartitionedDataSet.h"
#include "vtkSetGet.h"
#include "vtkStreamingDemandDrivenPipeline.h"
VTK_ABI_NAMESPACE_BEGIN
vtkStandardNewMacro(vtkHyperTreeGridGhostCellsGenerator);
+vtkCxxSetObjectMacro(vtkHyperTreeGridGhostCellsGenerator, Controller, vtkMultiProcessController);
//------------------------------------------------------------------------------
vtkHyperTreeGridGhostCellsGenerator::vtkHyperTreeGridGhostCellsGenerator()
{
this->AppropriateOutput = true;
+ this->SetController(vtkMultiProcessController::GetGlobalController());
+}
+
+//------------------------------------------------------------------------------
+vtkHyperTreeGridGhostCellsGenerator::~vtkHyperTreeGridGhostCellsGenerator()
+{
+ this->SetController(nullptr);
+}
+
+//------------------------------------------------------------------------------
+vtkMultiProcessController* vtkHyperTreeGridGhostCellsGenerator::GetController()
+{
+ return this->Controller.Get();
}
//------------------------------------------------------------------------------
@@ -30,9 +45,10 @@ void vtkHyperTreeGridGhostCellsGenerator::PrintSelf(ostream& os, vtkIndent inden
}
//------------------------------------------------------------------------------
-int vtkHyperTreeGridGhostCellsGenerator::FillOutputPortInformation(int, vtkInformation* info)
+int vtkHyperTreeGridGhostCellsGenerator::FillInputPortInformation(int, vtkInformation* info)
{
- info->Set(vtkDataObject::DATA_TYPE_NAME(), "vtkHyperTreeGrid");
+ info->Set(vtkAlgorithm::INPUT_REQUIRED_DATA_TYPE(), "vtkHyperTreeGrid");
+ info->Append(vtkAlgorithm::INPUT_REQUIRED_DATA_TYPE(), "vtkPartitionedDataSet");
return 1;
}
@@ -42,40 +58,89 @@ int vtkHyperTreeGridGhostCellsGenerator::RequestData(vtkInformation* vtkNotUsed(
{
this->UpdateProgress(0.);
- // Retrieve input and output
- vtkHyperTreeGrid* input = vtkHyperTreeGrid::GetData(inputVector[0], 0);
- if (!input)
+ vtkInformation* info = outputVector->GetInformationObject(0);
+ int currentPiece = info->Get(vtkStreamingDemandDrivenPipeline::UPDATE_PIECE_NUMBER());
+
+ // Make sure input is either a HTG or a PartitionedDataSet that contains a HTG piece.
+ vtkHyperTreeGrid* inputHTG = vtkHyperTreeGrid::GetData(inputVector[0], 0);
+ vtkPartitionedDataSet* inputPDS = vtkPartitionedDataSet::GetData(inputVector[0], 0);
+
+ if (!inputPDS && !inputHTG)
{
- vtkErrorMacro("No input available. Cannot proceed with hyper tree grid algorithm.");
+ vtkErrorMacro("Input data is neither HTG or PartitionedDataSet. Cannot proceed with ghost cell "
+ "generation.");
return 0;
}
- vtkDataObject* outputDO = vtkDataObject::GetData(outputVector, 0);
- if (!outputDO)
+
+ vtkHyperTreeGrid* outputHTG = vtkHyperTreeGrid::GetData(outputVector, 0);
+ vtkPartitionedDataSet* outputPDS = vtkPartitionedDataSet::GetData(outputVector, 0);
+ if (outputPDS)
+ {
+ outputPDS->CopyStructure(inputPDS);
+ }
+
+ // When the filter receives a PartitionedDataSet, the data for the current rank can be in either
+ // partition, depending on the data generation method. We survey the partitions to find the one
+ // that contains the actual data. There should be exactly one non-null HTG partition in each
+ // piece. If we find multiple, the HTG structure is not capable of merging multiple grids, so we
+ // simply use the last one.
+ if (inputPDS && outputPDS)
+ {
+ for (unsigned int partId = 0; partId < inputPDS->GetNumberOfPartitions(); partId++)
+ {
+ auto partHTG = vtkHyperTreeGrid::SafeDownCast(inputPDS->GetPartitionAsDataObject(partId));
+ if (partHTG)
+ {
+ if (inputHTG)
+ {
+ vtkWarningMacro("Found more than one non-null HTG in the partitioned dataset for piece "
+ << currentPiece << ". Generating ghost data only for partition " << partId);
+ }
+ inputHTG = partHTG;
+ vtkNew<vtkHyperTreeGrid> newOutputHTG;
+ outputPDS->SetPartition(partId, newOutputHTG);
+ outputHTG = newOutputHTG; // Not dangling, outputPDS maintains a reference.
+ }
+ }
+ }
+
+ if (!outputHTG && !outputPDS)
{
vtkErrorMacro("No output available. Cannot proceed with hyper tree grid algorithm.");
return 0;
}
- int correctExtent = input->GetExtent()[0] <= input->GetExtent()[1] &&
- input->GetExtent()[2] <= input->GetExtent()[3] &&
- input->GetExtent()[4] <= input->GetExtent()[5];
+ if (!inputHTG)
+ {
+ vtkWarningMacro("Incorrect HTG for piece " << currentPiece);
+ }
// Make sure every HTG piece has a correct extent and can be processed.
// This way, we make sure the `ProcessTrees` function will either be executed by all ranks
// or by none, and avoids getting stuck on barriers.
+ int correctExtent = inputHTG && inputHTG->GetExtent()[0] <= inputHTG->GetExtent()[1] &&
+ inputHTG->GetExtent()[2] <= inputHTG->GetExtent()[3] &&
+ inputHTG->GetExtent()[4] <= inputHTG->GetExtent()[5];
+
+ if (!correctExtent)
+ {
+ vtkWarningMacro("Piece " << currentPiece << " does not have a valid extend. Cannot process.");
+ }
+
int allCorrect = 1; // Reduction operation cannot be done on bools
- vtkMultiProcessController* controller = vtkMultiProcessController::GetGlobalController();
- controller->AllReduce(&correctExtent, &allCorrect, 1, vtkCommunicator::LOGICAL_AND_OP);
+ this->Controller->AllReduce(&correctExtent, &allCorrect, 1, vtkCommunicator::LOGICAL_AND_OP);
if (!allCorrect)
{
vtkWarningMacro("Every individual distributed process does not have a valid HTG extent. No "
"ghost cells will be generated.");
- vtkHyperTreeGrid* output = vtkHyperTreeGrid::SafeDownCast(outputDO);
- output->ShallowCopy(input);
+ if (outputHTG)
+ {
+ outputHTG->ShallowCopy(inputHTG);
+ }
return 1;
}
- else if (!this->ProcessTrees(input, outputDO))
+ else if (!this->ProcessTrees(inputHTG, outputHTG))
{
return 0;
}
@@ -89,8 +154,7 @@ int vtkHyperTreeGridGhostCellsGenerator::RequestData(vtkInformation* vtkNotUsed(
int vtkHyperTreeGridGhostCellsGenerator::ProcessTrees(
vtkHyperTreeGrid* input, vtkDataObject* outputDO)
{
- vtkMultiProcessController* controller = vtkMultiProcessController::GetGlobalController();
- int numberOfProcesses = controller->GetNumberOfProcesses();
+ int numberOfProcesses = this->Controller->GetNumberOfProcesses();
vtkHyperTreeGrid* output = vtkHyperTreeGrid::SafeDownCast(outputDO);
if (!output)
@@ -120,7 +184,7 @@ int vtkHyperTreeGridGhostCellsGenerator::ProcessTrees(
output->GetCellData()->CopyStructure(input->GetCellData());
}
- vtkHyperTreeGridGhostCellsGeneratorInternals subroutines{ this, controller, input, output };
+ vtkHyperTreeGridGhostCellsGeneratorInternals subroutines{ this, this->Controller, input, output };
subroutines.InitializeCellData();
this->UpdateProgress(0.1);
@@ -139,7 +203,7 @@ int vtkHyperTreeGridGhostCellsGenerator::ProcessTrees(
vtkErrorMacro("Failure during size exchange, aborting.");
return 0;
}
- controller->Barrier();
+ this->Controller->Barrier();
this->UpdateProgress(0.4);
vtkDebugMacro("Exchange tree decomposition and masks with neighbors");
@@ -148,7 +212,7 @@ int vtkHyperTreeGridGhostCellsGenerator::ProcessTrees(
vtkErrorMacro("Failure during mask exchange, aborting.");
return 0;
}
- controller->Barrier();
+ this->Controller->Barrier();
this->UpdateProgress(0.6);
vtkDebugMacro("Exchange cell data with neighbors");
@@ -158,7 +222,7 @@ int vtkHyperTreeGridGhostCellsGenerator::ProcessTrees(
return 0;
}
- controller->Barrier();
+ this->Controller->Barrier();
this->UpdateProgress(0.8);
vtkDebugMacro("Create ghost array and set output mask");
diff --git a/Filters/Parallel/vtkHyperTreeGridGhostCellsGenerator.h b/Filters/Parallel/vtkHyperTreeGridGhostCellsGenerator.h
index 58c21c81e9ae511ab698b6cc07ed1a2889312f70..265a5b1e32f09133279952a41f4c798155e1e70b 100644
--- a/Filters/Parallel/vtkHyperTreeGridGhostCellsGenerator.h
+++ b/Filters/Parallel/vtkHyperTreeGridGhostCellsGenerator.h
@@ -2,16 +2,22 @@
// SPDX-License-Identifier: BSD-3-Clause
/**
* @class vtkHyperTreeGridGhostCellsGenerator
- * @brief Generated ghost cells (HyperTree's distributed).
+ * @brief Generate ghost cells for distributed vtkHyperTreeGrids
*
- * This filter generates ghost cells for vtkHyperTreeGrid type data. The input vtkHyperTreeGrid
- * should have hyper trees distributed to a single process. This filter produces ghost hyper trees
- * at the interfaces between different processes, only composed of the nodes and leafs at this
- * interface to avoid data waste.
+ * This filter generates ghost cells for vtkHyperTreeGrid type data.
+ * It can also take a vtkPartitionedDataSet composed of HyperTreeGrid partition,
+ * where each process has a single non-null partition. In case of a PartitionedDataSet, the output
+ * structure is identical to the input structure.
+ *
+ * This filter produces ghost hyper trees at the interfaces between different processes,
+ * only composed of the nodes and leaves at this interface to avoid data waste.
*
* This filter should be used in a multi-processes environment, and is only required if wanting to
* filter a vtkHyperTreeGrid with algorithms using Von Neumann or Moore supercursors afterwards.
*
+ * All processes should have a single HTG with a correct extent, even if it does not contain any
+ * actual unmasked cells.
+ *
* @par Thanks:
* This class was written by Jacques-Bernard Lekien, 2019
* This work was supported by Commissariat a l'Energie Atomique
@@ -23,6 +29,7 @@
#include "vtkFiltersParallelModule.h" // For export macro
#include "vtkHyperTreeGridAlgorithm.h"
+#include "vtkWeakPointer.h" // for vtkWeakPointer
#include <vector> // For vtkHypertreeGridGhostCellsGenerator::ExtractInterface
@@ -42,16 +49,25 @@ public:
vtkTypeMacro(vtkHyperTreeGridGhostCellsGenerator, vtkHyperTreeGridAlgorithm);
void PrintSelf(ostream& os, vtkIndent indent) override;
+ ///@{
+ /**
+ * Get/Set the controller to use. By default
+ * vtkMultiProcessController::GlobalController will be used.
+ */
+ virtual void SetController(vtkMultiProcessController*);
+ vtkMultiProcessController* GetController();
+ ///@}
+
protected:
vtkHyperTreeGridGhostCellsGenerator();
- ~vtkHyperTreeGridGhostCellsGenerator() override = default;
+ ~vtkHyperTreeGridGhostCellsGenerator() override;
struct vtkInternals;
/**
- * For this algorithm the output is a vtkHyperTreeGrid instance
+ * Input must be either HTG or vtkPartitionnedDataSet composed of HTG partitions.
*/
- int FillOutputPortInformation(int, vtkInformation*) override;
+ int FillInputPortInformation(int, vtkInformation*) override;
/**
* Override RequestData, to make sure every HTG piece can be processed, hence avoiding that one
@@ -97,6 +113,8 @@ protected:
private:
vtkHyperTreeGridGhostCellsGenerator(const vtkHyperTreeGridGhostCellsGenerator&) = delete;
void operator=(const vtkHyperTreeGridGhostCellsGenerator&) = delete;
+
+ vtkWeakPointer<vtkMultiProcessController> Controller;
};
VTK_ABI_NAMESPACE_END
diff --git a/Filters/Parallel/vtkHyperTreeGridGhostCellsGeneratorInternals.cxx b/Filters/Parallel/vtkHyperTreeGridGhostCellsGeneratorInternals.cxx
index 3ce31af0a47ae6b4ee67908294ec41e25812855e..8d51727e25823054e6ddaec52cd244a25f46460e 100644
--- a/Filters/Parallel/vtkHyperTreeGridGhostCellsGeneratorInternals.cxx
+++ b/Filters/Parallel/vtkHyperTreeGridGhostCellsGeneratorInternals.cxx
@@ -15,6 +15,7 @@
#include "vtkSmartPointer.h"
#include "vtkUnsignedCharArray.h"
+#include <cassert>
#include <vector>
VTK_ABI_NAMESPACE_BEGIN
diff --git a/Filters/ParallelDIY2/vtkGhostCellsGenerator.cxx b/Filters/ParallelDIY2/vtkGhostCellsGenerator.cxx
index 64f9a1a5e8168f819c5154523f8360795cd40c19..2342053fb97c1c731e98987e02527bb87ffe29ad 100644
--- a/Filters/ParallelDIY2/vtkGhostCellsGenerator.cxx
+++ b/Filters/ParallelDIY2/vtkGhostCellsGenerator.cxx
@@ -190,8 +190,14 @@ int vtkGhostCellsGenerator::GenerateGhostCells(
error = true;
}
- if (vtkHyperTreeGrid::SafeDownCast(inputPartition) ||
- vtkExplicitStructuredGrid::SafeDownCast(inputPartition))
+ if (vtkHyperTreeGrid::SafeDownCast(inputPartition))
+ {
+ error = true;
+ vtkErrorMacro(<< "vtkHyperTreeGrid is not supported by this filter. Please use the "
+ "appropriate vtkHyperTreeGridGhostCellsGenerator filter instead.");
+ }
+
+ if (vtkExplicitStructuredGrid::SafeDownCast(inputPartition))
{
error = true;
vtkErrorMacro(<< "Input data set type " << inputPartition->GetClassName()