diff --git a/Documentation/release/dev/fix-AMReXParticlesReader-in-MPI.md b/Documentation/release/dev/fix-AMReXParticlesReader-in-MPI.md new file mode 100644 index 0000000000000000000000000000000000000000..453dd612d9e64c72b618067cc371b3c086982af9 --- /dev/null +++ b/Documentation/release/dev/fix-AMReXParticlesReader-in-MPI.md @@ -0,0 +1,5 @@ +## Fix vtkAMReXParticlesReader when using MPI + +The `vtkAMReXParticlesReader` can now function correctly in parallel mode, with MPI. +Earlier, a bug caused incorrect output data when the number of grids were not +exactly divided by the number of MPI processes. diff --git a/IO/AMR/Testing/Cxx/CMakeLists.txt b/IO/AMR/Testing/Cxx/CMakeLists.txt index ad32895e353528dea26dd3191229e8a509ac5e3a..db475d9fb7781a195bce317beb2820dbd0901e63 100644 --- a/IO/AMR/Testing/Cxx/CMakeLists.txt +++ b/IO/AMR/Testing/Cxx/CMakeLists.txt @@ -6,6 +6,15 @@ ExternalData_Expand_Arguments(VTKData _ "DATA{${_vtk_build_TEST_INPUT_DATA_DIRECTORY}/Data/AMReX/NonZeroOrigin/,RECURSE:,REGEX:.*}" ) +if (TARGET VTK::ParallelMPI) + set(TestAMReXParticlesReader_NUMPROCS 3) + vtk_add_test_mpi(vtkIOAMRCxxTests-MPI tests + TESTING_DATA + TestAMReXParticlesReader.cxx,NO_VALID,NO_OUTPUT + ) + vtk_test_cxx_executable(vtkIOAMRCxxTests-MPI tests) +endif() + vtk_add_test_cxx(vtkIOAMRCxxTests tests NO_VALID NO_OUTPUT TestAMRFlashReader.cxx diff --git a/IO/AMR/Testing/Cxx/TestAMReXParticlesReader.cxx b/IO/AMR/Testing/Cxx/TestAMReXParticlesReader.cxx index cb24df8a92fdc22842b411a4373f8fec5ebfc1d0..3c4e032beb0fc5b1cba659477c616453ee8338e8 100644 --- a/IO/AMR/Testing/Cxx/TestAMReXParticlesReader.cxx +++ b/IO/AMR/Testing/Cxx/TestAMReXParticlesReader.cxx @@ -1,8 +1,16 @@ // SPDX-FileCopyrightText: Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen // SPDX-License-Identifier: BSD-3-Clause + +#include <cstdlib> +#if VTK_MODULE_ENABLE_VTK_ParallelMPI +#include "vtkMPIController.h" +#else +#include "vtkDummyController.h" +#endif + #include "vtkAMReXParticlesReader.h" #include "vtkDataArraySelection.h" -#include "vtkIdTypeArray.h" +#include "vtkLogger.h" #include "vtkMultiBlockDataSet.h" #include "vtkMultiPieceDataSet.h" #include "vtkNew.h" @@ -16,24 +24,40 @@ if (!(x)) \ { \ cerr << "FAILED: " << msg << endl; \ + controller->Finalize(); \ return EXIT_FAILURE; \ } \ } while (false) int Validate(vtkMultiBlockDataSet* mb) { + auto* controller = vtkMultiProcessController::GetGlobalController(); ensure(mb != nullptr, "expecting vtkMultiBlockDataSet."); ensure(mb->GetNumberOfBlocks() == 1, "expecting num-blocks == num-levels == 1"); auto mp = vtkMultiPieceDataSet::SafeDownCast(mb->GetBlock(0)); ensure(mp != nullptr, "expecting level is maintained in a vtkMultiPieceDataSet."); ensure(mp->GetNumberOfPieces() == 8, "expecting 8 datasets in level 0"); - for (int cc = 0; cc < 8; ++cc) + vtkIdType numberOfPointsPerProcess = 0; + for (unsigned int cc = 0; cc < mp->GetNumberOfPieces(); ++cc) { - auto pd = vtkPolyData::SafeDownCast(mp->GetPiece(cc)); - ensure(pd != nullptr, "expecting polydata for index " << cc); - ensure(pd->GetNumberOfPoints() > 0, "expecting non-null points."); - ensure(pd->GetPointData()->GetArray("density") != nullptr, "missing density"); + if (auto pd = vtkPolyData::SafeDownCast(mp->GetPiece(cc))) + { + ensure(pd != nullptr, "expecting polydata for index " << cc); + numberOfPointsPerProcess += pd->GetNumberOfPoints(); + ensure(numberOfPointsPerProcess > 0, "expecting non-null points."); + ensure(pd->GetPointData()->GetArray("density") != nullptr, "missing density"); + } + } + vtkIdType totalNumberOfPoints = 0; + controller->AllReduce( + &numberOfPointsPerProcess, &totalNumberOfPoints, 1, vtkCommunicator::SUM_OP); + if (totalNumberOfPoints != 9776) + { + vtkLog(ERROR, << "# points per process: " << numberOfPointsPerProcess); + vtkLog(ERROR, << "Expected total # points: 9776"); + vtkLog(ERROR, << "Got total # points: " << totalNumberOfPoints); + return EXIT_FAILURE; } return EXIT_SUCCESS; @@ -41,6 +65,16 @@ int Validate(vtkMultiBlockDataSet* mb) int TestAMReXParticlesReader(int argc, char* argv[]) { +#if VTK_MODULE_ENABLE_VTK_ParallelMPI + vtkNew<vtkMPIController> controller; +#else + vtkNew<vtkDummyController> controller; +#endif + controller->Initialize(&argc, &argv); + const int processId = controller->GetLocalProcessId(); + const int numberOfProcesses = controller->GetNumberOfProcesses(); + vtkLogger::SetThreadName("processId=" + std::to_string(processId)); + vtkMultiProcessController::SetGlobalController(controller); // Test 3D { char* fname = vtkTestUtilities::ExpandDataFileName(argc, argv, "Data/AMReX/MFIX-Exa/plt00000"); @@ -53,9 +87,10 @@ int TestAMReXParticlesReader(int argc, char* argv[]) reader->UpdateInformation(); ensure(reader->GetPointDataArraySelection()->ArrayIsEnabled("proc") == 0, "`proc` should be disabled."); - reader->Update(); + reader->UpdatePiece(processId, numberOfProcesses, 0); if (Validate(reader->GetOutput()) == EXIT_FAILURE) { + controller->Finalize(); return EXIT_FAILURE; } } @@ -76,5 +111,6 @@ int TestAMReXParticlesReader(int argc, char* argv[]) ensure(bds[4] == bds[5], "expecting 2D dataset"); } + controller->Finalize(); return EXIT_SUCCESS; } diff --git a/IO/AMR/vtk.module b/IO/AMR/vtk.module index 7893328ce0aee3422325634a57a5ba36291fe66a..ea7ed46186eafd4aaacdc588fa2b7020f859cc89 100644 --- a/IO/AMR/vtk.module +++ b/IO/AMR/vtk.module @@ -24,3 +24,6 @@ TEST_DEPENDS VTK::IOXML VTK::TestingCore VTK::TestingRendering +TEST_OPTIONAL_DEPENDS + VTK::ParallelMPI + VTK::ParallelCore diff --git a/IO/AMR/vtkAMReXParticlesReader.cxx b/IO/AMR/vtkAMReXParticlesReader.cxx index 68142f3fa6b6fc8603149e5b53a89b15829b26f7..ca86add6267ef208ee9b196ff91ceb51ac0360ec 100644 --- a/IO/AMR/vtkAMReXParticlesReader.cxx +++ b/IO/AMR/vtkAMReXParticlesReader.cxx @@ -831,7 +831,8 @@ bool vtkAMReXParticlesReader::ReadLevel( const int quotient = num_grids / num_pieces; const int remainder = num_grids % num_pieces; - const int start_grid_idx = (piece_idx * quotient) + ((piece_idx < remainder) ? 1 : 0); + const int start_grid_idx = + (piece_idx * quotient) + ((piece_idx < remainder) ? piece_idx : remainder); const int grids_count = quotient + ((piece_idx < remainder) ? 1 : 0); levelDS->SetNumberOfPieces(num_grids);