Add vtkMPIMultiBlockPLOT3DReader -- MPI-IO enabled reader for Plot3D.

Adding new vtkMPIMultiBlockPLOT3DReader which uses MPI-IO to read plot3D
files in parallel. vtkMPIMultiBlockPLOT3DReader overrides
vtkMultiBlockPLOT3DReader using vtkObjectFactory mechanisms. It only
supports reading binary 3D-Plot3D files.

IO/MPIParallel/CMakeLists.txt

 include(vtkMPI)
 
-# We don't build the parallel WindBlade reader on windows
+# We don't build the parallel readers on windows
 # because there's problems with the MPI_File_open()
 # function when there's paths in the filenames.
-set(PWindBladeReader vtkPWindBladeReader.cxx)
+set (vtk_module_overrides)
 if(WIN32)
-  set(PWindBladeReader)
-endif()
-
-set(Module_SRCS
-  ${PWindBladeReader}
-  ${CMAKE_CURRENT_BINARY_DIR}/${vtk-module}ObjectFactory.cxx
-  )
+  set(Module_SRCS)
+else()
+  set(Module_SRCS
+    vtkPWindBladeReader.cxx
+    vtkMPIMultiBlockPLOT3DReader.cxx
+    )
 
-set_source_files_properties(
-  ${vtk-module}ObjectFactory
-  WRAP_EXCLUDE
-  )
+  vtk_add_override(vtkWindBladeReader vtkPWindBladeReader)
+  vtk_add_override(vtkMultiBlockPLOT3DReader vtkMPIMultiBlockPLOT3DReader)
+endif()
 
 # Now to generate our object factory.
-set( vtk_module_overrides
-  WindBladeReader
+vtk_object_factory_configure("${vtk_module_overrides}")
+list(APPEND Module_SRCS
+  ${CMAKE_CURRENT_BINARY_DIR}/${vtk-module}ObjectFactory.cxx
   )
-if(WIN32)
-  set( vtk_module_overrides )
-endif()
-
-foreach(_class ${vtk_module_overrides})
-  set(_override vtkP${_class})
-  set(_vtk_override_includes "${_vtk_override_includes} #include \"${_override}.h\"")
-  set(_vtk_override_creates "${_vtk_override_creates}
-VTK_CREATE_CREATE_FUNCTION(${_override})")
-  set(_vtk_override_do "${_vtk_override_do}
-  this->RegisterOverride(\"vtk${_class}\",
-                         \"${_override}\",
-                         \"Override for ${vtk-module} module\", 1,
-                         vtkObjectFactoryCreate${_override});")
-endforeach()
-
-# Now lets create the object factory classes
-string(TOUPPER ${vtk-module} VTK-MODULE)
-configure_file(${VTK_CMAKE_DIR}/vtkObjectFactory.h.in
-  ${CMAKE_CURRENT_BINARY_DIR}/${vtk-module}ObjectFactory.h)
-configure_file(${VTK_CMAKE_DIR}/vtkObjectFactory.cxx.in
-  ${CMAKE_CURRENT_BINARY_DIR}/${vtk-module}ObjectFactory.cxx)
-
+set_source_files_properties(${vtk-module}ObjectFactory WRAP_EXCLUDE)
 vtk_module_library(${vtk-module} ${Module_SRCS})
 vtk_mpi_link(${vtk-module})

IO/MPIParallel/module.cmake

@@ -5,9 +5,10 @@ vtk_module(vtkIOMPIParallel
     MPI
   DEPENDS
     vtkIOGeometry
-    vtkParallelMPI
+    vtkIOParallel
   PRIVATE_DEPENDS
     vtksys
+    vtkParallelMPI
   TEST_DEPENDS
     vtkRendering${VTK_RENDERING_BACKEND}
     vtkTestingRendering

IO/MPIParallel/vtkMPIMultiBlockPLOT3DReader.cxx

+/*=========================================================================
+
+  Program:   Visualization Toolkit
+  Module:    vtkMPIMultiBlockPLOT3DReader.cxx
+
+  Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
+  All rights reserved.
+  See Copyright.txt or http://www.kitware.com/Copyright.htm for details.
+
+     This software is distributed WITHOUT ANY WARRANTY; without even
+     the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+     PURPOSE.  See the above copyright notice for more information.
+
+=========================================================================*/
+#include "vtkMPIMultiBlockPLOT3DReader.h"
+
+#include "vtkByteSwap.h"
+#include "vtkDataArrayDispatcher.h"
+#include "vtkErrorCode.h"
+#include "vtkMPICommunicator.h"
+#include "vtkMPIController.h"
+#include "vtkMPI.h"
+#include "vtkMultiBlockPLOT3DReaderInternals.h"
+#include "vtkObjectFactory.h"
+#include "vtkStructuredData.h"
+
+#include <exception>
+#include <cassert>
+
+#define DEFINE_MPI_TYPE(ctype, mpitype) \
+  template <> struct mpi_type<ctype> { static MPI_Datatype type() { return mpitype; }  };
+
+namespace
+{
+  template <class T> struct mpi_type {};
+  DEFINE_MPI_TYPE(char, MPI_CHAR);
+  DEFINE_MPI_TYPE(signed char, MPI_SIGNED_CHAR);
+  DEFINE_MPI_TYPE(unsigned char, MPI_UNSIGNED_CHAR);
+  DEFINE_MPI_TYPE(short, MPI_SHORT);
+  DEFINE_MPI_TYPE(unsigned short, MPI_UNSIGNED_SHORT);
+  DEFINE_MPI_TYPE(int, MPI_INT);
+  DEFINE_MPI_TYPE(unsigned int, MPI_UNSIGNED);
+  DEFINE_MPI_TYPE(long, MPI_LONG);
+  DEFINE_MPI_TYPE(unsigned long, MPI_UNSIGNED_LONG);
+  DEFINE_MPI_TYPE(float, MPI_FLOAT);
+  DEFINE_MPI_TYPE(double, MPI_DOUBLE);
+#if defined(VTK_TYPE_USE_LONG_LONG)
+  DEFINE_MPI_TYPE(long long, MPI_LONG_LONG);
+  DEFINE_MPI_TYPE(unsigned long long, MPI_UNSIGNED_LONG_LONG);
+#endif
+#if defined(VTK_TYPE_USE___INT64)
+# if VTK_SIZEOF_LONG == 8
+  DEFINE_MPI_TYPE(__int64, MPI_LONG);
+  DEFINE_MPI_TYPE(unsigned __int64, MPI_UNSIGNED_LONG);
+# elif defined(MPI_LONG_LONG)
+  DEFINE_MPI_TYPE(__int64, MPI_LONG_LONG);
+  DEFINE_MPI_TYPE(__int64, MPI_UNSIGNED_LONG_LONG);
+# endif
+#endif
+
+  class MPIPlot3DException : public std::exception
+  {
+  };
+
+  class SetupFileViewFunctor
+  {
+    vtkMPICommunicatorOpaqueComm& Communicator;
+    vtkMPIOpaqueFileHandle& Handle;
+    int Extent[6];
+    int WExtent[6];
+    vtkTypeUInt64 Offset;
+    int ByteOrder;
+
+  public:
+    SetupFileViewFunctor(
+      vtkMPICommunicatorOpaqueComm& comm,
+      vtkMPIOpaqueFileHandle& handle,
+        const int extent[6], const int wextent[6], vtkTypeUInt64 offset, int byteOrder)
+      : Communicator(comm), Handle(handle), Offset(offset), ByteOrder(byteOrder)
+    {
+    std::copy(extent, extent+6, this->Extent);
+    std::copy(wextent, wextent+6, this->WExtent);
+    }
+
+    template <class T>
+    void operator()(vtkDataArrayDispatcherPointer<T> array) const
+    {
+    int array_of_sizes[3];
+    int array_of_subsizes[3];
+    int array_of_starts[3];
+    for (int cc=0; cc < 3; ++cc)
+      {
+      array_of_sizes[cc]    = this->WExtent[cc*2+1] - this->WExtent[cc*2] + 1;
+      array_of_subsizes[cc] = this->Extent[cc*2+1] - this->Extent[cc*2] + 1;
+      array_of_starts[cc]   = this->Extent[cc*2];
+      }
+
+    // Define the file type for this process.
+    MPI_Datatype filetype;
+    if (MPI_Type_create_subarray(3,
+        array_of_sizes,
+        array_of_subsizes,
+        array_of_starts,
+        MPI_ORDER_FORTRAN,
+        mpi_type<T>::type(),
+        &filetype) != MPI_SUCCESS)
+      {
+      throw MPIPlot3DException();
+      }
+    MPI_Type_commit(&filetype);
+
+    // For each component.
+    const vtkIdType valuesToRead = vtkStructuredData::GetNumberOfPoints(
+      const_cast<int*>(this->Extent));
+    const vtkIdType valuesTotal = vtkStructuredData::GetNumberOfPoints(
+      const_cast<int*>(this->WExtent));
+
+    const int _INT_MAX = 2e9 / sizeof(T); /// XXX: arbitrary limit that seems
+                                          /// to work when reading large files.
+
+    int numIterations = static_cast<int>(valuesToRead/_INT_MAX) + 1;
+    int maxNumIterations;
+    MPI_Allreduce(&numIterations, &maxNumIterations, 1, mpi_type<int>::type(),
+      MPI_MAX, *this->Communicator.GetHandle());
+
+    vtkTypeUInt64 byteOffset = this->Offset;
+    // simply use the VTK array for scalars.
+    T *buffer = array.NumberOfComponents == 1? array.RawPointer : new T[valuesToRead];
+    try
+      {
+      for (int comp=0; comp < array.NumberOfComponents; ++comp)
+        {
+        // Define the view.
+        if (MPI_File_set_view(this->Handle.Handle, byteOffset, mpi_type<T>::type(),
+            filetype, const_cast<char*>("native"), MPI_INFO_NULL) != MPI_SUCCESS)
+          {
+          throw MPIPlot3DException();
+          }
+
+        // Read contents from the file in 2GB chunks.
+        vtkIdType valuesRead = 0;
+        for (int iteration = 0; iteration < maxNumIterations; ++iteration)
+          {
+          vtkIdType valuesToReadInIteration =
+            std::min((valuesToRead - valuesRead), static_cast<vtkIdType>(_INT_MAX));
+          if (MPI_File_read_all(this->Handle.Handle, buffer + valuesRead,
+              static_cast<int>(valuesToReadInIteration),
+              mpi_type<T>::type(), MPI_STATUS_IGNORE) != MPI_SUCCESS)
+            {
+            throw MPIPlot3DException();
+            }
+          valuesRead += valuesToReadInIteration;
+          }
+        if (buffer != array.RawPointer)
+          {
+          for (vtkIdType i=0; i < valuesToRead; ++i)
+            {
+            array.RawPointer[array.NumberOfComponents*i + comp] = buffer[i];
+            }
+          }
+        byteOffset += valuesTotal * sizeof(T);
+        }
+      }
+    catch (MPIPlot3DException)
+      {
+      if (buffer != array.RawPointer)
+        {
+        delete [] buffer;
+        }
+      MPI_Type_free(&filetype);
+      throw MPIPlot3DException();
+      }
+    if (buffer != array.RawPointer)
+      {
+      delete [] buffer;
+      }
+    MPI_Type_free(&filetype);
+    // finally, handle byte order.
+    if (this->ByteOrder == vtkMultiBlockPLOT3DReader::FILE_LITTLE_ENDIAN)
+      {
+      vtkByteSwap::SwapLERange(array.RawPointer, array.NumberOfComponents*array.NumberOfTuples);
+      }
+    else
+      {
+      vtkByteSwap::SwapBERange(array.RawPointer, array.NumberOfComponents*array.NumberOfTuples);
+      }
+    }
+  };
+}
+
+vtkStandardNewMacro(vtkMPIMultiBlockPLOT3DReader);
+//----------------------------------------------------------------------------
+vtkMPIMultiBlockPLOT3DReader::vtkMPIMultiBlockPLOT3DReader()
+{
+  this->UseMPIIO = true;
+}
+
+//----------------------------------------------------------------------------
+vtkMPIMultiBlockPLOT3DReader::~vtkMPIMultiBlockPLOT3DReader()
+{
+}
+
+//----------------------------------------------------------------------------
+bool vtkMPIMultiBlockPLOT3DReader::CanUseMPIIO()
+{
+  return (this->UseMPIIO && this->BinaryFile &&
+    this->Internal->Settings.NumberOfDimensions == 3 &&
+    vtkMPIController::SafeDownCast(this->Controller) != NULL);
+}
+
+//----------------------------------------------------------------------------
+int vtkMPIMultiBlockPLOT3DReader::OpenFileForDataRead(void*& vfp, const char* fname)
+{
+  if (!this->CanUseMPIIO())
+    {
+    return this->Superclass::OpenFileForDataRead(vfp, fname);
+    }
+
+  vtkMPICommunicator* mpiComm = vtkMPICommunicator::SafeDownCast(
+    this->Controller->GetCommunicator());
+  assert(mpiComm);
+
+  vtkMPIOpaqueFileHandle* handle = new vtkMPIOpaqueFileHandle();
+  try
+    {
+    if (MPI_File_open(*mpiComm->GetMPIComm()->GetHandle(),
+                      const_cast<char*>(fname), MPI_MODE_RDONLY,
+                      MPI_INFO_NULL, &handle->Handle) != MPI_SUCCESS)
+      {
+      this->SetErrorCode(vtkErrorCode::FileNotFoundError);
+      vtkErrorMacro("File: " << fname << " not found.");
+      throw MPIPlot3DException();
+      }
+    }
+  catch (MPIPlot3DException)
+    {
+    delete handle;
+    vfp = NULL;
+    return VTK_ERROR;
+    }
+  vfp = handle;
+  return VTK_OK;
+}
+
+//----------------------------------------------------------------------------
+void vtkMPIMultiBlockPLOT3DReader::CloseFile(void* vfp)
+{
+  if (!this->CanUseMPIIO())
+    {
+    this->Superclass::CloseFile(vfp);
+    return;
+    }
+
+  vtkMPIOpaqueFileHandle* handle = reinterpret_cast<vtkMPIOpaqueFileHandle*>(vfp);
+  assert(handle);
+  if (MPI_File_close(&handle->Handle) != MPI_SUCCESS)
+    {
+    vtkErrorMacro("Failed to close file!");
+    }
+}
+
+//----------------------------------------------------------------------------
+int vtkMPIMultiBlockPLOT3DReader::ReadIntScalar(
+  void* vfp, int extent[6], int wextent[6],
+  vtkDataArray* scalar, vtkTypeUInt64 offset)
+{
+  if (!this->CanUseMPIIO())
+    {
+    return this->Superclass::ReadScalar(vfp, extent, wextent, scalar, offset);
+    }
+
+  return this->ReadScalar(vfp, extent, wextent, scalar, offset);
+}
+
+//----------------------------------------------------------------------------
+int vtkMPIMultiBlockPLOT3DReader::ReadScalar(
+    void* vfp,
+    int extent[6], int wextent[6],
+    vtkDataArray* scalar, vtkTypeUInt64 offset)
+{
+  if (!this->CanUseMPIIO())
+    {
+    return this->Superclass::ReadScalar(vfp, extent, wextent, scalar, offset);
+    }
+
+  vtkMPIOpaqueFileHandle* handle = reinterpret_cast<vtkMPIOpaqueFileHandle*>(vfp);
+  assert(handle);
+
+  vtkMPICommunicatorOpaqueComm* comm =
+    vtkMPICommunicator::SafeDownCast(this->Controller->GetCommunicator())->GetMPIComm();
+  assert(comm);
+
+  SetupFileViewFunctor work(*comm, *handle, extent, wextent, offset, this->Internal->Settings.ByteOrder);
+  vtkDataArrayDispatcher<SetupFileViewFunctor> dispatcher(work);
+  try
+    {
+    dispatcher.Go(scalar);
+    }
+  catch (MPIPlot3DException)
+    {
+    return 0;
+    }
+  return 1;
+}
+
+//----------------------------------------------------------------------------
+int vtkMPIMultiBlockPLOT3DReader::ReadVector(
+  void* vfp,
+  int extent[6], int wextent[6],
+  int numDims, vtkDataArray* vector, vtkTypeUInt64 offset)
+{
+  if (!this->CanUseMPIIO())
+    {
+    return this->Superclass::ReadVector(vfp, extent, wextent, numDims, vector, offset);
+    }
+
+  vtkMPIOpaqueFileHandle* handle = reinterpret_cast<vtkMPIOpaqueFileHandle*>(vfp);
+  assert(handle);
+
+  vtkMPICommunicatorOpaqueComm* comm =
+    vtkMPICommunicator::SafeDownCast(this->Controller->GetCommunicator())->GetMPIComm();
+  assert(comm);
+
+  SetupFileViewFunctor work(*comm, *handle, extent, wextent, offset, this->Internal->Settings.ByteOrder);
+  vtkDataArrayDispatcher<SetupFileViewFunctor> dispatcher(work);
+  try
+    {
+    dispatcher.Go(vector);
+    }
+  catch (MPIPlot3DException)
+    {
+    return 0;
+    }
+  return 1;
+}
+
+//----------------------------------------------------------------------------
+void vtkMPIMultiBlockPLOT3DReader::PrintSelf(ostream& os, vtkIndent indent)
+{
+  this->Superclass::PrintSelf(os, indent);
+  os << indent << "UseMPIIO: " << this->UseMPIIO << endl;
+}

IO/MPIParallel/vtkMPIMultiBlockPLOT3DReader.h

+/*=========================================================================
+
+  Program:   Visualization Toolkit
+  Module:    vtkMPIMultiBlockPLOT3DReader.h
+
+  Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
+  All rights reserved.
+  See Copyright.txt or http://www.kitware.com/Copyright.htm for details.
+
+     This software is distributed WITHOUT ANY WARRANTY; without even
+     the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+     PURPOSE.  See the above copyright notice for more information.
+
+=========================================================================*/
+// .NAME vtkMPIMultiBlockPLOT3DReader - vtkMultiBlockPLOT3DReader subclass that
+// uses MPI-IO to efficiently read binary files for 3D domains in parallel using
+// MPI-IO.
+// .SECTION Description
+// vtkMPIMultiBlockPLOT3DReader extends vtkMultiBlockPLOT3DReader to use MPI-IO
+// instead of POSIX IO to read file in parallel.
+
+#ifndef vtkMPIMultiBlockPLOT3DReader_h
+#define vtkMPIMultiBlockPLOT3DReader_h
+
+#include "vtkMultiBlockPLOT3DReader.h"
+#include "vtkIOMPIParallelModule.h" // For export macro
+
+class VTKIOMPIPARALLEL_EXPORT vtkMPIMultiBlockPLOT3DReader : public vtkMultiBlockPLOT3DReader
+{
+public:
+  static vtkMPIMultiBlockPLOT3DReader* New();
+  vtkTypeMacro(vtkMPIMultiBlockPLOT3DReader, vtkMultiBlockPLOT3DReader);
+  void PrintSelf(ostream& os, vtkIndent indent);
+
+  // Description:
+  // Use this to override using MPI-IO. When set to false (default is true),
+  // this class will simply forward all method calls to the superclass.
+  vtkSetMacro(UseMPIIO, bool);
+  vtkGetMacro(UseMPIIO, bool);
+  vtkBooleanMacro(UseMPIIO, bool);
+
+protected:
+  vtkMPIMultiBlockPLOT3DReader();
+  ~vtkMPIMultiBlockPLOT3DReader();
+
+  // Description:
+  // Determines we should use MPI-IO for the current file. We don't use MPI-IO
+  // for 2D files or ASCII files.
+  bool CanUseMPIIO();
+
+  virtual int OpenFileForDataRead(void*& fp, const char* fname);
+  virtual void CloseFile(void* fp);
+
+  virtual int ReadIntScalar(
+    void* vfp,
+    int extent[6], int wextent[6],
+    vtkDataArray* scalar, vtkTypeUInt64 offset);
+  virtual int ReadScalar(
+    void* vfp,
+    int extent[6], int wextent[6],
+    vtkDataArray* scalar, vtkTypeUInt64 offset);
+  virtual int ReadVector(
+    void* vfp,
+    int extent[6], int wextent[6],
+    int numDims, vtkDataArray* vector, vtkTypeUInt64 offset);
+
+  bool UseMPIIO;
+private:
+  vtkMPIMultiBlockPLOT3DReader(const vtkMPIMultiBlockPLOT3DReader&); // Not implemented.
+  void operator=(const vtkMPIMultiBlockPLOT3DReader&); // Not implemented.
+};
+
+#endif