Commit d9c4d3d8 authored by Kenneth Moreland's avatar Kenneth Moreland

ENH: Added an image reader that uses MPI I/O to efficiently read from parallel file systems.

parent 3c906302
......@@ -24,7 +24,7 @@
#include "vtkStreamingDemandDrivenPipeline.h"
#include "vtkTransform.h"
vtkCxxRevisionMacro(vtkImageReader, "1.122");
vtkCxxRevisionMacro(vtkImageReader, "1.123");
vtkStandardNewMacro(vtkImageReader);
vtkCxxSetObjectMacro(vtkImageReader,Transform,vtkTransform);
......@@ -46,9 +46,8 @@ vtkImageReader::vtkImageReader()
{
this->DataVOI[idx*2] = this->DataVOI[idx*2 + 1] = 0;
}
// Left over from short reader
this->DataMask = 0xffff;
this->DataMask = static_cast<vtkTypeUInt64>(~0UL);
this->Transform = NULL;
this->ScalarArrayName = NULL;
......@@ -211,7 +210,7 @@ void vtkImageReaderUpdate2(vtkImageReader *self, vtkImageData *data,
int comp, pixelSkip;
long filePos, correction = 0;
unsigned long count = 0;
unsigned short DataMask;
vtkTypeUInt64 DataMask;
unsigned long target;
// Get the requested extents.
......@@ -327,7 +326,7 @@ void vtkImageReaderUpdate2(vtkImageReader *self, vtkImageData *data,
for (idx0 = dataExtent[0]; idx0 <= dataExtent[1]; ++idx0)
{
// Copy pixel into the output.
if (DataMask == 0xffff)
if (DataMask == static_cast<vtkTypeUInt64>(~0UL))
{
for (comp = 0; comp < pixelSkip; comp++)
{
......@@ -336,10 +335,9 @@ void vtkImageReaderUpdate2(vtkImageReader *self, vtkImageData *data,
}
else
{
// left over from short reader (what about other types.
for (comp = 0; comp < pixelSkip; comp++)
{
outPtr0[comp] = (OT)((short)(inPtr[comp]) & DataMask);
outPtr0[comp] = (OT)((vtkTypeUInt64)(inPtr[comp]) & DataMask);
}
}
// move to next pixel
......
......@@ -45,17 +45,13 @@ public:
vtkGetVector6Macro(DataVOI,int);
// Description:
// Set/Get the Data mask.
vtkGetMacro(DataMask,unsigned short);
void SetDataMask(int val)
{
if (val == this->DataMask)
{
return;
}
this->DataMask = static_cast<unsigned short>(val);
this->Modified();
}
// Set/Get the Data mask. The data mask is a simply integer whose bits are
// treated as a mask to the bits read from disk. That is, the data mask is
// bitwise-and'ed to the numbers read from disk. This ivar is stored as 64
// bits, the largest mask you will need. The mask will be truncated to the
// data size required to be read (using the least significant bits).
vtkGetMacro(DataMask, vtkTypeUInt64);
vtkSetMacro(DataMask, vtkTypeUInt64);
// Description:
// Set/Get transformation matrix to transform the data from slice space
......@@ -82,7 +78,7 @@ protected:
vtkImageReader();
~vtkImageReader();
unsigned short DataMask; // Mask each pixel with ...
vtkTypeUInt64 DataMask;
vtkTransform *Transform;
......
......@@ -45,6 +45,7 @@ vtkExtractPiece.cxx
vtkExtractPolyDataPiece.cxx
vtkExtractUnstructuredGridPiece.cxx
vtkExtractUserDefinedPiece.cxx
vtkMPIImageReader.cxx
vtkMultiProcessController.cxx
vtkMultiProcessStream.cxx
vtkParallelFactory.cxx
......@@ -58,6 +59,7 @@ vtkPieceRequestFilter.cxx
vtkPieceScalars.cxx
vtkPKdTree.cxx
vtkPLinearExtrusionFilter.cxx
vtkPNrrdReader.cxx
vtkPOPReader.cxx
${_VTK_OPENFOAM_SOURCES}
vtkPOutlineCornerFilter.cxx
......
......@@ -349,6 +349,16 @@ IF(VTK_USE_DISPLAY AND VTK_USE_RENDERING)
${VTK_MPI_POSTFLAGS})
ENDIF(VTK_MPI_MAX_NUMPROCS GREATER 1)
IF(VTK_MPI_MAX_NUMPROCS GREATER 6)
ADD_TEST(ParallelIso-7proc-image
${VTK_MPIRUN_EXE} ${VTK_MPI_PRENUMPROC_FLAGS} ${VTK_MPI_NUMPROC_FLAG} 7 ${VTK_MPI_PREFLAGS}
${CXX_TEST_PATH}/ParallelIsoTest
-D ${VTK_DATA_ROOT}
-T ${VTK_BINARY_DIR}/Testing/Temporary
-V Baseline/Parallel/ParallelIso.7proc.png
${VTK_MPI_POSTFLAGS})
ENDIF(VTK_MPI_MAX_NUMPROCS GREATER 6)
ENDIF (VTK_MPIRUN_EXE)
#
# If we do not have the data, still run the tests that we can
......
......@@ -1052,6 +1052,22 @@ int ExerciseMultiProcessController(vtkMultiProcessController *controller)
args.retval = 1;
}
int color = (group1->GetLocalProcessId() >= 0) ? 1 : 2;
vtkMultiProcessController *subcontroller
= controller->PartitionController(color, 0);
subcontroller->SetSingleMethod(Run, &args);
subcontroller->SingleMethodExecute();
subcontroller->Delete();
try
{
CheckSuccess(controller, !args.retval);
}
catch (ExerciseMultiProcessControllerError)
{
args.retval = 1;
}
return args.retval;
}
......@@ -13,7 +13,7 @@
=========================================================================*/
// This example demonstrates the use of data parallelism in VTK. The
// pipeline ( vtkImageReader -> vtkContourFilter -> vtkElevationFilter )
// pipeline ( vtkMPIImageReader -> vtkContourFilter -> vtkElevationFilter )
// is created in parallel and each process is assigned 1 piece to process.
// All satellite processes send the result to the first process which
// collects and renders them.
......@@ -27,10 +27,10 @@
#include "vtkContourFilter.h"
#include "vtkDataSet.h"
#include "vtkElevationFilter.h"
#include "vtkImageReader.h"
#include "vtkMath.h"
#include "vtkMPIController.h"
#include "vtkParallelFactory.h"
#include "vtkPNrrdReader.h"
#include "vtkPolyData.h"
#include "vtkPolyDataMapper.h"
#include "vtkTestUtilities.h"
......@@ -87,7 +87,7 @@ void SetIsoValueRMI(void *localArg, void* vtkNotUsed(remoteArg),
// This will be called by all processes
void MyMain( vtkMultiProcessController *controller, void *arg )
{
vtkImageReader *reader;
vtkPNrrdReader *reader;
vtkContourFilter *iso;
vtkElevationFilter *elev;
int myid, numProcs;
......@@ -102,12 +102,13 @@ void MyMain( vtkMultiProcessController *controller, void *arg )
// Create the reader, the data file name might have
// to be changed depending on where the data files are.
char* fname = vtkTestUtilities::ExpandDataFileName(args->argc, args->argv,
"Data/headsq/quarter");
reader = vtkImageReader::New();
reader->SetDataByteOrderToLittleEndian();
reader->SetDataExtent(0, 63, 0, 63, 1, 93);
reader->SetFilePrefix(fname);
reader->SetDataSpacing(3.2, 3.2, 1.5);
"Data/headsq/quarter.nhdr");
reader = vtkPNrrdReader::New();
reader->SetFileName(fname);
// reader->SetDataByteOrderToLittleEndian();
// reader->SetDataExtent(0, 63, 0, 63, 1, 93);
// reader->SetFilePrefix(fname);
// reader->SetDataSpacing(3.2, 3.2, 1.5);
delete[] fname;
// Iso-surface.
......@@ -129,6 +130,9 @@ void MyMain( vtkMultiProcessController *controller, void *arg )
exec->SetUpdateNumberOfPieces(exec->GetOutputInformation(0), numProcs);
exec->SetUpdatePiece(exec->GetOutputInformation(0), myid);
// Make sure all processes update at the same time.
elev->Update();
if (myid != 0)
{
// If I am not the root process
......
......@@ -52,6 +52,13 @@ public:
MPI_Comm* Handle;
};
class VTK_PARALLEL_EXPORT vtkMPIOpaqueFileHandle
{
public:
vtkMPIOpaqueFileHandle() : Handle(MPI_FILE_NULL) { }
MPI_File Handle;
};
//-----------------------------------------------------------------------------
class vtkMPICommunicatorOpaqueRequest
{
......
......@@ -30,7 +30,7 @@
#include <vtkstd/vector>
vtkCxxRevisionMacro(vtkMPICommunicator, "1.55");
vtkCxxRevisionMacro(vtkMPICommunicator, "1.56");
vtkStandardNewMacro(vtkMPICommunicator);
vtkMPICommunicator* vtkMPICommunicator::WorldCommunicator = 0;
......@@ -552,6 +552,53 @@ int vtkMPICommunicator::Initialize(vtkProcessGroup *group)
return 1;
}
//-----------------------------------------------------------------------------
int vtkMPICommunicator::SplitInitialize(vtkCommunicator *oldcomm,
int color, int key)
{
if (this->Initialized) return 0;
vtkMPICommunicator *mpiComm = vtkMPICommunicator::SafeDownCast(oldcomm);
if (!mpiComm)
{
vtkErrorMacro("Split communicator must be an MPI communicator.");
return 0;
}
// If mpiComm has been initialized, it is guaranteed (unless the MPI calls
// return an error somewhere) to have valid Communicator.
if (!mpiComm->Initialized)
{
vtkWarningMacro("The communicator passed has not been initialized!");
return 0;
}
this->KeepHandleOff();
this->MPIComm->Handle = new MPI_Comm;
int err;
if ( (err = MPI_Comm_split(*(mpiComm->MPIComm->Handle), color, key,
this->MPIComm->Handle))
!= MPI_SUCCESS )
{
delete this->MPIComm->Handle;
this->MPIComm->Handle = 0;
char *msg = vtkMPIController::ErrorString(err);
vtkErrorMacro("MPI error occured: " << msg);
delete[] msg;
return 0;
}
this->InitializeNumberOfProcesses();
this->Initialized = 1;
this->Modified();
return 1;
}
//----------------------------------------------------------------------------
// Start the copying process
void vtkMPICommunicator::InitializeCopy(vtkMPICommunicator* source)
......
......@@ -85,6 +85,11 @@ public:
// DO NOT CALL. Deprecated in VTK 5.2.
VTK_LEGACY(int Initialize(vtkMPICommunicator* mpiComm, vtkMPIGroup* group));
// Description:
// Used to initialize the communicator (i.e. create the underlying MPI_Comm)
// using MPI_Comm_split on the given communicator.
int SplitInitialize(vtkCommunicator *oldcomm, int color, int key);
// Description:
// Performs the actual communication. You will usually use the convenience
// Send functions defined in the superclass.
......
......@@ -70,9 +70,9 @@ void vtkMPIController::CreateOutputWindow()
vtkOutputWindow::SetInstance(this->OutputWindow);
}
vtkCxxRevisionMacro(vtkMPIOutputWindow, "1.28");
vtkCxxRevisionMacro(vtkMPIOutputWindow, "1.29");
vtkCxxRevisionMacro(vtkMPIController, "1.28");
vtkCxxRevisionMacro(vtkMPIController, "1.29");
vtkStandardNewMacro(vtkMPIController);
//----------------------------------------------------------------------------
......@@ -324,3 +324,19 @@ vtkMPIController *vtkMPIController::CreateSubController(vtkProcessGroup *group)
controller->SetCommunicator(subcomm);
return controller;
}
//-----------------------------------------------------------------------------
vtkMPIController *vtkMPIController::PartitionController(int localColor,
int localKey)
{
VTK_CREATE(vtkMPICommunicator, subcomm);
if (!subcomm->SplitInitialize(this->Communicator, localColor, localKey))
{
return NULL;
}
vtkMPIController *controller = vtkMPIController::New();
controller->SetCommunicator(subcomm);
return controller;
}
......@@ -114,6 +114,8 @@ public:
virtual vtkMPIController *CreateSubController(vtkProcessGroup *group);
virtual vtkMPIController *PartitionController(int localColor, int localKey);
//BTX
// Description:
......
This diff is collapsed.
// -*- c++ -*-
/*=========================================================================
Program: Visualization Toolkit
Module: vtkMPIImageReader.h
Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
All rights reserved.
See Copyright.txt or http://www.kitware.com/Copyright.htm for details.
This software is distributed WITHOUT ANY WARRANTY; without even
the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE. See the above copyright notice for more information.
=========================================================================*/
/*----------------------------------------------------------------------------
Copyright (c) Sandia Corporation
See Copyright.txt or http://www.paraview.org/HTML/Copyright.html for details.
----------------------------------------------------------------------------*/
// .NAME vtkMPIImageReader Superclass of parallel binary image file readers.
//
// .SECTION Description
//
// vtkMPIImageReader provides the mechanism to read a brick of bytes (or shorts,
// or ints, or floats, or doubles, ...) from a file or series of files. You can
// use it to read raw image data from files. You may also be able to subclass
// this to read simple file formats.
//
// What distinguishes this class from vtkImageReader and vtkImageReader2 is that
// it performs synchronized parallel I/O using the MPIIO layer. This can make a
// huge difference in file read times, especially when reading in parallel from
// a parallel file system.
//
// Dispite the name of this class, vtkMPIImageReader will work even if MPI is
// not available. If MPI is not available or MPIIO is not available or the
// given Controller is not a vtkMPIController (or NULL), then this class will
// silently work exactly like its superclass. The point is that you can safely
// use this class in applications that may or may not be compiled with MPI (or
// may or may not actually be run with MPI).
//
// .SECTION See Also
// vtkMultiProcessController, vtkImageReader, vtkImageReader2
//
#ifndef __vtkMPIImageReader_h
#define __vtkMPIImageReader_h
#include "vtkImageReader.h"
class vtkMPIOpaqueFileHandle;
class vtkMultiProcessController;
class VTK_PARALLEL_EXPORT vtkMPIImageReader : public vtkImageReader
{
public:
vtkTypeRevisionMacro(vtkMPIImageReader, vtkImageReader);
static vtkMPIImageReader *New();
virtual void PrintSelf(ostream &os, vtkIndent indent);
// Description:
// Get/set the multi process controller to use for coordinated reads. By
// default, set to the global controller.
vtkGetObjectMacro(Controller, vtkMultiProcessController);
virtual void SetController(vtkMultiProcessController *);
protected:
vtkMPIImageReader();
~vtkMPIImageReader();
vtkMultiProcessController *Controller;
// Description:
// Returns the size, in bytes of the scalar data type (GetDataScalarType).
int GetDataScalarTypeSize();
// Description:
// Break up the controller based on the files each process reads. Each group
// comprises the processes that read the same files in the same order.
// this->GroupedController is set to the group for the current process.
virtual void PartitionController(const int extent[6]);
// Description:
// Get the header size of the given open file. This should be used in liu of
// the GetHeaderSize methods of the superclass.
virtual unsigned long GetHeaderSize(vtkMPIOpaqueFileHandle &file);
// Description:
// Set up a "view" on the open file that will allow you to read the 2D or 3D
// subarray from the file in one read. Once you call this method, the file
// will look as if it contains only the data the local process needs to read
// in.
virtual void SetupFileView(vtkMPIOpaqueFileHandle &file, const int extent[6]);
// Description:
// Given a slice of the data, open the appropriate file, read the data into
// given buffer, and close the file. For three dimensional data, always
// use "slice" 0. Make sure the GroupedController is properly created before
// calling this using the PartitionController method.
virtual void ReadSlice(int slice, const int extent[6], void *buffer);
// Description:
// Transform the data from the order read from a file to the order to place
// in the output data (as defined by the transform).
virtual void TransformData(vtkImageData *data);
// Description:
// A group of processes that are reading the same file (as determined by
// PartitionController.
void SetGroupedController(vtkMultiProcessController *);
vtkMultiProcessController *GroupedController;
virtual void ExecuteData(vtkDataObject *data);
private:
vtkMPIImageReader(const vtkMPIImageReader &); // Not implemented
void operator=(const vtkMPIImageReader &); // Not implemented
};
#endif //__vtkMPIImageReader_h
......@@ -29,6 +29,12 @@
#include "vtkMPIController.h"
#endif
#include "vtkSmartPointer.h"
#define VTK_CREATE(type, name) \
vtkSmartPointer<type> name = vtkSmartPointer<type>::New()
#include <vtkstd/list>
#include <vtkstd/vector>
#include <vtksys/hash_map.hxx>
//----------------------------------------------------------------------------
......@@ -66,10 +72,10 @@ protected:
void operator=(const vtkMultiProcessControllerRMI&);
};
vtkCxxRevisionMacro(vtkMultiProcessControllerRMI, "1.38");
vtkCxxRevisionMacro(vtkMultiProcessControllerRMI, "1.39");
vtkStandardNewMacro(vtkMultiProcessControllerRMI);
vtkCxxRevisionMacro(vtkMultiProcessController, "1.38");
vtkCxxRevisionMacro(vtkMultiProcessController, "1.39");
//----------------------------------------------------------------------------
// An RMI function that will break the "ProcessRMIs" loop.
......@@ -318,6 +324,60 @@ vtkMultiProcessController *vtkMultiProcessController::CreateSubController(
return subcontroller;
}
//-----------------------------------------------------------------------------
vtkMultiProcessController *vtkMultiProcessController::PartitionController(
int localColor,
int localKey)
{
vtkMultiProcessController *subController = NULL;
int numProc = this->GetNumberOfProcesses();
vtkstd::vector<int> allColors(numProc);
this->AllGather(&localColor, &allColors[0], 1);
vtkstd::vector<int> allKeys(numProc);
this->AllGather(&localKey, &allKeys[0], 1);
vtkstd::vector<bool> inPartition;
inPartition.assign(numProc, false);
for (int i = 0; i < numProc; i++)
{
if (inPartition[i]) continue;
int targetColor = allColors[i];
vtkstd::list<int> partitionIds; // Make sorted list, then put in group.
for (int j = i; j < numProc; j++)
{
if (allColors[j] != targetColor) continue;
inPartition[j] = true;
vtkstd::list<int>::iterator iter = partitionIds.begin();
while ((iter != partitionIds.end()) && (allKeys[*iter] <= allKeys[j]))
{
iter++;
}
partitionIds.insert(iter, j);
}
// Copy list into process group.
VTK_CREATE(vtkProcessGroup, group);
group->Initialize(this);
group->RemoveAllProcessIds();
for (vtkstd::list<int>::iterator iter = partitionIds.begin();
iter != partitionIds.end(); iter++)
{
group->AddProcessId(*iter);
}
// Use group to create controller.
vtkMultiProcessController *sc = this->CreateSubController(group);
if (sc)
{
subController = sc;
}
}
return subController;
}
//----------------------------------------------------------------------------
int vtkMultiProcessController::RemoveFirstRMI(int tag)
{
......
......@@ -161,6 +161,18 @@ public:
// returned on all process not in the group.
virtual vtkMultiProcessController *CreateSubController(
vtkProcessGroup *group);
// Description:
// Partitions this controller based on a coloring. That is, each process
// passes in a color. All processes with the same color are grouped into the
// same partition. The processes are ordered by their self-assigned key.
// Lower keys have lower process ids. Ties are broken by the current process
// ids. (For example, if all the keys are 0, then the resulting processes
// will be ordered in the same way.) This method returns a new controller to
// each process that represents the local partition. This is basically the
// same operation as MPI_Comm_split.
virtual vtkMultiProcessController *PartitionController(int localColor,
int localKey);
//------------------ RMIs --------------------
//BTX
......
This diff is collapsed.
// -*- c++ -*-
/*=========================================================================
Program: Visualization Toolkit
Module: vtkPNrrdReader.h
Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
All rights reserved.
See Copyright.txt or http://www.kitware.com/Copyright.htm for details.
This software is distributed WITHOUT ANY WARRANTY; without even
the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE. See the above copyright notice for more information.
=========================================================================*/
/*----------------------------------------------------------------------------
Copyright (c) Sandia Corporation
See Copyright.txt or http://www.paraview.org/HTML/Copyright.html for details.
----------------------------------------------------------------------------*/
// .NAME vtkPNrrdReader - Read nrrd files efficiently from parallel file systems (and reasonably well elsewhere).
//
// .SECTION Description
//
// vtkPNrrdReader is a subclass of vtkMPIImageReader that will read Nrrd format
// header information of the image before reading the data. This means that the
// reader will automatically set information like file dimensions.
//
// .SECTION Bugs
//
// There are several limitations on what type of nrrd files we can read. This
// reader only supports nrrd files in raw format. Other encodings like ascii
// and hex will result in errors. When reading in detached headers, this only
// supports reading one file that is detached.
//
#ifndef __vtkPNrrdReader_h
#define __vtkPNrrdReader_h
#include "vtkMPIImageReader.h"
class vtkCharArray;
class VTK_PARALLEL_EXPORT vtkPNrrdReader : public vtkMPIImageReader
{
public:
vtkTypeRevisionMacro(vtkPNrrdReader, vtkMPIImageReader);
static vtkPNrrdReader *New();
virtual void PrintSelf(ostream &os, vtkIndent indent);
virtual int CanReadFile(const char *filename);
protected:
vtkPNrrdReader();
~vtkPNrrdReader();
virtual int RequestInformation(vtkInformation *request,
vtkInformationVector **inputVector,
vtkInformationVector *outputVector);
virtual int RequestData(vtkInformation *request,
vtkInformationVector **inputVector,
vtkInformationVector *outputVector);
virtual int ReadHeader();
virtual int ReadHeader(vtkCharArray *headerBuffer);
vtkStringArray *DataFiles;
private:
vtkPNrrdReader(const vtkPNrrdReader &); // Not implemented.
void operator=(const vtkPNrrdReader &); // Not implemented.
};
#endif //__vtkPNrrdReader_h
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment