Skip to content

GitLab

Commit 521435d9 authored by Kenneth Leiter's avatar Kenneth Leiter
Browse files
Options

ENH: Add ability to write in parallel to hdf5 file. 

Modified writer to utilize hyperslabs to write in parallel to file.
Updated DSM to newest version (0.9.9).
Modified DSM test to write in parallel to DSM buffer.
Add test for new hyperslab writing mode.
parent 9c1652ad
Hide whitespace changes
Inline Side-by-side
Showing with 376 additions and 145 deletions
core/CMakeLists.txt
View file @ 521435d9
......@@ -93,13 +93,13 @@ if(XDMF_BUILD_DSM)
find_package(H5FDdsm REQUIRED NO_MODULE)
if(H5FDdsm_FOUND)
include_directories(${H5FD_DSM_INCLUDE_DIR})
get_filename_component(H5FD_DSM_LIBRARY_DIR
"${H5FD_DSM_INCLUDE_DIR}/../lib" REALPATH)
set(XDMF_LIBRARY_DIRS ${XDMF_LIBRARY_DIRS} ${H5FD_DSM_LIBRARY_DIR})
include_directories(${H5FDdsm_INCLUDE_DIR})
get_filename_component(H5FDdsm_LIBRARY_DIR
"${H5FDdsm_INCLUDE_DIR}/../lib" REALPATH)
set(XDMF_LIBRARY_DIRS ${XDMF_LIBRARY_DIRS} ${H5FDdsm_LIBRARY_DIR})
get_property(h5fddsm_lib_location TARGET H5FDdsm PROPERTY LOCATION)
set(H5FD_DSM_LIBRARIES ${h5fddsm_lib_location})
set(XDMF_LIBRARIES ${XDMF_LIBRARIES} ${H5FD_DSM_LIBRARIES})
set(H5FDdsm_LIBRARIES ${h5fddsm_lib_location})
set(XDMF_LIBRARIES ${XDMF_LIBRARIES} ${H5FDdsm_LIBRARIES})
else(H5FDdsm_FOUND)
message(SEND_ERROR
"Cannot find HDF5 DSM! Please set H5FDdsm_DIR and configure again.")
......
core/XdmfArray.cpp
View file @ 521435d9
......@@ -824,6 +824,7 @@ XdmfArray::populateItem(const std::map<std::string, std::string> & itemPropertie
0),
std::vector<unsigned int>(mDimensions.size(),
1),
mDimensions,
mDimensions);
}
else if(formatVal.compare("XML") == 0) {
......
core/XdmfHDF5Controller.cpp
View file @ 521435d9
......@@ -25,9 +25,9 @@
#include <sstream>
#include "XdmfArray.hpp"
#include "XdmfArrayType.hpp"
#include "XdmfError.hpp"
#include "XdmfHDF5Controller.hpp"
#include "XdmfSystemUtils.hpp"
#include "XdmfError.hpp"
shared_ptr<XdmfHDF5Controller>
XdmfHDF5Controller::New(const std::string & hdf5FilePath,
......@@ -35,14 +35,17 @@ XdmfHDF5Controller::New(const std::string & hdf5FilePath,
const shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count)
const std::vector<unsigned int> & dimensions,
const std::vector<unsigned int> & dataspaceDimensions)
{
shared_ptr<XdmfHDF5Controller> p(new XdmfHDF5Controller(hdf5FilePath,
dataSetPath,
type,
start,
stride,
count));
shared_ptr<XdmfHDF5Controller>
p(new XdmfHDF5Controller(hdf5FilePath,
dataSetPath,
type,
start,
stride,
dimensions,
dataspaceDimensions));
return p;
}
......@@ -51,18 +54,16 @@ XdmfHDF5Controller::XdmfHDF5Controller(const std::string & hdf5FilePath,
const shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count) :
const std::vector<unsigned int> & dimensions,
const std::vector<unsigned int> & dataspaceDimensions) :
XdmfHeavyDataController(hdf5FilePath,
dataSetPath,
type,
count),
mStart(start),
mStride(stride)
start,
stride,
dimensions,
dataspaceDimensions)
{
if(!(mStart.size() == mStride.size() && mStride.size() == mDimensions.size()))
XdmfError::message(XdmfError::FATAL,
"mStart, mStride, mDimensions must all be of equal "
"length in XdmfHDF5Controller constructor");
}
XdmfHDF5Controller::~XdmfHDF5Controller()
......
core/XdmfHDF5Controller.hpp
View file @ 521435d9
......@@ -53,8 +53,11 @@ public:
* the hdf5 data set.
* @param stride the number of elements to move in each dimension from the
* hdf5 data set.
* @param count the number of elements to select in each dimension from the
* hdf5 data set. (size in each dimension)
* @param dimensions the number of elements to select in each
* dimension from the hdf5 data set. (size in each dimension)
* @param dataspaceDimensions the number of elements in the entire
* hdf5 data set (may be larger that dimensions if using
* hyperslabs).
*
* @return new HDF5 Controller.
*/
......@@ -64,7 +67,8 @@ public:
const shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count);
const std::vector<unsigned int> & dimensions,
const std::vector<unsigned int> & dataspaceDimensions);
virtual std::string getName() const;
......@@ -77,7 +81,8 @@ protected:
const shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count);
const std::vector<unsigned int> & dimensions,
const std::vector<unsigned int> & dataspaceDimensions);
void read(XdmfArray * const array, const int fapl);
......@@ -86,9 +91,6 @@ private:
XdmfHDF5Controller(const XdmfHDF5Controller &); // Not implemented.
void operator=(const XdmfHDF5Controller &); // Not implemented.
const std::vector<unsigned int> mStart;
const std::vector<unsigned int> mStride;
};
#endif /* XDMFHDF5CONTROLLER_HPP_ */
core/XdmfHDF5ControllerDSM.cpp
View file @ 521435d9
......@@ -31,16 +31,19 @@ XdmfHDF5ControllerDSM::New(const std::string & hdf5FilePath,
const shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count,
const std::vector<unsigned int> & dimensions,
const std::vector<unsigned int> & datspaceDimensions,
H5FDdsmBuffer * const dsmBuffer)
{
shared_ptr<XdmfHDF5ControllerDSM> p(new XdmfHDF5ControllerDSM(hdf5FilePath,
dataSetPath,
type,
start,
stride,
count,
dsmBuffer));
shared_ptr<XdmfHDF5ControllerDSM>
p(new XdmfHDF5ControllerDSM(hdf5FilePath,
dataSetPath,
type,
start,
stride,
dimensions,
datspaceDimensions,
dsmBuffer));
return p;
}
......@@ -49,9 +52,16 @@ XdmfHDF5ControllerDSM::XdmfHDF5ControllerDSM(const std::string & hdf5FilePath,
const shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count,
const std::vector<unsigned int> & dimensions,
const std::vector<unsigned int> & dataspaceDimensions,
H5FDdsmBuffer * const dsmBuffer) :
XdmfHDF5Controller(hdf5FilePath, dataSetPath, type, start, stride, count),
XdmfHDF5Controller(hdf5FilePath,
dataSetPath,
type,
start,
stride,
dimensions,
dataspaceDimensions),
mDSMBuffer(dsmBuffer)
{
}
......@@ -71,7 +81,7 @@ void XdmfHDF5ControllerDSM::read(XdmfArray * const array)
hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
// Use DSM driver
H5Pset_fapl_dsm(fapl, MPI_COMM_WORLD, mDSMBuffer);
H5Pset_fapl_dsm(fapl, MPI_COMM_WORLD, mDSMBuffer, 0);
// Read from DSM Buffer
XdmfHDF5Controller::read(array, fapl);
......
core/XdmfHDF5ControllerDSM.hpp
View file @ 521435d9
......@@ -55,7 +55,8 @@ public:
const shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count,
const std::vector<unsigned int> & dimensions,
const std::vector<unsigned int> & dataspaceDimensions,
H5FDdsmBuffer * const dsmBuffer);
std::string getName() const;
......@@ -69,7 +70,8 @@ protected:
const shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count,
const std::vector<unsigned int> & dimensions,
const std::vector<unsigned int> & dataspaceDimensions,
H5FDdsmBuffer * const dsmBuffer);
private:
......
core/XdmfHDF5Writer.cpp
View file @ 521435d9
......@@ -60,7 +60,6 @@ public:
openFile(const std::string & filePath,
const int fapl)
{
if(mHDF5Handle >= 0) {
// Perhaps we should throw a warning.
closeFile();
......@@ -129,14 +128,16 @@ XdmfHDF5Writer::createHDF5Controller(const std::string & hdf5FilePath,
const shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count)
const std::vector<unsigned int> & dimensions,
const std::vector<unsigned int> & dataspaceDimensions)
{
return XdmfHDF5Controller::New(hdf5FilePath,
dataSetPath,
type,
start,
stride,
count);
dimensions,
dataspaceDimensions);
}
void
......@@ -169,7 +170,6 @@ void
XdmfHDF5Writer::write(XdmfArray & array,
const int fapl)
{
hid_t datatype = -1;
if(array.isInitialized()) {
......@@ -212,18 +212,29 @@ XdmfHDF5Writer::write(XdmfArray & array,
std::stringstream dataSetPath;
if((mMode == Overwrite || mMode == Append)
&& array.getHeavyDataController()) {
shared_ptr<XdmfHeavyDataController> heavyDataController =
array.getHeavyDataController();
const std::vector<unsigned int> & dimensions = array.getDimensions();
std::vector<unsigned int> dataspaceDimensions = dimensions;
std::vector<unsigned int> start(dimensions.size(), 0);
std::vector<unsigned int> stride(dimensions.size(), 1);
if((mMode == Overwrite || mMode == Append || mMode == Hyperslab)
&& heavyDataController) {
// Write to the previous dataset
dataSetPath << array.getHeavyDataController()->getDataSetPath();
hdf5FilePath = array.getHeavyDataController()->getFilePath();
dataSetPath << heavyDataController->getDataSetPath();
hdf5FilePath = heavyDataController->getFilePath();
if(mMode == Hyperslab) {
dataspaceDimensions = heavyDataController->getDataspaceDimensions();
start = heavyDataController->getStart();
stride = heavyDataController->getStride();
}
}
else {
dataSetPath << "Data" << mDataSetId;
}
const std::vector<unsigned int> & dimensions = array.getDimensions();
// Open a hdf5 dataset and write to it on disk.
herr_t status;
hsize_t size = array.getSize();
......@@ -261,75 +272,109 @@ XdmfHDF5Writer::write(XdmfArray & array,
}
}
}
// Restore previous error handler
H5Eset_auto2(0, old_func, old_client_data);
hid_t dataspace = H5S_ALL;
hid_t memspace = H5S_ALL;
std::vector<hsize_t> current_dims(dimensions.begin(), dimensions.end());
std::vector<hsize_t> current_dims(dataspaceDimensions.begin(),
dataspaceDimensions.end());
if(dataset < 0) {
std::vector<hsize_t> maximum_dims(dimensions.size(), H5S_UNLIMITED);
memspace = H5Screate_simple(dimensions.size(),
&current_dims[0],
&maximum_dims[0]);
dataspace = H5Screate_simple(dimensions.size(),
&current_dims[0],
&maximum_dims[0]);
hid_t property = H5Pcreate(H5P_DATASET_CREATE);
std::vector<hsize_t> chunk_size(dimensions.size(), 1024);
status = H5Pset_chunk(property, dimensions.size(), &chunk_size[0]);
dataset = H5Dcreate(mImpl->mHDF5Handle,
dataSetPath.str().c_str(),
datatype,
memspace,
dataspace,
H5P_DEFAULT,
property,
H5P_DEFAULT);
status = H5Pclose(property);
}
else {
if(mMode == Append) {
// Need to resize dataset to fit new data
if(mMode == Append) {
// Get size of old dataset
dataspace = H5Dget_space(dataset);
hssize_t datasize = H5Sget_simple_extent_npoints(dataspace);
status = H5Sclose(dataspace);
// Resize to fit size of old and new data.
hsize_t newSize = size + datasize;
status = H5Dset_extent(dataset, &newSize);
// Select hyperslab to write to.
memspace = H5Screate_simple(1, &size, NULL);
dataspace = H5Dget_space(dataset);
hsize_t start = datasize;
status = H5Sselect_hyperslab(dataspace,
H5S_SELECT_SET,
&start,
NULL,
&size,
NULL) ;
}
else {
// Overwriting - dataset rank must remain the same (hdf5 constraint)
hid_t dataspace = H5Dget_space(dataset);
const unsigned int ndims = H5Sget_simple_extent_ndims(dataspace);
if(ndims != current_dims.size())
XdmfError::message(XdmfError::FATAL, \
"Data set rank different -- ndims != "
"current_dims.size() -- in "
"XdmfHDF5Writer::write");
status = H5Dset_extent(dataset, &current_dims[0]);
if(status < 0) {
XdmfError::message(XdmfError::FATAL,
"H5Dset_extent returned failure in "
"XdmfHDF5Writer::write -- status: " + status);
}
// Get size of old dataset
dataspace = H5Dget_space(dataset);
hssize_t datasize = H5Sget_simple_extent_npoints(dataspace);
status = H5Sclose(dataspace);
// Resize to fit size of old and new data.
hsize_t newSize = size + datasize;
status = H5Dset_extent(dataset, &newSize);
// Select hyperslab to write to.
memspace = H5Screate_simple(1, &size, NULL);
dataspace = H5Dget_space(dataset);
hsize_t start = datasize;
status = H5Sselect_hyperslab(dataspace,
H5S_SELECT_SET,
&start,
NULL,
&size,
NULL);
}
else if(mMode == Overwrite) {
// Overwriting - dataset rank must remain the same (hdf5 constraint)
dataspace = H5Dget_space(dataset);
const unsigned int ndims = H5Sget_simple_extent_ndims(dataspace);
if(ndims != current_dims.size())
XdmfError::message(XdmfError::FATAL, \
"Data set rank different -- ndims != "
"current_dims.size() -- in "
"XdmfHDF5Writer::write");
status = H5Dset_extent(dataset, &current_dims[0]);
dataspace = H5Dget_space(dataset);
}
else if(mMode == Hyperslab) {
// Hyperslab - dataset rank must remain the same (hdf5 constraint)
dataspace = H5Dget_space(dataset);
const unsigned int ndims = H5Sget_simple_extent_ndims(dataspace);
if(ndims != current_dims.size())
XdmfError::message(XdmfError::FATAL, \
"Data set rank different -- ndims != "
"current_dims.size() -- in "
"XdmfHDF5Writer::write");
status = H5Dset_extent(dataset, &current_dims[0]);
dataspace = H5Dget_space(dataset);
std::vector<hsize_t> count(dimensions.begin(),
dimensions.end());
std::vector<hsize_t> currStride(stride.begin(),
stride.end());
std::vector<hsize_t> currStart(start.begin(),
start.end());
memspace = H5Screate_simple(count.size(),
&(count[0]),
NULL);
status = H5Sselect_hyperslab(dataspace,
H5S_SELECT_SET,
&currStart[0],
&currStride[0],
&count[0],
NULL) ;
if(status < 0) {
XdmfError::message(XdmfError::FATAL,
"H5Dset_extent returned failure in "
"XdmfHDF5Writer::write -- status: " + status);
}
}
status = H5Dwrite(dataset,
datatype,
memspace,
......@@ -359,18 +404,19 @@ XdmfHDF5Writer::write(XdmfArray & array,
shared_ptr<XdmfHDF5Controller>();
unsigned int newSize = array.getSize();
if(mMode == Append && array.getHeavyDataController()) {
newSize = array.getSize() + array.getHeavyDataController()->getSize();
if(mMode == Append && heavyDataController) {
newSize = array.getSize() + heavyDataController->getSize();
newDataController =
this->createHDF5Controller(hdf5FilePath,
dataSetPath.str(),
array.getArrayType(),
std::vector<unsigned int>(1, 0),
std::vector<unsigned int>(1, 1),
std::vector<unsigned int>(1, newSize),
std::vector<unsigned int>(1, newSize));
}
if(mMode == Default || !array.getHeavyDataController()) {
if(mMode == Default || !heavyDataController) {
++mDataSetId;
}
......@@ -379,11 +425,10 @@ XdmfHDF5Writer::write(XdmfArray & array,
this->createHDF5Controller(hdf5FilePath,
dataSetPath.str(),
array.getArrayType(),
std::vector<unsigned int>(dimensions.size(),
0),
std::vector<unsigned int>(dimensions.size(),
1),
dimensions);
start,
stride,
dimensions,
dataspaceDimensions);
}
array.setHeavyDataController(newDataController);
......
core/XdmfHDF5Writer.hpp
View file @ 521435d9
......@@ -85,8 +85,11 @@ protected:
* the hdf5 data set.
* @param stride the number of elements to move in each dimension from the
* hdf5 data set.
* @param count the number of elements to select in each dimension from the
* hdf5 data set. (size in each dimension)
* @param dimensions the number of elements to select in each
* dimension from the hdf5 data set. (size in each dimension)
* @param dataspaceDimensions the number of elements in the entire
* hdf5 data set (may be larger that dimensions if using
* hyperslabs).
*
* @return new HDF5 Controller.
*/
......@@ -96,7 +99,8 @@ protected:
const shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count);
const std::vector<unsigned int> & dimensions,
const std::vector<unsigned int> & dataspaceDimensions);
/**
* Open hdf5 file with a fapl.
......
core/XdmfHDF5WriterDSM.cpp
View file @ 521435d9
......@@ -53,14 +53,16 @@ XdmfHDF5WriterDSM::createHDF5Controller(const std::string & hdf5FilePath,
const shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count)
const std::vector<unsigned int> & dimensions,
const std::vector<unsigned int> & dataspaceDimensions)
{
return XdmfHDF5ControllerDSM::New(hdf5FilePath,
dataSetPath,
type,
start,
stride,
count,
dimensions,
dataspaceDimensions,
mDSMBuffer);
}
......@@ -71,7 +73,6 @@ XdmfHDF5WriterDSM::closeFile()
herr_t status = H5Pclose(mFAPL);
mFAPL = -1;
}
XdmfHDF5Writer::closeFile();
}
......@@ -86,7 +87,7 @@ XdmfHDF5WriterDSM::openFile()
mFAPL = H5Pcreate(H5P_FILE_ACCESS);
// Use DSM driver
H5Pset_fapl_dsm(mFAPL, MPI_COMM_WORLD, mDSMBuffer);
H5Pset_fapl_dsm(mFAPL, MPI_COMM_WORLD, mDSMBuffer, 0);
XdmfHDF5Writer::openFile(mFAPL);
}
......@@ -95,13 +96,14 @@ void XdmfHDF5WriterDSM::visit(XdmfArray & array,
const shared_ptr<XdmfBaseVisitor>)
{
bool closeFAPL = false;
if(mFAPL < 0) {
// Set file access property list for DSM
mFAPL = H5Pcreate(H5P_FILE_ACCESS);
// Use DSM driver
H5Pset_fapl_dsm(mFAPL, MPI_COMM_WORLD, mDSMBuffer);
H5Pset_fapl_dsm(mFAPL, MPI_COMM_WORLD, mDSMBuffer, 0);
closeFAPL = true;
}
......
core/XdmfHDF5WriterDSM.hpp
View file @ 521435d9
......@@ -79,7 +79,8 @@ protected:
const shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count);
const std::vector<unsigned int> & dimensions,
const std::vector<unsigned int> & dataspaceDimensions);
private:
......
core/XdmfHeavyDataController.cpp
View file @ 521435d9
......@@ -23,18 +23,30 @@
#include <functional>
#include <numeric>
#include "XdmfError.hpp"
#include "XdmfHeavyDataController.hpp"
#include "XdmfSystemUtils.hpp"
XdmfHeavyDataController::XdmfHeavyDataController(const std::string & filePath,
const std::string & dataSetPath,
const shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & dimensions) :
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & dimensions,
const std::vector<unsigned int> & dataspaceDimensions) :
mDataSetPath(dataSetPath),
mDataspaceDimensions(dataspaceDimensions),
mDimensions(dimensions),
mFilePath(filePath),
mStart(start),
mStride(stride),
mType(type)
{
if(!(mStart.size() == mStride.size() &&
mStride.size() == mDimensions.size()))
XdmfError::message(XdmfError::FATAL,
"mStart, mStride, mDimensions must all be of equal "
"length in XdmfHeavyDataController constructor");
}
XdmfHeavyDataController::~XdmfHeavyDataController()
......@@ -47,6 +59,12 @@ XdmfHeavyDataController::getDataSetPath() const
return mDataSetPath;
}
std::vector<unsigned int>
XdmfHeavyDataController::getDataspaceDimensions() const
{
return mDataspaceDimensions;
}
std::vector<unsigned int> XdmfHeavyDataController::getDimensions() const
{
return mDimensions;
......@@ -67,6 +85,18 @@ XdmfHeavyDataController::getSize() const
std::multiplies<unsigned int>());
}
std::vector<unsigned int>
XdmfHeavyDataController::getStart() const
{
return mStart;
}