Commit d5ebbc9f authored by Kenneth Leiter's avatar Kenneth Leiter

ENH: Performance improvements for HDF5 writer.

Choose dataset name more carefully to avoid querying the file many times.
Use a more efficient function to check if a dataset already exists in a file.
Get DSM working with new changes.
Fix gcc compiler warning in test.
parent decc7dd4
...@@ -47,25 +47,10 @@ if(HDF5_FOUND) ...@@ -47,25 +47,10 @@ if(HDF5_FOUND)
get_filename_component(MPI_LIBRARY_DIR ${MPI_LIBRARY} PATH) get_filename_component(MPI_LIBRARY_DIR ${MPI_LIBRARY} PATH)
set(XDMF_LIBRARY_DIRS ${XDMF_LIBRARY_DIRS} ${MPI_LIBRARY_DIR}) set(XDMF_LIBRARY_DIRS ${XDMF_LIBRARY_DIRS} ${MPI_LIBRARY_DIR})
endif(MPI_FOUND) endif(MPI_FOUND)
option(XDMF_BUILD_DSM OFF)
if(XDMF_BUILD_DSM)
find_package(H5FDdsm REQUIRED NO_MODULE)
if(H5FDdsm_FOUND)
include_directories(${H5FD_DSM_INCLUDE_DIR})
get_filename_component(H5FD_DSM_LIBRARY_DIR
"${H5FD_DSM_INCLUDE_DIR}/../lib" REALPATH)
set(XDMF_LIBRARY_DIRS ${XDMF_LIBRARY_DIRS} ${H5FD_DSM_LIBRARY_DIR})
get_property(h5fddsm_lib_location TARGET H5FDdsm PROPERTY LOCATION)
set(H5FD_DSM_LIBRARIES ${h5fddsm_lib_location})
else(H5FDdsm_FOUND)
message(SEND_ERROR
"Cannot find HDF5 DSM! Please set H5FDdsm_DIR and configure again.")
endif(H5FDdsm_FOUND)
endif(XDMF_BUILD_DSM)
endif(HDF5_IS_PARALLEL) endif(HDF5_IS_PARALLEL)
get_filename_component(HDF5_ROOT "${HDF5_INCLUDE_DIR}/../" REALPATH) get_filename_component(HDF5_ROOT "${HDF5_INCLUDE_DIR}/../" REALPATH)
set(HDF5_BINARY_DIRS ${HDF5_ROOT}/bin ${HDF5_ROOT}/dll) set(HDF5_BINARY_DIRS ${HDF5_ROOT}/bin ${HDF5_ROOT}/dll)
set(XDMF_LIBRARIES ${XDMF_LIBRARIES} ${HDF5_LIBRARIES} ${H5FD_DSM_LIBRARIES}) set(XDMF_LIBRARIES ${XDMF_LIBRARIES} ${HDF5_LIBRARIES})
set(XDMF_HDF5_BINARY_DIRS ${HDF5_BINARY_DIRS} PARENT_SCOPE) set(XDMF_HDF5_BINARY_DIRS ${HDF5_BINARY_DIRS} PARENT_SCOPE)
set(XDMF_BINARIES ${XDMF_BINARIES} ${HDF5_BINARY_DIRS}) set(XDMF_BINARIES ${XDMF_BINARIES} ${HDF5_BINARY_DIRS})
endif(HDF5_FOUND) endif(HDF5_FOUND)
...@@ -98,11 +83,26 @@ set(XdmfCoreSources ...@@ -98,11 +83,26 @@ set(XdmfCoreSources
XdmfVisitor XdmfVisitor
XdmfWriter) XdmfWriter)
option(XDMF_BUILD_DSM OFF)
if(XDMF_BUILD_DSM) if(XDMF_BUILD_DSM)
set(XdmfCoreSources set(XdmfCoreSources
${XdmfCoreSources} ${XdmfCoreSources}
XdmfHDF5ControllerDSM XdmfHDF5ControllerDSM
XdmfHDF5WriterDSM) XdmfHDF5WriterDSM)
find_package(H5FDdsm REQUIRED NO_MODULE)
if(H5FDdsm_FOUND)
include_directories(${H5FD_DSM_INCLUDE_DIR})
get_filename_component(H5FD_DSM_LIBRARY_DIR
"${H5FD_DSM_INCLUDE_DIR}/../lib" REALPATH)
set(XDMF_LIBRARY_DIRS ${XDMF_LIBRARY_DIRS} ${H5FD_DSM_LIBRARY_DIR})
get_property(h5fddsm_lib_location TARGET H5FDdsm PROPERTY LOCATION)
set(H5FD_DSM_LIBRARIES ${h5fddsm_lib_location})
set(XDMF_LIBRARIES ${XDMF_LIBRARIES} ${H5FD_DSM_LIBRARIES})
else(H5FDdsm_FOUND)
message(SEND_ERROR
"Cannot find HDF5 DSM! Please set H5FDdsm_DIR and configure again.")
endif(H5FDdsm_FOUND)
endif(XDMF_BUILD_DSM) endif(XDMF_BUILD_DSM)
add_library(XdmfCore ${XdmfCoreSources}) add_library(XdmfCore ${XdmfCoreSources})
......
...@@ -56,8 +56,9 @@ public: ...@@ -56,8 +56,9 @@ public:
} }
}; };
void int
openFile(const std::string & filePath) openFile(const std::string & filePath,
const int fapl)
{ {
if(mHDF5Handle >= 0) { if(mHDF5Handle >= 0) {
...@@ -71,12 +72,16 @@ public: ...@@ -71,12 +72,16 @@ public:
H5Eget_auto(0, &old_func, &old_client_data); H5Eget_auto(0, &old_func, &old_client_data);
H5Eset_auto2(0, NULL, NULL); H5Eset_auto2(0, NULL, NULL);
int fapl = H5P_DEFAULT; int toReturn = 0;
if(H5Fis_hdf5(filePath.c_str()) > 0) { if(H5Fis_hdf5(filePath.c_str()) > 0) {
mHDF5Handle = H5Fopen(filePath.c_str(), mHDF5Handle = H5Fopen(filePath.c_str(),
H5F_ACC_RDWR, H5F_ACC_RDWR,
fapl); fapl);
hsize_t numObjects;
herr_t status = H5Gget_num_objs(mHDF5Handle,
&numObjects);
toReturn = numObjects;
} }
else { else {
mHDF5Handle = H5Fcreate(filePath.c_str(), mHDF5Handle = H5Fcreate(filePath.c_str(),
...@@ -88,6 +93,8 @@ public: ...@@ -88,6 +93,8 @@ public:
// Restore previous error handler // Restore previous error handler
H5Eset_auto2(0, old_func, old_client_data); H5Eset_auto2(0, old_func, old_client_data);
return toReturn;
} }
hid_t mHDF5Handle; hid_t mHDF5Handle;
...@@ -141,7 +148,14 @@ XdmfHDF5Writer::closeFile() ...@@ -141,7 +148,14 @@ XdmfHDF5Writer::closeFile()
void void
XdmfHDF5Writer::openFile() XdmfHDF5Writer::openFile()
{ {
mImpl->openFile(mFilePath); this->openFile(H5P_DEFAULT);
}
void
XdmfHDF5Writer::openFile(const int fapl)
{
mDataSetId = mImpl->openFile(mFilePath,
fapl);
} }
void void
...@@ -222,7 +236,8 @@ XdmfHDF5Writer::write(XdmfArray & array, ...@@ -222,7 +236,8 @@ XdmfHDF5Writer::write(XdmfArray & array,
bool closeFile = false; bool closeFile = false;
if(mImpl->mHDF5Handle < 0) { if(mImpl->mHDF5Handle < 0) {
mImpl->openFile(hdf5FilePath); mImpl->openFile(hdf5FilePath,
fapl);
closeFile = true; closeFile = true;
} }
...@@ -232,12 +247,19 @@ XdmfHDF5Writer::write(XdmfArray & array, ...@@ -232,12 +247,19 @@ XdmfHDF5Writer::write(XdmfArray & array,
// if default mode find a new data set to write to (keep // if default mode find a new data set to write to (keep
// incrementing dataSetId) // incrementing dataSetId)
while(dataset >= 0 && mMode == Default) { if(dataset >= 0 && mMode == Default) {
while(true) {
dataSetPath.str(std::string()); dataSetPath.str(std::string());
dataSetPath << "Data" << ++mDataSetId; dataSetPath << "Data" << ++mDataSetId;
if(!H5Lexists(mImpl->mHDF5Handle,
dataSetPath.str().c_str(),
H5P_DEFAULT)) {
dataset = H5Dopen(mImpl->mHDF5Handle, dataset = H5Dopen(mImpl->mHDF5Handle,
dataSetPath.str().c_str(), dataSetPath.str().c_str(),
H5P_DEFAULT); H5P_DEFAULT);
break;
}
}
} }
// Restore previous error handler // Restore previous error handler
......
...@@ -63,9 +63,9 @@ public: ...@@ -63,9 +63,9 @@ public:
virtual ~XdmfHDF5Writer(); virtual ~XdmfHDF5Writer();
void closeFile(); virtual void closeFile();
void openFile(); virtual void openFile();
virtual void visit(XdmfArray & array, virtual void visit(XdmfArray & array,
const shared_ptr<XdmfBaseVisitor> visitor); const shared_ptr<XdmfBaseVisitor> visitor);
...@@ -98,6 +98,13 @@ protected: ...@@ -98,6 +98,13 @@ protected:
const std::vector<unsigned int> & stride, const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count); const std::vector<unsigned int> & count);
/**
* Open hdf5 file with a fapl.
*
* @param fapl the file access property list for the hdf5 file.
*/
void openFile(const int fapl);
/** /**
* Write the XdmfArray to a hdf5 file. * Write the XdmfArray to a hdf5 file.
* *
......
...@@ -38,7 +38,8 @@ XdmfHDF5WriterDSM::New(const std::string & filePath, ...@@ -38,7 +38,8 @@ XdmfHDF5WriterDSM::New(const std::string & filePath,
XdmfHDF5WriterDSM::XdmfHDF5WriterDSM(const std::string & filePath, XdmfHDF5WriterDSM::XdmfHDF5WriterDSM(const std::string & filePath,
H5FDdsmBuffer * const dsmBuffer) : H5FDdsmBuffer * const dsmBuffer) :
XdmfHDF5Writer(filePath), XdmfHDF5Writer(filePath),
mDSMBuffer(dsmBuffer) mDSMBuffer(dsmBuffer),
mFAPL(-1)
{ {
} }
...@@ -63,18 +64,54 @@ XdmfHDF5WriterDSM::createHDF5Controller(const std::string & hdf5FilePath, ...@@ -63,18 +64,54 @@ XdmfHDF5WriterDSM::createHDF5Controller(const std::string & hdf5FilePath,
mDSMBuffer); mDSMBuffer);
} }
void
XdmfHDF5WriterDSM::closeFile()
{
if(mFAPL >= 0) {
herr_t status = H5Pclose(mFAPL);
mFAPL = -1;
}
XdmfHDF5Writer::closeFile();
}
void
XdmfHDF5WriterDSM::openFile()
{
if(mFAPL >= 0) {
this->closeFile();
}
// Set file access property list for DSM
mFAPL = H5Pcreate(H5P_FILE_ACCESS);
// Use DSM driver
H5Pset_fapl_dsm(mFAPL, MPI_COMM_WORLD, mDSMBuffer);
XdmfHDF5Writer::openFile(mFAPL);
}
void XdmfHDF5WriterDSM::visit(XdmfArray & array, void XdmfHDF5WriterDSM::visit(XdmfArray & array,
const shared_ptr<XdmfBaseVisitor>) const shared_ptr<XdmfBaseVisitor>)
{ {
bool closeFAPL = false;
if(mFAPL < 0) {
// Set file access property list for DSM // Set file access property list for DSM
hid_t fapl = H5Pcreate(H5P_FILE_ACCESS); mFAPL = H5Pcreate(H5P_FILE_ACCESS);
// Use DSM driver // Use DSM driver
H5Pset_fapl_dsm(fapl, MPI_COMM_WORLD, mDSMBuffer); H5Pset_fapl_dsm(mFAPL, MPI_COMM_WORLD, mDSMBuffer);
closeFAPL = true;
}
// Write to DSM Buffer // Write to DSM Buffer
this->write(array, fapl); this->write(array, mFAPL);
if(closeFAPL) {
// Close file access property list // Close file access property list
herr_t status = H5Pclose(fapl); herr_t status = H5Pclose(mFAPL);
mFAPL = -1;
}
} }
...@@ -61,6 +61,10 @@ public: ...@@ -61,6 +61,10 @@ public:
virtual ~XdmfHDF5WriterDSM(); virtual ~XdmfHDF5WriterDSM();
void closeFile();
void openFile();
void visit(XdmfArray & array, void visit(XdmfArray & array,
const shared_ptr<XdmfBaseVisitor> visitor); const shared_ptr<XdmfBaseVisitor> visitor);
...@@ -83,6 +87,7 @@ private: ...@@ -83,6 +87,7 @@ private:
void operator=(const XdmfHDF5WriterDSM &); // Not implemented. void operator=(const XdmfHDF5WriterDSM &); // Not implemented.
H5FDdsmBuffer * mDSMBuffer; H5FDdsmBuffer * mDSMBuffer;
int mFAPL;
}; };
......
...@@ -11,8 +11,6 @@ ...@@ -11,8 +11,6 @@
int main(int, char *) int main(int, char *)
{ {
const double epsilon = 1e-6;
shared_ptr<XdmfTopologyConverter> converter = shared_ptr<XdmfTopologyConverter> converter =
XdmfTopologyConverter::New(); XdmfTopologyConverter::New();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment