ENH: Performance improvements for HDF5 writer.

Choose dataset name more carefully to avoid querying the file many times.
Use a more efficient function to check if a dataset already exists in a file.
Get DSM working with new changes.
Fix gcc compiler warning in test.

core/CMakeLists.txt

@@ -47,25 +47,10 @@ if(HDF5_FOUND)
       get_filename_component(MPI_LIBRARY_DIR ${MPI_LIBRARY} PATH)
       set(XDMF_LIBRARY_DIRS ${XDMF_LIBRARY_DIRS} ${MPI_LIBRARY_DIR})
     endif(MPI_FOUND)
-    option(XDMF_BUILD_DSM OFF)
-    if(XDMF_BUILD_DSM)
-      find_package(H5FDdsm REQUIRED NO_MODULE)
-      if(H5FDdsm_FOUND)
-        include_directories(${H5FD_DSM_INCLUDE_DIR})
-        get_filename_component(H5FD_DSM_LIBRARY_DIR 
-                               "${H5FD_DSM_INCLUDE_DIR}/../lib" REALPATH)
-	set(XDMF_LIBRARY_DIRS ${XDMF_LIBRARY_DIRS} ${H5FD_DSM_LIBRARY_DIR})
-        get_property(h5fddsm_lib_location TARGET H5FDdsm PROPERTY LOCATION)
-        set(H5FD_DSM_LIBRARIES ${h5fddsm_lib_location})
-      else(H5FDdsm_FOUND)
-        message(SEND_ERROR
-          "Cannot find HDF5 DSM!  Please set H5FDdsm_DIR and configure again.")
-      endif(H5FDdsm_FOUND)
-    endif(XDMF_BUILD_DSM)
   endif(HDF5_IS_PARALLEL)
   get_filename_component(HDF5_ROOT "${HDF5_INCLUDE_DIR}/../" REALPATH)
   set(HDF5_BINARY_DIRS ${HDF5_ROOT}/bin ${HDF5_ROOT}/dll)
-  set(XDMF_LIBRARIES ${XDMF_LIBRARIES} ${HDF5_LIBRARIES} ${H5FD_DSM_LIBRARIES})
+  set(XDMF_LIBRARIES ${XDMF_LIBRARIES} ${HDF5_LIBRARIES})
   set(XDMF_HDF5_BINARY_DIRS ${HDF5_BINARY_DIRS} PARENT_SCOPE)
   set(XDMF_BINARIES ${XDMF_BINARIES} ${HDF5_BINARY_DIRS})
 endif(HDF5_FOUND)
@@ -98,11 +83,26 @@ set(XdmfCoreSources
   XdmfVisitor
   XdmfWriter)
 
+option(XDMF_BUILD_DSM OFF)
 if(XDMF_BUILD_DSM)
   set(XdmfCoreSources
     ${XdmfCoreSources}
     XdmfHDF5ControllerDSM
     XdmfHDF5WriterDSM)
+
+  find_package(H5FDdsm REQUIRED NO_MODULE)
+  if(H5FDdsm_FOUND)
+    include_directories(${H5FD_DSM_INCLUDE_DIR})
+    get_filename_component(H5FD_DSM_LIBRARY_DIR 
+                           "${H5FD_DSM_INCLUDE_DIR}/../lib" REALPATH)
+    set(XDMF_LIBRARY_DIRS ${XDMF_LIBRARY_DIRS} ${H5FD_DSM_LIBRARY_DIR})
+    get_property(h5fddsm_lib_location TARGET H5FDdsm PROPERTY LOCATION)
+    set(H5FD_DSM_LIBRARIES ${h5fddsm_lib_location})
+    set(XDMF_LIBRARIES ${XDMF_LIBRARIES} ${H5FD_DSM_LIBRARIES})
+  else(H5FDdsm_FOUND)
+    message(SEND_ERROR
+      "Cannot find HDF5 DSM!  Please set H5FDdsm_DIR and configure again.")
+  endif(H5FDdsm_FOUND)
 endif(XDMF_BUILD_DSM)
 
 add_library(XdmfCore ${XdmfCoreSources})

core/XdmfHDF5Writer.cpp

@@ -56,8 +56,9 @@ public:
     }
   };  
 
-  void
-  openFile(const std::string & filePath)
+  int
+  openFile(const std::string & filePath,
+           const int fapl)
   {
 
     if(mHDF5Handle >= 0) {
@@ -71,12 +72,16 @@ public:
     H5Eget_auto(0, &old_func, &old_client_data);
     H5Eset_auto2(0, NULL, NULL);
   
-    int fapl = H5P_DEFAULT;
+    int toReturn = 0;
 
     if(H5Fis_hdf5(filePath.c_str()) > 0) {
       mHDF5Handle = H5Fopen(filePath.c_str(), 
                             H5F_ACC_RDWR, 
                             fapl);
+      hsize_t numObjects;
+      herr_t status = H5Gget_num_objs(mHDF5Handle,
+                                      &numObjects);
+      toReturn = numObjects;
     }
     else {
       mHDF5Handle = H5Fcreate(filePath.c_str(),
@@ -88,6 +93,8 @@ public:
     // Restore previous error handler
     H5Eset_auto2(0, old_func, old_client_data);
 
+    return toReturn;
+
   }
 
   hid_t mHDF5Handle;
@@ -141,7 +148,14 @@ XdmfHDF5Writer::closeFile()
 void 
 XdmfHDF5Writer::openFile()
 {
-  mImpl->openFile(mFilePath);
+  this->openFile(H5P_DEFAULT);
+}
+
+void
+XdmfHDF5Writer::openFile(const int fapl)
+{
+  mDataSetId = mImpl->openFile(mFilePath,
+                               fapl);
 }
 
 void
@@ -222,7 +236,8 @@ XdmfHDF5Writer::write(XdmfArray & array,
    
     bool closeFile = false;
     if(mImpl->mHDF5Handle < 0) {
-      mImpl->openFile(hdf5FilePath);
+      mImpl->openFile(hdf5FilePath,
+                      fapl);
       closeFile = true;
     }
 
@@ -232,14 +247,21 @@ XdmfHDF5Writer::write(XdmfArray & array,
 
     // if default mode find a new data set to write to (keep
     // incrementing dataSetId)
-    while(dataset >= 0 && mMode == Default) {
-      dataSetPath.str(std::string());
-      dataSetPath << "Data" << ++mDataSetId;
-      dataset = H5Dopen(mImpl->mHDF5Handle,
-                        dataSetPath.str().c_str(),
-                        H5P_DEFAULT);
+    if(dataset >= 0 && mMode == Default) {
+      while(true) {
+        dataSetPath.str(std::string());
+        dataSetPath << "Data" << ++mDataSetId;
+        if(!H5Lexists(mImpl->mHDF5Handle,
+                      dataSetPath.str().c_str(),
+                      H5P_DEFAULT)) {
+          dataset = H5Dopen(mImpl->mHDF5Handle,
+                            dataSetPath.str().c_str(),
+                            H5P_DEFAULT); 
+          break;
+        }
+      }
     }
-
+    
     // Restore previous error handler
     H5Eset_auto2(0, old_func, old_client_data);
 

core/XdmfHDF5Writer.hpp

@@ -63,9 +63,9 @@ public:
 
   virtual ~XdmfHDF5Writer();
 
-  void closeFile();
+  virtual void closeFile();
 
-  void openFile();
+  virtual void openFile();
 
   virtual void visit(XdmfArray & array,
                      const shared_ptr<XdmfBaseVisitor> visitor);
@@ -98,6 +98,13 @@ protected:
                        const std::vector<unsigned int> & stride,
                        const std::vector<unsigned int> & count);
 
+  /**
+   * Open hdf5 file with a fapl.
+   *
+   * @param fapl the file access property list for the hdf5 file.
+   */
+  void openFile(const int fapl);
+
   /**
    * Write the XdmfArray to a hdf5 file.
    *

core/XdmfHDF5WriterDSM.cpp

@@ -38,7 +38,8 @@ XdmfHDF5WriterDSM::New(const std::string & filePath,
 XdmfHDF5WriterDSM::XdmfHDF5WriterDSM(const std::string & filePath,
                                      H5FDdsmBuffer * const dsmBuffer) :
   XdmfHDF5Writer(filePath),
-  mDSMBuffer(dsmBuffer)
+  mDSMBuffer(dsmBuffer),
+  mFAPL(-1)
 {
 }
 
@@ -63,18 +64,54 @@ XdmfHDF5WriterDSM::createHDF5Controller(const std::string & hdf5FilePath,
                                     mDSMBuffer);
 }
 
-void XdmfHDF5WriterDSM::visit(XdmfArray & array,
-                              const shared_ptr<XdmfBaseVisitor>)
+void 
+XdmfHDF5WriterDSM::closeFile()
+{
+  if(mFAPL >= 0) {
+    herr_t status = H5Pclose(mFAPL);
+    mFAPL = -1;
+  }
+
+  XdmfHDF5Writer::closeFile();
+}
+
+void 
+XdmfHDF5WriterDSM::openFile()
 {
+  if(mFAPL >= 0) {
+    this->closeFile();
+  }
+
   // Set file access property list for DSM
-  hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
+  mFAPL = H5Pcreate(H5P_FILE_ACCESS);
 
   // Use DSM driver
-  H5Pset_fapl_dsm(fapl, MPI_COMM_WORLD, mDSMBuffer);
+  H5Pset_fapl_dsm(mFAPL, MPI_COMM_WORLD, mDSMBuffer);
+
+  XdmfHDF5Writer::openFile(mFAPL);
+}
+
+void XdmfHDF5WriterDSM::visit(XdmfArray & array,
+                              const shared_ptr<XdmfBaseVisitor>)
+{
+  bool closeFAPL = false;
+
+  if(mFAPL < 0) {
+    // Set file access property list for DSM
+    mFAPL = H5Pcreate(H5P_FILE_ACCESS);
+
+    // Use DSM driver
+    H5Pset_fapl_dsm(mFAPL, MPI_COMM_WORLD, mDSMBuffer);
+
+    closeFAPL = true;
+  }
 
   // Write to DSM Buffer
-  this->write(array, fapl);
+  this->write(array, mFAPL);
 
-  // Close file access property list
-  herr_t status = H5Pclose(fapl);
+  if(closeFAPL) {
+    // Close file access property list
+    herr_t status = H5Pclose(mFAPL);
+    mFAPL = -1;
+  }
 }

core/XdmfHDF5WriterDSM.hpp

@@ -61,6 +61,10 @@ public:
 
   virtual ~XdmfHDF5WriterDSM();
 
+  void closeFile();
+
+  void openFile();
+
   void visit(XdmfArray & array,
              const shared_ptr<XdmfBaseVisitor> visitor);
 
@@ -83,6 +87,7 @@ private:
   void operator=(const XdmfHDF5WriterDSM &);  // Not implemented.
 
   H5FDdsmBuffer * mDSMBuffer;
+  int mFAPL;
 
 };
 

utils/tests/Cxx/TestXdmfTopologyConverter.cpp

@@ -11,8 +11,6 @@
 
 int main(int, char *)
 {
-  const double epsilon = 1e-6;
-
   shared_ptr<XdmfTopologyConverter> converter =
     XdmfTopologyConverter::New();