ENH: Add ability to write in parallel to hdf5 file.

Modified writer to utilize hyperslabs to write in parallel to file.
Updated DSM to newest version (0.9.9).
Modified DSM test to write in parallel to DSM buffer.
Add test for new hyperslab writing mode.

core/CMakeLists.txt

@@ -93,13 +93,13 @@ if(XDMF_BUILD_DSM)
 
   find_package(H5FDdsm REQUIRED NO_MODULE)
   if(H5FDdsm_FOUND)
-    include_directories(${H5FD_DSM_INCLUDE_DIR})
-    get_filename_component(H5FD_DSM_LIBRARY_DIR 
-                           "${H5FD_DSM_INCLUDE_DIR}/../lib" REALPATH)
-    set(XDMF_LIBRARY_DIRS ${XDMF_LIBRARY_DIRS} ${H5FD_DSM_LIBRARY_DIR})
+    include_directories(${H5FDdsm_INCLUDE_DIR})
+    get_filename_component(H5FDdsm_LIBRARY_DIR 
+                           "${H5FDdsm_INCLUDE_DIR}/../lib" REALPATH)
+    set(XDMF_LIBRARY_DIRS ${XDMF_LIBRARY_DIRS} ${H5FDdsm_LIBRARY_DIR})
     get_property(h5fddsm_lib_location TARGET H5FDdsm PROPERTY LOCATION)
-    set(H5FD_DSM_LIBRARIES ${h5fddsm_lib_location})
-    set(XDMF_LIBRARIES ${XDMF_LIBRARIES} ${H5FD_DSM_LIBRARIES})
+    set(H5FDdsm_LIBRARIES ${h5fddsm_lib_location})
+    set(XDMF_LIBRARIES ${XDMF_LIBRARIES} ${H5FDdsm_LIBRARIES})
   else(H5FDdsm_FOUND)
     message(SEND_ERROR
       "Cannot find HDF5 DSM!  Please set H5FDdsm_DIR and configure again.")

core/XdmfArray.cpp

@@ -824,6 +824,7 @@ XdmfArray::populateItem(const std::map<std::string, std::string> & itemPropertie
                                                         0),
                               std::vector<unsigned int>(mDimensions.size(),
                                                         1),
+                              mDimensions,
                               mDimensions);
   }
   else if(formatVal.compare("XML") == 0) {

core/XdmfHDF5Controller.cpp

@@ -25,9 +25,9 @@
 #include <sstream>
 #include "XdmfArray.hpp"
 #include "XdmfArrayType.hpp"
+#include "XdmfError.hpp"
 #include "XdmfHDF5Controller.hpp"
 #include "XdmfSystemUtils.hpp"
-#include "XdmfError.hpp"
 
 shared_ptr<XdmfHDF5Controller>
 XdmfHDF5Controller::New(const std::string & hdf5FilePath,
@@ -35,14 +35,17 @@ XdmfHDF5Controller::New(const std::string & hdf5FilePath,
                         const shared_ptr<const XdmfArrayType> type,
                         const std::vector<unsigned int> & start,
                         const std::vector<unsigned int> & stride,
-                        const std::vector<unsigned int> & count)
+                        const std::vector<unsigned int> & dimensions,
+                        const std::vector<unsigned int> & dataspaceDimensions)
 {
-  shared_ptr<XdmfHDF5Controller> p(new XdmfHDF5Controller(hdf5FilePath,
-                                                          dataSetPath,
-                                                          type,
-                                                          start,
-                                                          stride,
-                                                          count));
+  shared_ptr<XdmfHDF5Controller> 
+    p(new XdmfHDF5Controller(hdf5FilePath,
+                             dataSetPath,
+                             type,
+                             start,
+                             stride,
+                             dimensions,
+                             dataspaceDimensions));
   return p;
 }
 
@@ -51,18 +54,16 @@ XdmfHDF5Controller::XdmfHDF5Controller(const std::string & hdf5FilePath,
                                        const shared_ptr<const XdmfArrayType> type,
                                        const std::vector<unsigned int> & start,
                                        const std::vector<unsigned int> & stride,
-                                       const std::vector<unsigned int> & count) :
+                                       const std::vector<unsigned int> & dimensions,
+                                       const std::vector<unsigned int> & dataspaceDimensions) :
   XdmfHeavyDataController(hdf5FilePath,
                           dataSetPath,
                           type,
-                          count),
-  mStart(start),
-  mStride(stride)
+                          start,
+                          stride,
+                          dimensions,
+                          dataspaceDimensions)
 {
-  if(!(mStart.size() == mStride.size() && mStride.size() == mDimensions.size()))
-    XdmfError::message(XdmfError::FATAL,
-                       "mStart, mStride, mDimensions must all be of equal "
-                       "length in XdmfHDF5Controller constructor");
 }
 
 XdmfHDF5Controller::~XdmfHDF5Controller()

core/XdmfHDF5Controller.hpp

@@ -53,8 +53,11 @@ public:
    * the hdf5 data set.
    * @param stride the number of elements to move in each dimension from the
    * hdf5 data set.
-   * @param count the number of elements to select in each dimension from the
-   * hdf5 data set. (size in each dimension)
+   * @param dimensions the number of elements to select in each
+   * dimension from the hdf5 data set. (size in each dimension)
+   * @param dataspaceDimensions the number of elements in the entire
+   * hdf5 data set (may be larger that dimensions if using
+   * hyperslabs).
    *
    * @return new HDF5 Controller.
    */
@@ -64,7 +67,8 @@ public:
       const shared_ptr<const XdmfArrayType> type,
       const std::vector<unsigned int> & start,
       const std::vector<unsigned int> & stride,
-      const std::vector<unsigned int> & count);
+      const std::vector<unsigned int> & dimensions,
+      const std::vector<unsigned int> & dataspaceDimensions);
 
   virtual std::string getName() const;
 
@@ -77,7 +81,8 @@ protected:
                      const shared_ptr<const XdmfArrayType> type,
                      const std::vector<unsigned int> & start,
                      const std::vector<unsigned int> & stride,
-                     const std::vector<unsigned int> & count);
+                     const std::vector<unsigned int> & dimensions,
+                     const std::vector<unsigned int> & dataspaceDimensions);
 
   void read(XdmfArray * const array, const int fapl);
 
@@ -86,9 +91,6 @@ private:
   XdmfHDF5Controller(const XdmfHDF5Controller &);  // Not implemented.
   void operator=(const XdmfHDF5Controller &);  // Not implemented.
 
-  const std::vector<unsigned int> mStart;
-  const std::vector<unsigned int> mStride;
-
 };
 
 #endif /* XDMFHDF5CONTROLLER_HPP_ */

core/XdmfHDF5ControllerDSM.cpp

@@ -31,16 +31,19 @@ XdmfHDF5ControllerDSM::New(const std::string & hdf5FilePath,
                            const shared_ptr<const XdmfArrayType> type,
                            const std::vector<unsigned int> & start,
                            const std::vector<unsigned int> & stride,
-                           const std::vector<unsigned int> & count,
+                           const std::vector<unsigned int> & dimensions,
+                           const std::vector<unsigned int> & datspaceDimensions,
                            H5FDdsmBuffer * const dsmBuffer)
 {
-  shared_ptr<XdmfHDF5ControllerDSM> p(new XdmfHDF5ControllerDSM(hdf5FilePath,
-                                                                dataSetPath,
-                                                                type,
-                                                                start,
-                                                                stride,
-                                                                count,
-                                                                dsmBuffer));
+  shared_ptr<XdmfHDF5ControllerDSM> 
+    p(new XdmfHDF5ControllerDSM(hdf5FilePath,
+                                dataSetPath,
+                                type,
+                                start,
+                                stride,
+                                dimensions,
+                                datspaceDimensions,
+                                dsmBuffer));
   return p;
 }
 
@@ -49,9 +52,16 @@ XdmfHDF5ControllerDSM::XdmfHDF5ControllerDSM(const std::string & hdf5FilePath,
                                              const shared_ptr<const XdmfArrayType> type,
                                              const std::vector<unsigned int> & start,
                                              const std::vector<unsigned int> & stride,
-                                             const std::vector<unsigned int> & count,
+                                             const std::vector<unsigned int> & dimensions,
+                                             const std::vector<unsigned int> & dataspaceDimensions,
                                              H5FDdsmBuffer * const dsmBuffer) :
-  XdmfHDF5Controller(hdf5FilePath, dataSetPath, type, start, stride, count),
+  XdmfHDF5Controller(hdf5FilePath, 
+                     dataSetPath, 
+                     type, 
+                     start,
+                     stride,
+                     dimensions, 
+                     dataspaceDimensions),
   mDSMBuffer(dsmBuffer)
 {
 }
@@ -71,7 +81,7 @@ void XdmfHDF5ControllerDSM::read(XdmfArray * const array)
   hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
 
   // Use DSM driver
-  H5Pset_fapl_dsm(fapl, MPI_COMM_WORLD, mDSMBuffer);
+  H5Pset_fapl_dsm(fapl, MPI_COMM_WORLD, mDSMBuffer, 0);
 
   // Read from DSM Buffer
   XdmfHDF5Controller::read(array, fapl);

core/XdmfHDF5ControllerDSM.hpp

@@ -55,7 +55,8 @@ public:
       const shared_ptr<const XdmfArrayType> type,
       const std::vector<unsigned int> & start,
       const std::vector<unsigned int> & stride,
-      const std::vector<unsigned int> & count,
+      const std::vector<unsigned int> & dimensions,
+      const std::vector<unsigned int> & dataspaceDimensions,
       H5FDdsmBuffer * const dsmBuffer);
 
   std::string getName() const;
@@ -69,7 +70,8 @@ protected:
                         const shared_ptr<const XdmfArrayType> type,
                         const std::vector<unsigned int> & start,
                         const std::vector<unsigned int> & stride,
-                        const std::vector<unsigned int> & count,
+                        const std::vector<unsigned int> & dimensions,
+                        const std::vector<unsigned int> & dataspaceDimensions,
                         H5FDdsmBuffer * const dsmBuffer);
 
 private:

core/XdmfHDF5Writer.hpp

@@ -85,8 +85,11 @@ protected:
    * the hdf5 data set.
    * @param stride the number of elements to move in each dimension from the
    * hdf5 data set.
-   * @param count the number of elements to select in each dimension from the
-   * hdf5 data set. (size in each dimension)
+   * @param dimensions the number of elements to select in each
+   * dimension from the hdf5 data set. (size in each dimension)
+   * @param dataspaceDimensions the number of elements in the entire
+   * hdf5 data set (may be larger that dimensions if using
+   * hyperslabs).
    *
    * @return new HDF5 Controller.
    */
@@ -96,7 +99,8 @@ protected:
                        const shared_ptr<const XdmfArrayType> type,
                        const std::vector<unsigned int> & start,
                        const std::vector<unsigned int> & stride,
-                       const std::vector<unsigned int> & count);
+                       const std::vector<unsigned int> & dimensions,
+                       const std::vector<unsigned int> & dataspaceDimensions);
 
   /**
    * Open hdf5 file with a fapl.

core/XdmfHDF5WriterDSM.cpp

@@ -53,14 +53,16 @@ XdmfHDF5WriterDSM::createHDF5Controller(const std::string & hdf5FilePath,
                                         const shared_ptr<const XdmfArrayType> type,
                                         const std::vector<unsigned int> & start,
                                         const std::vector<unsigned int> & stride,
-                                        const std::vector<unsigned int> & count)
+                                        const std::vector<unsigned int> & dimensions,
+                                        const std::vector<unsigned int> & dataspaceDimensions)
 {
   return XdmfHDF5ControllerDSM::New(hdf5FilePath,
                                     dataSetPath,
                                     type,
                                     start,
                                     stride,
-                                    count,
+                                    dimensions,
+                                    dataspaceDimensions,
                                     mDSMBuffer);
 }
 
@@ -71,7 +73,6 @@ XdmfHDF5WriterDSM::closeFile()
     herr_t status = H5Pclose(mFAPL);
     mFAPL = -1;
   }
-
   XdmfHDF5Writer::closeFile();
 }
 
@@ -86,7 +87,7 @@ XdmfHDF5WriterDSM::openFile()
   mFAPL = H5Pcreate(H5P_FILE_ACCESS);
 
   // Use DSM driver
-  H5Pset_fapl_dsm(mFAPL, MPI_COMM_WORLD, mDSMBuffer);
+  H5Pset_fapl_dsm(mFAPL, MPI_COMM_WORLD, mDSMBuffer, 0);
 
   XdmfHDF5Writer::openFile(mFAPL);
 }
@@ -95,13 +96,14 @@ void XdmfHDF5WriterDSM::visit(XdmfArray & array,
                               const shared_ptr<XdmfBaseVisitor>)
 {
   bool closeFAPL = false;
-
+  
   if(mFAPL < 0) {
+
     // Set file access property list for DSM
     mFAPL = H5Pcreate(H5P_FILE_ACCESS);
 
     // Use DSM driver
-    H5Pset_fapl_dsm(mFAPL, MPI_COMM_WORLD, mDSMBuffer);
+    H5Pset_fapl_dsm(mFAPL, MPI_COMM_WORLD, mDSMBuffer, 0);
 
     closeFAPL = true;
   }

core/XdmfHDF5WriterDSM.hpp

@@ -79,7 +79,8 @@ protected:
                        const shared_ptr<const XdmfArrayType> type,
                        const std::vector<unsigned int> & start,
                        const std::vector<unsigned int> & stride,
-                       const std::vector<unsigned int> & count);
+                       const std::vector<unsigned int> & dimensions,
+                       const std::vector<unsigned int> & dataspaceDimensions);
 
 private:
 

core/XdmfHeavyDataController.cpp

@@ -23,18 +23,30 @@
 
 #include <functional>
 #include <numeric>
+#include "XdmfError.hpp"
 #include "XdmfHeavyDataController.hpp"
 #include "XdmfSystemUtils.hpp"
 
 XdmfHeavyDataController::XdmfHeavyDataController(const std::string & filePath,
                                                  const std::string & dataSetPath,
                                                  const shared_ptr<const XdmfArrayType> type,
-                                                 const std::vector<unsigned int> & dimensions) :
+                                                 const std::vector<unsigned int> & start,
+                                                 const std::vector<unsigned int> & stride,
+                                                 const std::vector<unsigned int> & dimensions,
+                                                 const std::vector<unsigned int> & dataspaceDimensions) :
   mDataSetPath(dataSetPath),
+  mDataspaceDimensions(dataspaceDimensions),
   mDimensions(dimensions),
   mFilePath(filePath),
+  mStart(start),
+  mStride(stride),
   mType(type)
 {
+  if(!(mStart.size() == mStride.size() && 
+       mStride.size() == mDimensions.size()))
+    XdmfError::message(XdmfError::FATAL,
+                       "mStart, mStride, mDimensions must all be of equal "
+                       "length in XdmfHeavyDataController constructor");
 }
 
 XdmfHeavyDataController::~XdmfHeavyDataController()
@@ -47,6 +59,12 @@ XdmfHeavyDataController::getDataSetPath() const
   return mDataSetPath;
 }
 
+std::vector<unsigned int> 
+XdmfHeavyDataController::getDataspaceDimensions() const
+{
+  return mDataspaceDimensions;
+}
+
 std::vector<unsigned int> XdmfHeavyDataController::getDimensions() const
 {
   return mDimensions;
@@ -67,6 +85,18 @@ XdmfHeavyDataController::getSize() const
                          std::multiplies<unsigned int>());
 }
 
+std::vector<unsigned int> 
+XdmfHeavyDataController::getStart() const
+{
+  return mStart;
+}
+
+std::vector<unsigned int> 
+XdmfHeavyDataController::getStride() const
+{
+  return mStride;
+}
+
 shared_ptr<const XdmfArrayType>
 XdmfHeavyDataController::getType() const
 {

core/XdmfHeavyDataController.hpp

@@ -61,6 +61,17 @@ public:
    */
   std::string getDataSetPath() const;
 
+  /**
+   * Get the dimensions of the dataspace owned by this
+   * controller. This is the dimension of the entire heavy dataset,
+   * which may be larger than the dimensions of the array (if reading
+   * a piece of a larger dataset).
+   *
+   * @return a vector containing the size in each dimension of the dataspace
+   * owned by this controller.
+   */
+  std::vector<unsigned int> getDataspaceDimensions() const;
+
   /**
    * Get the dimensions of the heavy data set owned by this controller.
    *
@@ -93,6 +104,22 @@ public:
    */
   unsigned int getSize() const;
 
+  /**
+   * Get the start index of the heavy data set owned by this controller.
+   *
+   * @return a vector containing the start index in each dimension of
+   * the heavy data set owned by this controller.
+   */
+  std::vector<unsigned int> getStart() const;
+
+  /**
+   * Get the stride of the heavy data set owned by this controller.
+   *
+   * @return a vector containing the stride in each dimension of the
+   * heavy data set owned by this controller.
+   */
+  std::vector<unsigned int> getStride() const;
+
   /**
    * Get the array type of the heavy data set owned by this
    * controller.
@@ -114,11 +141,17 @@ protected:
   XdmfHeavyDataController(const std::string & filePath,
                           const std::string & dataSetPath,
                           const shared_ptr<const XdmfArrayType> type,
-                          const std::vector<unsigned int> & dimensions);
+                          const std::vector<unsigned int> & start,
+                          const std::vector<unsigned int> & stride,
+                          const std::vector<unsigned int> & dimensions,
+                          const std::vector<unsigned int> & dataspaceDimensions);
 
   const std::string mDataSetPath;
+  const std::vector<unsigned int> mDataspaceDimensions;
   const std::vector<unsigned int> mDimensions;
   const std::string mFilePath;
+  const std::vector<unsigned int> mStart;
+  const std::vector<unsigned int> mStride;
   const shared_ptr<const XdmfArrayType> mType;
 
 private:

core/XdmfHeavyDataWriter.hpp

@@ -54,9 +54,14 @@ class XdmfArray;
  *   Overwrite - If an initialized XdmfArray is attached to an heavy dataset
  *               via an XdmfHeavyDataController the writer will write values
  *               to that location, overwriting all previous written values.
+ *               The dataset on disk will be resized appropriately.
  *   Append - If an initialized XdmfArray is attached to an heavy dataset via
  *            an XdmfHeavyDataController the writer will append the values to
  *            the end of the dataset on disk.
+ *   Hyperslab - If an initialized XdmfArray is attached to a heavy dataset
+ *               via an XdmfHeavyDataController the writer will write to a
+ *               hyperslab in the dataset based on the start, stride, and
+ *               dimensions of the XdmfHeavyDataController.
  */
 class XDMFCORE_EXPORT XdmfHeavyDataWriter : public XdmfVisitor,
                                             public Loki::Visitor<XdmfArray> {
@@ -66,7 +71,8 @@ public:
   enum Mode {
     Default,
     Overwrite,
-    Append
+    Append,
+    Hyperslab
   };
 
   virtual ~XdmfHeavyDataWriter() = 0;

core/tests/Cxx/CMakeLists.txt

@@ -21,6 +21,7 @@ ADD_TEST_CXX_PATH("${XDMF_BINARIES}")
 ADD_TEST_CXX(TestXdmfArray)
 ADD_TEST_CXX(TestXdmfArrayMultidimensional)
 ADD_TEST_CXX(TestXdmfArrayWriteRead)
+ADD_TEST_CXX(TestXdmfArrayWriteReadHyperSlabs)
 ADD_TEST_CXX(TestXdmfHDF5Controller)
 ADD_TEST_CXX(TestXdmfHDF5Writer)
 if(XDMF_BUILD_DSM)
@@ -38,6 +39,8 @@ CLEAN_TEST_CXX(TestXdmfArray)
 CLEAN_TEST_CXX(TestXdmfArrayMultidimensional)
 CLEAN_TEST_CXX(TestXdmfArrayWriteRead
   test.h5)
+CLEAN_TEST_CXX(TestXdmfArrayWriteRead
+  testHyperslab.h5)
 CLEAN_TEST_CXX(TestXdmfHDF5Controller)
 CLEAN_TEST_CXX(TestXdmfHDF5Writer
   hdf5WriterTest.h5)

core/tests/Cxx/TestXdmfArrayWriteReadHyperSlabs.cpp

+#include "XdmfArray.hpp"
+#include "XdmfArrayType.hpp"
+#include "XdmfHDF5Controller.hpp"
+#include "XdmfHDF5Writer.hpp"
+
+int main(int, char **)
+{
+  int values1[] = {1, 2};
+  int values2[] = {3, 4};
+
+  shared_ptr<XdmfArray> array1 = XdmfArray::New();
+  array1->insert(0, &values1[0], 2, 1, 1);
+  assert(array1->getSize() == 2);
+  assert(array1->getValuesString().compare("1 2") == 0);
+
+  shared_ptr<XdmfArray> array2 = XdmfArray::New();
+  array2->insert(0, &values2[0], 2, 1, 1);
+  assert(array2->getSize() == 2);
+  assert(array2->getValuesString().compare("3 4") == 0);
+
+  //
+  // write array1 to first part of dataset
+  //
+  shared_ptr<XdmfHDF5Controller> controller1 = 
+    XdmfHDF5Controller::New("testHyperslab.h5",
+                            "data",
+                            XdmfArrayType::Int32(),
+                            std::vector<unsigned int>(1, 0),
+                            std::vector<unsigned int>(1, 1),
+                            std::vector<unsigned int>(1, 2),
+                            std::vector<unsigned int>(1, 4));
+  array1->setHeavyDataController(controller1);
+  shared_ptr<XdmfHDF5Writer> writer = XdmfHDF5Writer::New("testHyperslab.h5");
+  writer->setMode(XdmfHeavyDataWriter::Hyperslab);
+  array1->accept(writer);
+
+  //
+  // write array2 to second part of dataset
+  //
+  shared_ptr<XdmfHDF5Controller> controller2 = 
+    XdmfHDF5Controller::New("testHyperslab.h5",
+                            "data",
+                            XdmfArrayType::Int32(),
+                            std::vector<unsigned int>(1, 2),
+                            std::vector<unsigned int>(1, 1),
+                            std::vector<unsigned int>(1, 2),
+                            std::vector<unsigned int>(1, 4));
+  array2->setHeavyDataController(controller2);
+  array2->accept(writer);
+
+  //
+  // assert we can release and read same data we wrote
+  //
+  array1->release();
+  array1->read();
+  assert(array1->getValuesString().compare("1 2") == 0);
+  array2->release();
+  array2->read();
+  std::cout << array2->getValuesString() << std::endl;
+  assert(array2->getValuesString().compare("3 4") == 0);
+}

core/tests/Cxx/TestXdmfHDF5Controller.cpp

@@ -11,17 +11,11 @@ int main(int, char **)
                             XdmfArrayType::Int8(),
                             std::vector<unsigned int>(1, 0),
                             std::vector<unsigned int>(1, 1),
+                            std::vector<unsigned int>(1, 10),
                             std::vector<unsigned int>(1, 10));
   assert(controller->getDataSetPath().compare("/foo/data1") == 0);
   assert(controller->getSize() == 10);
   assert(controller->getType() == XdmfArrayType::Int8());
 
-/*
-  shared_ptr<XdmfHDF5Controller> hyperslabController = XdmfHDF5Controller::New("DislocSegment_10x10x20_8K.h5", "/Attribute/Node/GlobalNodeId", XdmfArrayType::Int32(), std::vector<unsigned int>(1, 0), std::vector<unsigned int>(1, 1), std::vector<unsigned int>(1, 1533));
-  shared_ptr<XdmfArray> array = XdmfArray::New();
-  array->setHeavyDataController(hyperslabController);
-  array->read();
-  std::cout << array->getValuesString() << std::endl;
-*/
   return 0;
 }

core/tests/Cxx/TestXdmfHDF5WriterDSM.cpp

@@ -2,6 +2,7 @@
 #include <H5FDdsmManager.h>
 #include "XdmfArray.hpp"
 #include "XdmfArrayType.hpp"
+#include "XdmfHDF5ControllerDSM.hpp"
 #include "XdmfHDF5WriterDSM.hpp"
 
 int main(int argc, char *argv[])
@@ -22,16 +23,19 @@ int main(int argc, char *argv[])
       std::cout << "# MPI_THREAD_MULTIPLE is OK" << std::endl;
     }
   }
-
+  
   // Create DSM
   H5FDdsmManager * dsmManager = new H5FDdsmManager();
-  dsmManager->SetCommunicator(comm);
+  dsmManager->SetMpiComm(comm);
+  dsmManager->SetLocalBufferSizeMBytes(dsmSize / size);
   dsmManager->SetLocalBufferSizeMBytes(dsmSize / size);
-  dsmManager->CreateDSM();
+  dsmManager->SetIsStandAlone(H5FD_DSM_TRUE);
+  dsmManager->Create();
+  H5FD_dsm_set_manager(dsmManager);
 
-  H5FDdsmBuffer * dsmBuffer = dsmManager->GetDSMHandle();
+  H5FD_dsm_set_options(H5FD_DSM_LOCK_ASYNCHRONOUS);
 
-  H5FD_dsm_set_mode(H5FD_DSM_MANUAL_SERVER_UPDATE, dsmBuffer);
+  H5FDdsmBuffer * dsmBuffer = dsmManager->GetDsmBuffer();
 
   // Get info from remote server
   double remoteMB = dsmBuffer->GetTotalLength() / (1024.0 * 1024.0);
@@ -47,24 +51,54 @@ int main(int argc, char *argv[])
   // Array should be distributed among processes
   shared_ptr<XdmfArray> array = XdmfArray::New();
   array->initialize<int>(0);
-  array->pushBack(0);
-  array->pushBack(1);
-  array->pushBack(2);
+  array->pushBack(rank*3);
+  array->pushBack(rank*3 + 1);
+  array->pushBack(rank*3 + 2);
+  shared_ptr<XdmfHDF5ControllerDSM> controller = 
+    XdmfHDF5ControllerDSM::New("dsm.h5",
+                               "data",
+                               XdmfArrayType::Int32(),
+                               std::vector<unsigned int>(1, rank*3),
+                               std::vector<unsigned int>(1, 1),
+                               std::vector<unsigned int>(1, 3),
+                               std::vector<unsigned int>(1, size*3),
+                               NULL);
+  array->setHeavyDataController(controller);
 
   // Create DSM Writer and write to DSM space.
   shared_ptr<XdmfHDF5WriterDSM> writer =
-    XdmfHDF5WriterDSM::New("dsm", dsmBuffer);
+    XdmfHDF5WriterDSM::New("dsm.h5", NULL);
+  writer->setMode(XdmfHeavyDataWriter::Hyperslab);
   array->accept(writer);
 
-  // Read data
-  shared_ptr<XdmfArray> readArray = XdmfArray::New();
-  readArray->setHeavyDataController(array->getHeavyDataController());
-  readArray->read();
-  assert(readArray->getSize() == 3);
-  assert(readArray->getArrayType() == XdmfArrayType::Int32());
-
-  for(unsigned int i=0; i<readArray->getSize(); ++i) {
-    assert(array->getValue<int>(i) == readArray->getValue<int>(i));
+  //H5FD_dsm_dump();
+  
+  // Release data and read back in to check if we wrote correctly
+  array->release();
+  array->read();
+  assert(array->getSize() == 3);
+  assert(array->getValue<int>(0) == 3*rank);
+  assert(array->getValue<int>(1) == 3*rank + 1);
+  assert(array->getValue<int>(2) == 3*rank + 2);
+  
+  MPI_Barrier(comm);
+  
+  // Adjust controller to read entire dataset onto each processor
+  shared_ptr<XdmfHDF5ControllerDSM> fullController = 
+    XdmfHDF5ControllerDSM::New("dsm.h5",
+                               "data",
+                               XdmfArrayType::Int32(),
+                               std::vector<unsigned int>(1, 0),
+                               std::vector<unsigned int>(1, 1),
+                               std::vector<unsigned int>(1, size*3),
+                               std::vector<unsigned int>(1, size*3),