ENH: Initial commit of binary heavy data file support.

core/CMakeLists.txt

@@ -6,6 +6,7 @@ endif (POLICY CMP0015)
 
 include(CheckCXXSourceCompiles)
 include(SetUpVersion)
+include(TestBigEndian)
 
 if(VERSION_CONTROL_AUTOUPDATE OR 
     NOT EXISTS ${CMAKE_BINARY_DIR}/XdmfVersion.hpp)
@@ -76,6 +77,9 @@ if(LIBXML2_FOUND)
 endif(LIBXML2_FOUND)
 
 # Perform compile-time checks and generate XdmfConfig.hpp
+
+TEST_BIG_ENDIAN(XDMF_BIG_ENDIAN)
+
 unset(HAVE_BOOST_SHARED_DYNAMIC_CAST CACHE)
 set(CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES} ${Boost_INCLUDE_DIRS})
 check_cxx_source_compiles("
@@ -98,6 +102,7 @@ set(XdmfCoreSources
   XdmfArray
   XdmfArrayReference
   XdmfArrayType
+  XdmfBinaryController
   XdmfCoreItemFactory
   XdmfCoreReader
   XdmfError

core/XdmfArray.cpp

@@ -31,11 +31,36 @@
 #include "XdmfArray.hpp"
 #include "XdmfArrayType.hpp"
 #include "XdmfArrayReference.hpp"
+#include "XdmfBinaryController.hpp"
 #include "XdmfHDF5Controller.hpp"
 #include "XdmfHeavyDataController.hpp"
 #include "XdmfVisitor.hpp"
 #include "XdmfError.hpp"
 
+namespace {
+
+  std::string
+  getFullHeavyDataPath(const std::string & filePath,
+                       const std::map<std::string, std::string> & itemProperties)
+  {
+    // FIXME: for other OS (e.g. windows)
+    if(filePath.size() > 0 && filePath[0] != '/') {
+      // Dealing with a relative path for heavyData location
+      std::map<std::string, std::string>::const_iterator xmlDir =
+        itemProperties.find("XMLDir");
+      if(xmlDir == itemProperties.end()) {
+        XdmfError::message(XdmfError::FATAL,
+                           "'XMLDir' not found in itemProperties in "
+                           "XdmfArray::populateItem");
+      }
+      std::stringstream newHeavyDataPath;
+      newHeavyDataPath << xmlDir->second << filePath;
+      return newHeavyDataPath.str();
+    }
+    return filePath;
+  }
+  
+}
 
 XDMF_CHILDREN_IMPLEMENTATION(XdmfArray,
                              XdmfHeavyDataController,
@@ -524,14 +549,11 @@ public:
   unsigned int
   operator()(const boost::blank & array) const
   {
-    if(mArray->mHeavyDataControllers.size()>0) {
-      int total = 0;
-      for (unsigned int i = 0; i < mArray->mHeavyDataControllers.size(); ++i) {
-        total += mArray->mHeavyDataControllers[i]->getSize();
-      }
-      return total;
+    unsigned int total = 0;
+    for (unsigned int i = 0; i < mArray->mHeavyDataControllers.size(); ++i) {
+      total += mArray->mHeavyDataControllers[i]->getSize();
     }
-    return 0;
+    return total;
   }
 
   template<typename T>
@@ -659,9 +681,8 @@ std::map<std::string, std::string>
 XdmfArray::getItemProperties() const
 {
   std::map<std::string, std::string> arrayProperties;
-  if(mHeavyDataControllers.size()>0) {
-    arrayProperties.insert(std::make_pair("Format",
-                                          mHeavyDataControllers[0]->getName()));
+  if(mHeavyDataControllers.size() > 0) {
+    mHeavyDataControllers[0]->getProperties(arrayProperties);
   }
   else {
     arrayProperties.insert(std::make_pair("Format", "XML"));
@@ -738,7 +759,7 @@ XdmfArray::getHeavyDataController()
 shared_ptr<const XdmfHeavyDataController>
 XdmfArray::getHeavyDataController() const
 {
-  if (mHeavyDataControllers.size()>0) {
+  if (mHeavyDataControllers.size() > 0) {
     return mHeavyDataControllers[0];
   }
   else {
@@ -747,7 +768,7 @@ XdmfArray::getHeavyDataController() const
 }
 
 void
-XdmfArray::initialize(const shared_ptr<const XdmfArrayType> arrayType,
+XdmfArray::initialize(const shared_ptr<const XdmfArrayType> & arrayType,
                       const unsigned int size)
 {
   if(arrayType == XdmfArrayType::Int8()) {
@@ -795,7 +816,7 @@ XdmfArray::initialize(const shared_ptr<const XdmfArrayType> arrayType,
 }
 
 void
-XdmfArray::initialize(const shared_ptr<const XdmfArrayType> arrayType,
+XdmfArray::initialize(const shared_ptr<const XdmfArrayType> & arrayType,
                       const std::vector<unsigned int> & dimensions)
 {
   mDimensions = dimensions;
@@ -1065,7 +1086,6 @@ XdmfArray::populateItem(const std::map<std::string, std::string> & itemPropertie
   }
   const std::string & formatVal = format->second;
 
-
   if(formatVal.compare("HDF") == 0) {
     contentIndex = 0;
     int contentStep = 2;
@@ -1088,20 +1108,9 @@ XdmfArray::populateItem(const std::map<std::string, std::string> & itemPropertie
       std::string dataSetPath = 
         contentVals[contentIndex].substr(colonLocation+1);
 
-      // FIXME: for other OS (e.g. windows)
-      if(hdf5Path.size() > 0 && hdf5Path[0] != '/') {
-        // Dealing with a relative path for hdf5 location
-        std::map<std::string, std::string>::const_iterator xmlDir =
-          itemProperties.find("XMLDir");
-        if(xmlDir == itemProperties.end()) {
-          XdmfError::message(XdmfError::FATAL,
-                             "'XMLDir' not found in itemProperties in "
-                             "XdmfArray::populateItem");
-        }
-        std::stringstream newHDF5Path;
-        newHDF5Path << xmlDir->second << hdf5Path;
-        hdf5Path = newHDF5Path.str();
-      }
+
+      hdf5Path = getFullHeavyDataPath(hdf5Path,
+                                      itemProperties);
 
       // Parse dimensions from the content
       std::vector<unsigned int> contentDims;
@@ -1126,8 +1135,7 @@ XdmfArray::populateItem(const std::map<std::string, std::string> & itemPropertie
         contentStep = 1;
       }
 
-
-      mHeavyDataControllers.push_back( 
+      mHeavyDataControllers.push_back(
         XdmfHDF5Controller::New(hdf5Path,
                                 dataSetPath,
                                 arrayType,
@@ -1137,8 +1145,8 @@ XdmfArray::populateItem(const std::map<std::string, std::string> & itemPropertie
                                                           1),
                                 contentDims,
                                 contentDims)
-      );
-      contentIndex+=contentStep;
+        );
+      contentIndex += contentStep;
     }
   }
   else if(formatVal.compare("XML") == 0) {
@@ -1164,6 +1172,43 @@ XdmfArray::populateItem(const std::map<std::string, std::string> & itemPropertie
       }
     }
   }
+  else if(formatVal.compare("Binary") == 0) {
+
+    XdmfBinaryController::Endian endian = XdmfBinaryController::NATIVE;
+    std::map<std::string, std::string>::const_iterator endianIter =
+      itemProperties.find("Endian");
+    if(endianIter != itemProperties.end()) {
+      if(endianIter->second.compare("Big") == 0) {
+        endian =  XdmfBinaryController::BIG;
+      }
+      else if(endianIter->second.compare("Little") == 0) {
+        endian =  XdmfBinaryController::LITTLE;
+      }
+      else if(endianIter->second.compare("Native") == 0) {
+        endian =  XdmfBinaryController::NATIVE;
+      }
+      else {
+        XdmfError(XdmfError::FATAL,
+                  "Invalid endianness type: " + endianIter->second);
+      }
+    }
+
+    unsigned int seek = 0;
+    std::map<std::string, std::string>::const_iterator seekIter =
+      itemProperties.find("Seek");
+    if(seekIter != itemProperties.end()) {
+      seek = std::atoi(seekIter->second.c_str());
+    }
+
+    const std::string binaryPath = getFullHeavyDataPath(contentVals[0],
+                                                        itemProperties);
+
+    mHeavyDataControllers.push_back(XdmfBinaryController::New(binaryPath,
+                                                              arrayType,
+                                                              endian,
+                                                              seek,
+                                                              mDimensions));
+  }
   else {
     try {
       XdmfError::message(XdmfError::FATAL, 

core/XdmfArray.hpp

@@ -649,7 +649,7 @@ public:
    * @param     arrayType       The type of array to initialize.
    * @param     size            The number of values in the initialized array.
    */
-  void initialize(const shared_ptr<const XdmfArrayType> arrayType,
+  void initialize(const shared_ptr<const XdmfArrayType> & arrayType,
                   const unsigned int size = 0);
 
   /**
@@ -678,7 +678,7 @@ public:
    * @param     arrayType       The type of array to initialize.
    * @param     dimensions      The number dimensions of the initialized array.
    */
-  void initialize(const shared_ptr<const XdmfArrayType> arrayType,
+  void initialize(const shared_ptr<const XdmfArrayType> & arrayType,
                   const std::vector<unsigned int> & dimensions);
 
   using XdmfItem::insert;

core/XdmfBinaryController.cpp

+/*****************************************************************************/
+/*                                    Xdmf                                   */
+/*                       eXtensible Data Model and Format                    */
+/*                                                                           */
+/*  Id : XdmfBinaryController.cpp                                              */
+/*                                                                           */
+/*  Author:                                                                  */
+/*     Kenneth Leiter                                                        */
+/*     kenneth.leiter@arl.army.mil                                           */
+/*     US Army Research Laboratory                                           */
+/*     Aberdeen Proving Ground, MD                                           */
+/*                                                                           */
+/*     Copyright @ 2011 US Army Research Laboratory                          */
+/*     All Rights Reserved                                                   */
+/*     See Copyright.txt for details                                         */
+/*                                                                           */
+/*     This software is distributed WITHOUT ANY WARRANTY; without            */
+/*     even the implied warranty of MERCHANTABILITY or FITNESS               */
+/*     FOR A PARTICULAR PURPOSE.  See the above copyright notice             */
+/*     for more information.                                                 */
+/*                                                                           */
+/*****************************************************************************/
+
+#include <fstream>
+#include <sstream>
+#include "XdmfArray.hpp"
+#include "XdmfArrayType.hpp"
+#include "XdmfBinaryController.hpp"
+#include "XdmfError.hpp"
+
+namespace {
+
+  template<size_t T>
+  struct ByteSwaper {
+    static inline void swap(void * p){}
+    static inline void swap(void * p,
+                            unsigned int length)
+    {
+      char * data = static_cast<char *>(p);
+      for(unsigned int i=0; i<length; ++i, data+=T){
+        ByteSwaper<T>::swap(data);
+      }
+    }
+  };
+
+  template<>
+  void ByteSwaper<2>::swap(void * p){
+    char one_byte;
+    char* data = static_cast<char*>(p);
+    one_byte = data[0]; data[0] = data[1]; data[1] = one_byte;
+  };
+
+  template<>
+  void ByteSwaper<4>::swap(void * p){
+    char one_byte;
+    char* data = static_cast<char*>(p);
+    one_byte = data[0]; data[0] = data[3]; data[3] = one_byte;
+    one_byte = data[1]; data[1] = data[2]; data[2] = one_byte;
+  
+  };
+  template<>
+  void ByteSwaper<8>::swap(void * p){
+    char one_byte;
+    char* data = static_cast<char*>(p);
+    one_byte = data[0]; data[0] = data[7]; data[7] = one_byte;
+    one_byte = data[1]; data[1] = data[6]; data[6] = one_byte;
+    one_byte = data[2]; data[2] = data[5]; data[5] = one_byte;
+    one_byte = data[3]; data[3] = data[4]; data[4] = one_byte;
+  };
+
+}
+
+shared_ptr<XdmfBinaryController>
+XdmfBinaryController::New(const std::string & filePath,
+                          const shared_ptr<const XdmfArrayType> & type,
+                          const Endian & endian,
+                          const unsigned int seek,
+                          const std::vector<unsigned int> & dimensions)
+{
+  try {
+    shared_ptr<XdmfBinaryController> 
+      p(new XdmfBinaryController(filePath,
+                                 type,
+                                 endian,
+                                 seek,
+                                 dimensions));
+    return p;
+  }
+  catch (XdmfError e) {
+    throw e;
+  }
+}
+
+XdmfBinaryController::XdmfBinaryController(const std::string & filePath,
+                                           const shared_ptr<const XdmfArrayType> & type,
+                                           const Endian & endian,
+                                           const unsigned int seek,
+                                           const std::vector<unsigned int> & dimensions) :
+  XdmfHeavyDataController(filePath,
+                          "",
+                          type,
+                          dimensions),
+  mEndian(endian),
+  mSeek(seek)
+{
+}
+
+XdmfBinaryController::~XdmfBinaryController()
+{
+}
+
+XdmfBinaryController::Endian
+XdmfBinaryController::getEndian() const
+{
+  return mEndian;
+}
+
+std::string
+XdmfBinaryController::getName() const
+{
+  return "Binary";
+}
+
+void
+XdmfBinaryController::getProperties(std::map<std::string, std::string> & collectedProperties) const
+{
+  collectedProperties["Format"] = this->getName();
+  std::stringstream seekStream;
+  seekStream << mSeek;
+  collectedProperties["Seek"] = seekStream.str();
+  if(mEndian == BIG) {
+    collectedProperties["Endian"] = "Big";
+  }
+  else if(mEndian == LITTLE) {
+    collectedProperties["Endian"] = "Little";
+  }
+}
+
+unsigned int
+XdmfBinaryController::getSeek() const
+{
+  return mSeek;
+}
+
+void
+XdmfBinaryController::read(XdmfArray * const array)
+{
+  array->initialize(mType, mDimensions);
+
+  std::ifstream fileStream(mFilePath.c_str(),
+                           std::ifstream::binary);
+
+  if(!fileStream.good()) {
+    XdmfError::message(XdmfError::FATAL,
+                       "Error reading " + mFilePath + 
+                       " in XdmfBinaryController::read");
+  }
+
+  fileStream.seekg(mSeek);
+  
+  if(!fileStream.good()) {
+    XdmfError::message(XdmfError::FATAL,
+                       "Error seeking " + mFilePath + 
+                       " in XdmfBinaryController::read");
+  }
+  
+  fileStream.read(static_cast<char *>(array->getValuesInternal()),
+                  array->getSize() * mType->getElementSize());
+  
+#if defined(XDMF_BIG_ENDIAN)
+  const bool needByteSwap = mEndian == LITTLE;
+#else
+  const bool needByteSwap = mEndian == BIG;
+#endif // XDMF_BIG_ENDIAN
+  
+  if(needByteSwap) {
+    switch(mType->getElementSize()){
+    case 1:
+      break;
+    case 2:
+      ByteSwaper<2>::swap(array->getValuesInternal(),
+                          array->getSize());
+        break;
+    case 4:
+      ByteSwaper<4>::swap(array->getValuesInternal(),
+                          array->getSize());
+      break;
+    case 8:
+      ByteSwaper<8>::swap(array->getValuesInternal(),
+                          array->getSize());
+      break;
+    default:
+      XdmfError::message(XdmfError::FATAL,
+                         "Cannot perform endianness swap for datatype");
+      break;
+    }
+  }
+
+}

core/XdmfBinaryController.hpp

+/*****************************************************************************/
+/*                                    XDMF                                   */
+/*                       eXtensible Data Model and Format                    */
+/*                                                                           */
+/*  Id : XdmfBinaryController.hpp                                              */
+/*                                                                           */
+/*  Author:                                                                  */
+/*     Kenneth Leiter                                                        */
+/*     kenneth.leiter@arl.army.mil                                           */
+/*     US Army Research Laboratory                                           */
+/*     Aberdeen Proving Ground, MD                                           */
+/*                                                                           */
+/*     Copyright @ 2011 US Army Research Laboratory                          */
+/*     All Rights Reserved                                                   */
+/*     See Copyright.txt for details                                         */
+/*                                                                           */
+/*     This software is distributed WITHOUT ANY WARRANTY; without            */
+/*     even the implied warranty of MERCHANTABILITY or FITNESS               */
+/*     FOR A PARTICULAR PURPOSE.  See the above copyright notice             */
+/*     for more information.                                                 */
+/*                                                                           */
+/*****************************************************************************/
+
+#ifndef XDMFBinaryCONTROLLER_HPP_
+#define XDMFBinaryCONTROLLER_HPP_
+
+// Includes
+#include "XdmfCore.hpp"
+#include "XdmfHeavyDataController.hpp"
+
+/**
+ * @brief Couples an XdmfArray with Binary data stored on disk.
+ *
+ * Serves as an interface between data stored in XdmfArrays and data
+ * stored in binary files on disk. When an Xdmf file is read from or
+ * written to disk an XdmfBinaryController is attached to
+ * XdmfArrays. This allows data to be released from memory but still
+ * be accessible or have its location written to light data.
+ */
+class XDMFCORE_EXPORT XdmfBinaryController : public XdmfHeavyDataController {
+
+public:
+
+  typedef enum Endian {
+    BIG,
+    LITTLE,
+    NATIVE
+  } Endian;
+
+  virtual ~XdmfBinaryController();
+
+  /**
+   * Create a new controller for an binary data set on disk.
+   *
+   * @param filePath the location of the binary file.
+   * @param type the data type of the dataset to read.
+   * @param endian the endianness of the data.
+   * @param seek in bytes to begin reading in file.
+   * @param dimensions the number of elements to select in each
+   * dimension from the hdf5 data set. (size in each dimension)
+   *
+   * @return New Binary Controller.
+   */
+  static shared_ptr<XdmfBinaryController>
+  New(const std::string & filePath,
+      const shared_ptr<const XdmfArrayType> & type,
+      const Endian & endian,
+      const unsigned int seek,
+      const std::vector<unsigned int> & dimensions);
+
+  virtual Endian getEndian() const;
+
+  virtual std::string getName() const;
+
+  virtual void 
+  getProperties(std::map<std::string, std::string> & collectedProperties) const;
+
+  virtual unsigned int getSeek() const;
+
+  virtual void read(XdmfArray * const array);
+
+protected:
+
+  XdmfBinaryController(const std::string & filePath,
+                       const shared_ptr<const XdmfArrayType> & type,
+                       const Endian & endian,
+                       const unsigned int seek,
+                       const std::vector<unsigned int> & dimensions);
+
+private:
+
+  XdmfBinaryController(const XdmfBinaryController &);  // Not implemented.
+  void operator=(const XdmfBinaryController &);  // Not implemented.
+
+  const Endian mEndian;
+  const unsigned int mSeek;
+
+};
+
+#endif /* XDMFBinaryCONTROLLER_HPP_ */

core/XdmfConfig.hpp.in

@@ -25,5 +25,6 @@
 #define XDMFCONFIG_HPP_
 
 #cmakedefine HAVE_BOOST_SHARED_DYNAMIC_CAST
+#cmakedefine XDMF_BIG_ENDIAN
 
 #endif /* XDMFSHAREDPTR_HPP_ */

core/XdmfHDF5Controller.cpp

@@ -36,7 +36,7 @@ std::map<std::string, unsigned int> XdmfHDF5Controller::mOpenFileUsage;
 shared_ptr<XdmfHDF5Controller>
 XdmfHDF5Controller::New(const std::string & hdf5FilePath,
                         const std::string & dataSetPath,
-                        const shared_ptr<const XdmfArrayType> type,
+                        const shared_ptr<const XdmfArrayType> & type,
                         const std::vector<unsigned int> & start,
                         const std::vector<unsigned int> & stride,
                         const std::vector<unsigned int> & dimensions,
@@ -60,7 +60,7 @@ XdmfHDF5Controller::New(const std::string & hdf5FilePath,
 
 XdmfHDF5Controller::XdmfHDF5Controller(const std::string & hdf5FilePath,
                                        const std::string & dataSetPath,
-                                       const shared_ptr<const XdmfArrayType> type,
+                                       const shared_ptr<const XdmfArrayType> & type,
                                        const std::vector<unsigned int> & start,
                                        const std::vector<unsigned int> & stride,
                                        const std::vector<unsigned int> & dimensions,
@@ -68,11 +68,24 @@ XdmfHDF5Controller::XdmfHDF5Controller(const std::string & hdf5FilePath,
   XdmfHeavyDataController(hdf5FilePath,
                           dataSetPath,
                           type,
-                          start,
-                          stride,
-                          dimensions,
-                          dataspaceDimensions)
+                          dimensions),
+  mDataspaceDimensions(dataspaceDimensions),
+  mStart(start),
+  mStride(stride)
 {
+  if(!(mStart.size() == mStride.size() && 
+       mStride.size() == mDimensions.size() &&
+       mDimensions.size() == mDataspaceDimensions.size())) {
+    try {
+      XdmfError::message(XdmfError::FATAL,
+                         "mStart, mStride, mDimensions, and "
+                         "mDataSpaceDimensions must all be of equal length in "
+                         "XdmfHDF5Controller constructor");
+    }
+    catch (XdmfError e) {
+      throw e;
+    }
+  }
 }
 
 XdmfHDF5Controller::~XdmfHDF5Controller()
@@ -91,6 +104,12 @@ XdmfHDF5Controller::closeFiles()
   mOpenFileUsage.clear();
 }
 
+std::vector<unsigned int> 
+XdmfHDF5Controller::getDataspaceDimensions() const
+{
+  return mDataspaceDimensions;
+}
+
 std::string
 XdmfHDF5Controller::getName() const
 {
@@ -103,6 +122,24 @@ XdmfHDF5Controller::getMaxOpenedFiles()
   return XdmfHDF5Controller::mMaxOpenedFiles;
 }
 
+void 
+XdmfHDF5Controller::getProperties(std::map<std::string, std::string> & collectedProperties) const
+{
+  collectedProperties["Format"] = this->getName();
+}
+
+std::vector<unsigned int> 
+XdmfHDF5Controller::getStart() const
+{
+  return mStart;
+}
+
+std::vector<unsigned int> 
+XdmfHDF5Controller::getStride() const
+{
+  return mStride;
+}
+
 void
 XdmfHDF5Controller::read(XdmfArray * const array)
 {