XdmfHDF5Controller.cpp 10.8 KB
Newer Older
Kenneth Leiter's avatar
Kenneth Leiter committed
1
/*****************************************************************************/
2
/*                                    Xdmf                                   */
Kenneth Leiter's avatar
Kenneth Leiter committed
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
/*                       eXtensible Data Model and Format                    */
/*                                                                           */
/*  Id : XdmfHDF5Controller.cpp                                              */
/*                                                                           */
/*  Author:                                                                  */
/*     Kenneth Leiter                                                        */
/*     kenneth.leiter@arl.army.mil                                           */
/*     US Army Research Laboratory                                           */
/*     Aberdeen Proving Ground, MD                                           */
/*                                                                           */
/*     Copyright @ 2011 US Army Research Laboratory                          */
/*     All Rights Reserved                                                   */
/*     See Copyright.txt for details                                         */
/*                                                                           */
/*     This software is distributed WITHOUT ANY WARRANTY; without            */
/*     even the implied warranty of MERCHANTABILITY or FITNESS               */
/*     FOR A PARTICULAR PURPOSE.  See the above copyright notice             */
/*     for more information.                                                 */
/*                                                                           */
/*****************************************************************************/
23

24
#include <hdf5.h>
25
#include <numeric>
26 27
#include <sstream>
#include "XdmfArray.hpp"
28
#include "XdmfArrayType.hpp"
29
#include "XdmfError.hpp"
30
#include "XdmfHDF5Controller.hpp"
31
#include "XdmfSystemUtils.hpp"
32

33 34 35 36
unsigned int XdmfHDF5Controller::mMaxOpenedFiles = 0;
static std::map<std::string, hid_t> mOpenFiles;
std::map<std::string, unsigned int> XdmfHDF5Controller::mOpenFileUsage;

37
shared_ptr<XdmfHDF5Controller>
38 39
XdmfHDF5Controller::New(const std::string & hdf5FilePath,
                        const std::string & dataSetPath,
40
                        const shared_ptr<const XdmfArrayType> & type,
41 42
                        const std::vector<unsigned int> & start,
                        const std::vector<unsigned int> & stride,
43 44
                        const std::vector<unsigned int> & dimensions,
                        const std::vector<unsigned int> & dataspaceDimensions)
45
{
46 47 48 49 50 51 52 53 54
  shared_ptr<XdmfHDF5Controller> 
    p(new XdmfHDF5Controller(hdf5FilePath,
                             dataSetPath,
                             type,
                             start,
                             stride,
                             dimensions,
                             dataspaceDimensions));
  return p;
55 56
}

57 58
XdmfHDF5Controller::XdmfHDF5Controller(const std::string & hdf5FilePath,
                                       const std::string & dataSetPath,
59
                                       const shared_ptr<const XdmfArrayType> & type,
60 61
                                       const std::vector<unsigned int> & start,
                                       const std::vector<unsigned int> & stride,
62 63
                                       const std::vector<unsigned int> & dimensions,
                                       const std::vector<unsigned int> & dataspaceDimensions) :
64 65 66
  XdmfHeavyDataController(hdf5FilePath,
                          dataSetPath,
                          type,
67 68 69 70
                          dimensions),
  mDataspaceDimensions(dataspaceDimensions),
  mStart(start),
  mStride(stride)
71
{
72 73 74
  if(!(mStart.size() == mStride.size() && 
       mStride.size() == mDimensions.size() &&
       mDimensions.size() == mDataspaceDimensions.size())) {
75 76 77 78
    XdmfError::message(XdmfError::FATAL,
                       "mStart, mStride, mDimensions, and "
                       "mDataSpaceDimensions must all be of equal length in "
                       "XdmfHDF5Controller constructor");
79
  }
80 81 82 83 84 85
}

XdmfHDF5Controller::~XdmfHDF5Controller()
{
}

86 87 88 89 90 91 92 93 94 95 96 97
void
XdmfHDF5Controller::closeFiles()
{
  for (std::map<std::string, hid_t>::iterator closeIter = mOpenFiles.begin();
       closeIter != mOpenFiles.end();
       ++closeIter) {
    H5Fclose(closeIter->second);
  }
  mOpenFiles.clear();
  mOpenFileUsage.clear();
}

98 99 100 101 102 103
std::vector<unsigned int> 
XdmfHDF5Controller::getDataspaceDimensions() const
{
  return mDataspaceDimensions;
}

104 105
std::string
XdmfHDF5Controller::getName() const
106
{
107
  return "HDF";
108
}
109

110 111 112 113 114 115
unsigned int
XdmfHDF5Controller::getMaxOpenedFiles()
{
  return XdmfHDF5Controller::mMaxOpenedFiles;
}

116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133
void 
XdmfHDF5Controller::getProperties(std::map<std::string, std::string> & collectedProperties) const
{
  collectedProperties["Format"] = this->getName();
}

std::vector<unsigned int> 
XdmfHDF5Controller::getStart() const
{
  return mStart;
}

std::vector<unsigned int> 
XdmfHDF5Controller::getStride() const
{
  return mStride;
}

134 135
void
XdmfHDF5Controller::read(XdmfArray * const array)
136
{
137
  this->read(array, H5P_DEFAULT);
138 139
}

140 141
void
XdmfHDF5Controller::read(XdmfArray * const array, const int fapl)
142
{
143
  herr_t status;
144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
  hid_t hdf5Handle;
  if (XdmfHDF5Controller::mMaxOpenedFiles == 0) {
    hdf5Handle = H5Fopen(mFilePath.c_str(), H5F_ACC_RDONLY, fapl);
  }
  else {
    std::map<std::string, hid_t>::iterator checkOpen = mOpenFiles.find(mFilePath);
    if (checkOpen == mOpenFiles.end()) {
      // If the number of open files would become larger than allowed
      if (mOpenFiles.size() + 1 > mMaxOpenedFiles) {
        // Close least used one
        std::map<std::string, unsigned int>::iterator walker = mOpenFileUsage.begin();
        std::string oldestFile = walker->first;
        while (walker != mOpenFileUsage.end()) {
          // We want the file with the fewest accesses
          // If two are tied, we use the older one
          if (mOpenFileUsage[oldestFile] > walker->second) {
            oldestFile = walker->first;
          }
          ++walker;
        }
        status = H5Fclose(mOpenFiles[oldestFile]);
        mOpenFiles.erase(oldestFile);
        mOpenFileUsage.erase(oldestFile);
      }
      hdf5Handle = H5Fopen(mFilePath.c_str(), H5F_ACC_RDONLY, fapl);
      mOpenFiles[mFilePath] = hdf5Handle;
      mOpenFileUsage[mFilePath] = 1;
    }
    else {
      hdf5Handle = checkOpen->second;
      mOpenFileUsage[mFilePath]++;
    }
  }
177

178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213
  const hid_t dataset = H5Dopen(hdf5Handle, mDataSetPath.c_str(), H5P_DEFAULT);
  const hid_t dataspace = H5Dget_space(dataset);

  const unsigned int dataspaceDims = H5Sget_simple_extent_ndims(dataspace);
  const std::vector<hsize_t> count(mDimensions.begin(), mDimensions.end());

  if(dataspaceDims != mDimensions.size()) {
    // special case where the number of dimensions of the hdf5 dataset
    // does not equal the number of dimensions in the light data
    // description - in this case we cannot properly take a hyperslab
    // selection, so we assume we are reading the entire dataset and
    // check whether that is ok to do
    const int numberValuesHDF5 = H5Sget_select_npoints(dataspace);
    const int numberValuesXdmf = 
      std::accumulate(mDimensions.begin(),
                      mDimensions.end(),
                      1,
                      std::multiplies<unsigned int>());
    if(numberValuesHDF5 != numberValuesXdmf) {
      XdmfError::message(XdmfError::FATAL,
                         "Number of dimensions in light data description in "
                         "Xdmf does not match number of dimensions in hdf5 "
                         "file.");
    }
  }
  else {
    const std::vector<hsize_t> start(mStart.begin(), mStart.end());
    const std::vector<hsize_t> stride(mStride.begin(), mStride.end());

    status = H5Sselect_hyperslab(dataspace,
                                 H5S_SELECT_SET,
                                 &start[0],
                                 &stride[0],
                                 &count[0],
                                 NULL);
  }
214

215
  const hssize_t numVals = H5Sget_select_npoints(dataspace);
216 217 218
  hid_t memspace = H5Screate_simple(mDimensions.size(),
                                    &count[0],
                                    NULL);
219 220

  /* status = H5Sselect_hyperslab(memspace,
221 222 223 224 225
     H5S_SELECT_SET,
     &memStart[0],
     &memStride[0],
     &memCount[0],
     NULL);*/
226

227
  hid_t datatype = H5T_NO_CLASS;
228
  bool closeDatatype = false;
229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
  if(mType == XdmfArrayType::Int8()) {
    datatype = H5T_NATIVE_CHAR;
  }
  else if(mType == XdmfArrayType::Int16()) {
    datatype = H5T_NATIVE_SHORT;
  }
  else if(mType == XdmfArrayType::Int32()) {
    datatype = H5T_NATIVE_INT;
  }
  else if(mType == XdmfArrayType::Int64()) {
    datatype = H5T_NATIVE_LONG;
  }
  else if(mType == XdmfArrayType::Float32()) {
    datatype = H5T_NATIVE_FLOAT;
  }
  else if(mType == XdmfArrayType::Float64()) {
    datatype = H5T_NATIVE_DOUBLE;
  }
  else if(mType == XdmfArrayType::UInt8()) {
    datatype = H5T_NATIVE_UCHAR;
  }
  else if(mType == XdmfArrayType::UInt16()) {
    datatype = H5T_NATIVE_USHORT;
  }
  else if(mType == XdmfArrayType::UInt32()) {
    datatype = H5T_NATIVE_UINT;
  }
256 257 258 259 260
  else if(mType == XdmfArrayType::String()) {
    datatype = H5Tcopy(H5T_C_S1);
    H5Tset_size(datatype, H5T_VARIABLE);
    closeDatatype = true;
  }
261
  else {
262 263 264
    XdmfError::message(XdmfError::FATAL,
                       "Unknown XdmfArrayType encountered in hdf5 "
                       "controller.");
265
  }
266

267 268
  array->initialize(mType, mDimensions);

269
  if(numVals != array->getSize()) {
270 271 272 273 274
    std::stringstream errOut;
    errOut << "Number of values in hdf5 dataset (" << numVals;
    errOut << ")\ndoes not match allocated size in XdmfArray (" << array->getSize() << ").";
    XdmfError::message(XdmfError::FATAL,
                       errOut.str());
275
  }
276 277
  if(closeDatatype) {
    char ** data = new char*[numVals];
278 279
    status = H5Dread(dataset,
                     datatype,
280 281
                     memspace,
                     dataspace,
282
                     H5P_DEFAULT,
283 284 285 286
                     data);
    for(hssize_t i=0; i<numVals; ++i) {
      array->insert<std::string>(i, data[i]);
    }
287 288 289
    status = H5Dvlen_reclaim(datatype,
                             dataspace,
                             H5P_DEFAULT,
290 291 292 293 294 295 296 297 298 299 300
                             data);
    delete [] data;
  }
  else {
    status = H5Dread(dataset,
                     datatype,
                     memspace,
                     dataspace,
                     H5P_DEFAULT,
                     array->getValuesInternal());
  }
301

302
  status = H5Sclose(dataspace);
303
  status = H5Sclose(memspace);
304
  status = H5Dclose(dataset);
305 306 307
  if(closeDatatype) {
    status = H5Tclose(datatype);
  }
308 309 310 311 312 313 314 315 316
  if (XdmfHDF5Controller::mMaxOpenedFiles == 0) {
    status = H5Fclose(hdf5Handle);
  }
}

void
XdmfHDF5Controller::setMaxOpenedFiles(unsigned int newMax)
{
  XdmfHDF5Controller::mMaxOpenedFiles = newMax;
317
}