XdmfHDF5Controller.cpp 10.8 KB
Newer Older
Kenneth Leiter's avatar
Kenneth Leiter committed
1
/*****************************************************************************/
2
/*                                    Xdmf                                   */
Kenneth Leiter's avatar
Kenneth Leiter committed
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
/*                       eXtensible Data Model and Format                    */
/*                                                                           */
/*  Id : XdmfHDF5Controller.cpp                                              */
/*                                                                           */
/*  Author:                                                                  */
/*     Kenneth Leiter                                                        */
/*     kenneth.leiter@arl.army.mil                                           */
/*     US Army Research Laboratory                                           */
/*     Aberdeen Proving Ground, MD                                           */
/*                                                                           */
/*     Copyright @ 2011 US Army Research Laboratory                          */
/*     All Rights Reserved                                                   */
/*     See Copyright.txt for details                                         */
/*                                                                           */
/*     This software is distributed WITHOUT ANY WARRANTY; without            */
/*     even the implied warranty of MERCHANTABILITY or FITNESS               */
/*     FOR A PARTICULAR PURPOSE.  See the above copyright notice             */
/*     for more information.                                                 */
/*                                                                           */
/*****************************************************************************/
23

24
#include <hdf5.h>
25
#include <numeric>
26
27
#include <sstream>
#include "XdmfArray.hpp"
28
#include "XdmfArrayType.hpp"
29
#include "XdmfError.hpp"
30
#include "XdmfHDF5Controller.hpp"
31
#include "XdmfSystemUtils.hpp"
32

33
34
35
36
unsigned int XdmfHDF5Controller::mMaxOpenedFiles = 0;
static std::map<std::string, hid_t> mOpenFiles;
std::map<std::string, unsigned int> XdmfHDF5Controller::mOpenFileUsage;

37
shared_ptr<XdmfHDF5Controller>
38
39
XdmfHDF5Controller::New(const std::string & hdf5FilePath,
                        const std::string & dataSetPath,
40
                        const shared_ptr<const XdmfArrayType> & type,
41
42
                        const std::vector<unsigned int> & start,
                        const std::vector<unsigned int> & stride,
43
44
                        const std::vector<unsigned int> & dimensions,
                        const std::vector<unsigned int> & dataspaceDimensions)
45
{
46
47
48
49
50
51
52
53
54
  shared_ptr<XdmfHDF5Controller> 
    p(new XdmfHDF5Controller(hdf5FilePath,
                             dataSetPath,
                             type,
                             start,
                             stride,
                             dimensions,
                             dataspaceDimensions));
  return p;
55
56
}

57
58
XdmfHDF5Controller::XdmfHDF5Controller(const std::string & hdf5FilePath,
                                       const std::string & dataSetPath,
59
                                       const shared_ptr<const XdmfArrayType> & type,
60
61
                                       const std::vector<unsigned int> & start,
                                       const std::vector<unsigned int> & stride,
62
63
                                       const std::vector<unsigned int> & dimensions,
                                       const std::vector<unsigned int> & dataspaceDimensions) :
64
65
66
  XdmfHeavyDataController(hdf5FilePath,
                          dataSetPath,
                          type,
67
68
69
70
                          dimensions),
  mDataspaceDimensions(dataspaceDimensions),
  mStart(start),
  mStride(stride)
71
{
72
73
74
  if(!(mStart.size() == mStride.size() && 
       mStride.size() == mDimensions.size() &&
       mDimensions.size() == mDataspaceDimensions.size())) {
75
76
77
78
    XdmfError::message(XdmfError::FATAL,
                       "mStart, mStride, mDimensions, and "
                       "mDataSpaceDimensions must all be of equal length in "
                       "XdmfHDF5Controller constructor");
79
  }
80
81
82
83
84
85
}

XdmfHDF5Controller::~XdmfHDF5Controller()
{
}

86
87
88
89
90
91
92
93
94
95
96
97
void
XdmfHDF5Controller::closeFiles()
{
  for (std::map<std::string, hid_t>::iterator closeIter = mOpenFiles.begin();
       closeIter != mOpenFiles.end();
       ++closeIter) {
    H5Fclose(closeIter->second);
  }
  mOpenFiles.clear();
  mOpenFileUsage.clear();
}

98
99
100
101
102
103
std::vector<unsigned int> 
XdmfHDF5Controller::getDataspaceDimensions() const
{
  return mDataspaceDimensions;
}

104
105
std::string
XdmfHDF5Controller::getName() const
106
{
107
  return "HDF";
108
}
109

110
111
112
113
114
115
unsigned int
XdmfHDF5Controller::getMaxOpenedFiles()
{
  return XdmfHDF5Controller::mMaxOpenedFiles;
}

116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
void 
XdmfHDF5Controller::getProperties(std::map<std::string, std::string> & collectedProperties) const
{
  collectedProperties["Format"] = this->getName();
}

std::vector<unsigned int> 
XdmfHDF5Controller::getStart() const
{
  return mStart;
}

std::vector<unsigned int> 
XdmfHDF5Controller::getStride() const
{
  return mStride;
}

134
135
void
XdmfHDF5Controller::read(XdmfArray * const array)
136
{
137
  this->read(array, H5P_DEFAULT);
138
139
}

140
141
void
XdmfHDF5Controller::read(XdmfArray * const array, const int fapl)
142
{
143
  herr_t status;
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
  hid_t hdf5Handle;
  if (XdmfHDF5Controller::mMaxOpenedFiles == 0) {
    hdf5Handle = H5Fopen(mFilePath.c_str(), H5F_ACC_RDONLY, fapl);
  }
  else {
    std::map<std::string, hid_t>::iterator checkOpen = mOpenFiles.find(mFilePath);
    if (checkOpen == mOpenFiles.end()) {
      // If the number of open files would become larger than allowed
      if (mOpenFiles.size() + 1 > mMaxOpenedFiles) {
        // Close least used one
        std::map<std::string, unsigned int>::iterator walker = mOpenFileUsage.begin();
        std::string oldestFile = walker->first;
        while (walker != mOpenFileUsage.end()) {
          // We want the file with the fewest accesses
          // If two are tied, we use the older one
          if (mOpenFileUsage[oldestFile] > walker->second) {
            oldestFile = walker->first;
          }
          ++walker;
        }
        status = H5Fclose(mOpenFiles[oldestFile]);
        mOpenFiles.erase(oldestFile);
        mOpenFileUsage.erase(oldestFile);
      }
      hdf5Handle = H5Fopen(mFilePath.c_str(), H5F_ACC_RDONLY, fapl);
      mOpenFiles[mFilePath] = hdf5Handle;
      mOpenFileUsage[mFilePath] = 1;
    }
    else {
      hdf5Handle = checkOpen->second;
      mOpenFileUsage[mFilePath]++;
    }
  }
177

178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
  const hid_t dataset = H5Dopen(hdf5Handle, mDataSetPath.c_str(), H5P_DEFAULT);
  const hid_t dataspace = H5Dget_space(dataset);

  const unsigned int dataspaceDims = H5Sget_simple_extent_ndims(dataspace);
  const std::vector<hsize_t> count(mDimensions.begin(), mDimensions.end());

  if(dataspaceDims != mDimensions.size()) {
    // special case where the number of dimensions of the hdf5 dataset
    // does not equal the number of dimensions in the light data
    // description - in this case we cannot properly take a hyperslab
    // selection, so we assume we are reading the entire dataset and
    // check whether that is ok to do
    const int numberValuesHDF5 = H5Sget_select_npoints(dataspace);
    const int numberValuesXdmf = 
      std::accumulate(mDimensions.begin(),
                      mDimensions.end(),
                      1,
                      std::multiplies<unsigned int>());
    if(numberValuesHDF5 != numberValuesXdmf) {
      XdmfError::message(XdmfError::FATAL,
                         "Number of dimensions in light data description in "
                         "Xdmf does not match number of dimensions in hdf5 "
                         "file.");
    }
  }
  else {
    const std::vector<hsize_t> start(mStart.begin(), mStart.end());
    const std::vector<hsize_t> stride(mStride.begin(), mStride.end());

    status = H5Sselect_hyperslab(dataspace,
                                 H5S_SELECT_SET,
                                 &start[0],
                                 &stride[0],
                                 &count[0],
                                 NULL);
  }
214

215
  const hssize_t numVals = H5Sget_select_npoints(dataspace);
216
217
218
  hid_t memspace = H5Screate_simple(mDimensions.size(),
                                    &count[0],
                                    NULL);
219
220

  /* status = H5Sselect_hyperslab(memspace,
221
222
223
224
225
     H5S_SELECT_SET,
     &memStart[0],
     &memStride[0],
     &memCount[0],
     NULL);*/
226

227
  hid_t datatype = H5T_NO_CLASS;
Kenneth Leiter's avatar
Kenneth Leiter committed
228
  bool closeDatatype = false;
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
  if(mType == XdmfArrayType::Int8()) {
    datatype = H5T_NATIVE_CHAR;
  }
  else if(mType == XdmfArrayType::Int16()) {
    datatype = H5T_NATIVE_SHORT;
  }
  else if(mType == XdmfArrayType::Int32()) {
    datatype = H5T_NATIVE_INT;
  }
  else if(mType == XdmfArrayType::Int64()) {
    datatype = H5T_NATIVE_LONG;
  }
  else if(mType == XdmfArrayType::Float32()) {
    datatype = H5T_NATIVE_FLOAT;
  }
  else if(mType == XdmfArrayType::Float64()) {
    datatype = H5T_NATIVE_DOUBLE;
  }
  else if(mType == XdmfArrayType::UInt8()) {
    datatype = H5T_NATIVE_UCHAR;
  }
  else if(mType == XdmfArrayType::UInt16()) {
    datatype = H5T_NATIVE_USHORT;
  }
  else if(mType == XdmfArrayType::UInt32()) {
    datatype = H5T_NATIVE_UINT;
  }
Kenneth Leiter's avatar
Kenneth Leiter committed
256
257
258
259
260
  else if(mType == XdmfArrayType::String()) {
    datatype = H5Tcopy(H5T_C_S1);
    H5Tset_size(datatype, H5T_VARIABLE);
    closeDatatype = true;
  }
261
  else {
262
263
264
    XdmfError::message(XdmfError::FATAL,
                       "Unknown XdmfArrayType encountered in hdf5 "
                       "controller.");
265
  }
266

267
268
  array->initialize(mType, mDimensions);

269
  if(numVals != array->getSize()) {
270
271
272
273
274
    std::stringstream errOut;
    errOut << "Number of values in hdf5 dataset (" << numVals;
    errOut << ")\ndoes not match allocated size in XdmfArray (" << array->getSize() << ").";
    XdmfError::message(XdmfError::FATAL,
                       errOut.str());
275
  }
Kenneth Leiter's avatar
Kenneth Leiter committed
276
277
  if(closeDatatype) {
    char ** data = new char*[numVals];
278
279
    status = H5Dread(dataset,
                     datatype,
Kenneth Leiter's avatar
Kenneth Leiter committed
280
281
                     memspace,
                     dataspace,
282
                     H5P_DEFAULT,
Kenneth Leiter's avatar
Kenneth Leiter committed
283
284
285
286
                     data);
    for(hssize_t i=0; i<numVals; ++i) {
      array->insert<std::string>(i, data[i]);
    }
287
288
289
    status = H5Dvlen_reclaim(datatype,
                             dataspace,
                             H5P_DEFAULT,
Kenneth Leiter's avatar
Kenneth Leiter committed
290
291
292
293
294
295
296
297
298
299
300
                             data);
    delete [] data;
  }
  else {
    status = H5Dread(dataset,
                     datatype,
                     memspace,
                     dataspace,
                     H5P_DEFAULT,
                     array->getValuesInternal());
  }
301

302
  status = H5Sclose(dataspace);
303
  status = H5Sclose(memspace);
304
  status = H5Dclose(dataset);
Kenneth Leiter's avatar
Kenneth Leiter committed
305
306
307
  if(closeDatatype) {
    status = H5Tclose(datatype);
  }
308
309
310
311
312
313
314
315
316
  if (XdmfHDF5Controller::mMaxOpenedFiles == 0) {
    status = H5Fclose(hdf5Handle);
  }
}

void
XdmfHDF5Controller::setMaxOpenedFiles(unsigned int newMax)
{
  XdmfHDF5Controller::mMaxOpenedFiles = newMax;
317
}