Commit dca545e1 authored by Kenneth Leiter's avatar Kenneth Leiter

ENH: Modify HDF5Controllers to accept start, stride, and count to pull...

ENH: Modify HDF5Controllers to accept start, stride, and count to pull hyperslabs from hdf5 data sets.
parent ae80f9d7
......@@ -663,10 +663,13 @@ XdmfArray::populateItem(const std::map<std::string, std::string> & itemPropertie
newHDF5Path << xmlDir->second << hdf5Path;
hdf5Path = newHDF5Path.str();
}
mHeavyDataController = XdmfHDF5Controller::New(hdf5Path,
dataSetPath,
sizeVal,
arrayType);
mHeavyDataController =
XdmfHDF5Controller::New(hdf5Path,
dataSetPath,
arrayType,
std::vector<unsigned int>(1, 0),
std::vector<unsigned int>(1, 1),
std::vector<unsigned int>(1, sizeVal));
}
else {
assert(false);
......
......@@ -31,21 +31,32 @@
boost::shared_ptr<XdmfHDF5Controller>
XdmfHDF5Controller::New(const std::string & hdf5FilePath,
const std::string & dataSetPath,
const unsigned int size,
const boost::shared_ptr<const XdmfArrayType> type)
const boost::shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count)
{
boost::shared_ptr<XdmfHDF5Controller> p(new XdmfHDF5Controller(hdf5FilePath,
dataSetPath,
size,
type));
type,
start,
stride,
count));
return p;
}
XdmfHDF5Controller::XdmfHDF5Controller(const std::string & hdf5FilePath,
const std::string & dataSetPath,
const unsigned int size,
const boost::shared_ptr<const XdmfArrayType> type) :
XdmfHeavyDataController(hdf5FilePath, dataSetPath, size, type)
const boost::shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count) :
XdmfHeavyDataController(hdf5FilePath,
dataSetPath,
type,
count),
mStart(start),
mStride(stride)
{
}
......@@ -68,23 +79,43 @@ XdmfHDF5Controller::read(XdmfArray * const array)
void
XdmfHDF5Controller::read(XdmfArray * const array, const int fapl)
{
herr_t status;
hid_t hdf5Handle = H5Fopen(mFilePath.c_str(), H5F_ACC_RDONLY, fapl);
hid_t dataset = H5Dopen(hdf5Handle, mDataSetPath.c_str(), H5P_DEFAULT);
hid_t dataspace = H5Dget_space(dataset);
hssize_t numVals = H5Sget_simple_extent_npoints(dataspace);
std::vector<hsize_t> start(mStart.begin(), mStart.end());
std::vector<hsize_t> stride(mStride.begin(), mStride.end());
std::vector<hsize_t> count(mDimensions.begin(), mDimensions.end());
status = H5Sselect_hyperslab(dataspace,
H5S_SELECT_SET,
&start[0],
&stride[0],
&count[0],
NULL);
hssize_t numVals = H5Sget_select_npoints(dataspace);
hsize_t dims[1];
dims[0] = numVals;
// hid_t memspace = H5Screate_simple(1, dims, NULL);
// hsize_t memStart[] = {0};
// hsize_t memStride[] = {1};
// hsize_t memCount[] = {10};
//status = H5Sselect_hyperslab(memspace, H5S_SELECT_SET, memStart, memStride, memCount, NULL);
hid_t datatype = H5Dget_type(dataset);
array->initialize(mType, (const unsigned int)numVals);
H5Dread(dataset,
datatype,
H5S_ALL,
H5S_ALL,
H5P_DEFAULT,
array->getValuesInternal());
status = H5Dread(dataset,
datatype,
H5S_ALL,
dataspace,
H5P_DEFAULT,
array->getValuesInternal());
herr_t status;
status = H5Tclose(datatype);
status = H5Sclose(dataspace);
// status = H5Sclose(memspace);
status = H5Dclose(dataset);
status = H5Fclose(hdf5Handle);
}
......@@ -48,16 +48,23 @@ class XDMFCORE_EXPORT XdmfHDF5Controller : public XdmfHeavyDataController {
*
* @param hdf5FilePath the location of the hdf5 file the data set resides in.
* @param dataSetPath the location of the dataset within the hdf5 file.
* @param size the size of the dataset to read.
* @param type the data type of the dataset to read.
* @param start the offset of the starting element in each dimension in
* the hdf5 data set.
* @param stride the number of elements to move in each dimension from the
* hdf5 data set.
* @param count the number of elements to select in each dimension from the
* hdf5 data set. (size in each dimension)
*
* @return new HDF5 Controller.
*/
static boost::shared_ptr<XdmfHDF5Controller>
New(const std::string & hdf5FilePath,
const std::string & dataSetPath,
const unsigned int size,
const boost::shared_ptr<const XdmfArrayType> type);
const boost::shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count);
virtual std::string getName() const;
......@@ -67,8 +74,10 @@ class XDMFCORE_EXPORT XdmfHDF5Controller : public XdmfHeavyDataController {
XdmfHDF5Controller(const std::string & hdf5FilePath,
const std::string & dataSetPath,
const unsigned int size,
const boost::shared_ptr<const XdmfArrayType> type);
const boost::shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count);
void read(XdmfArray * const array, const int fapl);
......@@ -77,6 +86,9 @@ class XDMFCORE_EXPORT XdmfHDF5Controller : public XdmfHeavyDataController {
XdmfHDF5Controller(const XdmfHDF5Controller &); // Not implemented.
void operator=(const XdmfHDF5Controller &); // Not implemented.
const std::vector<unsigned int> mStart;
const std::vector<unsigned int> mStride;
};
#endif /* XDMFHDF5CONTROLLER_HPP_ */
......@@ -28,25 +28,31 @@
boost::shared_ptr<XdmfHDF5ControllerDSM>
XdmfHDF5ControllerDSM::New(const std::string & hdf5FilePath,
const std::string & dataSetPath,
const unsigned int size,
const boost::shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count,
H5FDdsmBuffer * const dsmBuffer)
{
boost::shared_ptr<XdmfHDF5ControllerDSM>
p(new XdmfHDF5ControllerDSM(hdf5FilePath,
dataSetPath,
size,
type,
start,
stride,
count,
dsmBuffer));
return p;
}
XdmfHDF5ControllerDSM::XdmfHDF5ControllerDSM(const std::string & hdf5FilePath,
const std::string & dataSetPath,
const unsigned int size,
const boost::shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count,
H5FDdsmBuffer * const dsmBuffer) :
XdmfHDF5Controller(hdf5FilePath, dataSetPath, size, type),
XdmfHDF5Controller(hdf5FilePath, dataSetPath, type, start, stride, count),
mDSMBuffer(dsmBuffer)
{
}
......
......@@ -52,8 +52,10 @@ class XDMFCORE_EXPORT XdmfHDF5ControllerDSM : public XdmfHDF5Controller {
static boost::shared_ptr<XdmfHDF5ControllerDSM>
New(const std::string & hdf5FilePath,
const std::string & dataSetPath,
const unsigned int size,
const boost::shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count,
H5FDdsmBuffer * const dsmBuffer);
std::string getName() const;
......@@ -64,8 +66,10 @@ class XDMFCORE_EXPORT XdmfHDF5ControllerDSM : public XdmfHDF5Controller {
XdmfHDF5ControllerDSM(const std::string & hdf5FilePath,
const std::string & dataSetPath,
const unsigned int size,
const boost::shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count,
H5FDdsmBuffer * const dsmBuffer);
private:
......
......@@ -47,10 +47,17 @@ XdmfHDF5Writer::~XdmfHDF5Writer()
boost::shared_ptr<XdmfHDF5Controller>
XdmfHDF5Writer::createHDF5Controller(const std::string & hdf5FilePath,
const std::string & dataSetPath,
const unsigned int size,
const boost::shared_ptr<const XdmfArrayType> type)
const boost::shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count)
{
return XdmfHDF5Controller::New(hdf5FilePath, dataSetPath, size, type);
return XdmfHDF5Controller::New(hdf5FilePath,
dataSetPath,
type,
start,
stride,
count);
}
void
......@@ -200,15 +207,23 @@ XdmfHDF5Writer::write(XdmfArray & array,
// Restore previous error handler
H5Eset_auto2(0, old_func, old_client_data);
// Attach a new controller to the array if needed.
// Attach a new controller to the array
unsigned int newSize = array.getSize();
if(mMode == Append && array.getHeavyDataController()) {
newSize = newSize + array.getHeavyDataController()->getSize();
}
if(mMode == Default || !array.getHeavyDataController()) {
boost::shared_ptr<XdmfHDF5Controller> newDataSetController =
this->createHDF5Controller(hdf5FilePath,
dataSetPath.str(),
array.getSize(),
array.getArrayType());
array.setHeavyDataController(newDataSetController);
mDataSetId++;
++mDataSetId;
}
const boost::shared_ptr<XdmfHDF5Controller> newDataController =
this->createHDF5Controller(hdf5FilePath,
dataSetPath.str(),
array.getArrayType(),
std::vector<unsigned int>(1, 0),
std::vector<unsigned int>(1, 1),
std::vector<unsigned int>(1, newSize));
array.setHeavyDataController(newDataController);
}
}
......@@ -74,16 +74,23 @@ class XDMFCORE_EXPORT XdmfHDF5Writer : public XdmfHeavyDataWriter {
*
* @param hdf5FilePath the location of the hdf5 file the data set resides in.
* @param dataSetPath the location of the dataset within the hdf5 file.
* @param size the size of the dataset to read.
* @param type the data type of the dataset to read.
* @param start the offset of the starting element in each dimension in
* the hdf5 data set.
* @param stride the number of elements to move in each dimension from the
* hdf5 data set.
* @param count the number of elements to select in each dimension from the
* hdf5 data set. (size in each dimension)
*
* @return new HDF5 Controller.
*/
virtual boost::shared_ptr<XdmfHDF5Controller>
createHDF5Controller(const std::string & hdf5FilePath,
const std::string & dataSetPath,
const unsigned int size,
const boost::shared_ptr<const XdmfArrayType> type);
const boost::shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count);
/**
* Write the XdmfArray to a hdf5 file.
......
......@@ -49,13 +49,17 @@ XdmfHDF5WriterDSM::~XdmfHDF5WriterDSM()
boost::shared_ptr<XdmfHDF5Controller>
XdmfHDF5WriterDSM::createHDF5Controller(const std::string & hdf5FilePath,
const std::string & dataSetPath,
const unsigned int size,
const boost::shared_ptr<const XdmfArrayType> type)
const boost::shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count)
{
return XdmfHDF5ControllerDSM::New(hdf5FilePath,
dataSetPath,
size,
type,
start,
stride,
count,
mDSMBuffer);
}
......
......@@ -72,8 +72,10 @@ protected:
virtual boost::shared_ptr<XdmfHDF5Controller>
createHDF5Controller(const std::string & hdf5FilePath,
const std::string & dataSetPath,
const unsigned int size,
const boost::shared_ptr<const XdmfArrayType> type);
const boost::shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & count);
private:
......
......@@ -21,16 +21,18 @@
/* */
/*****************************************************************************/
#include <functional>
#include <numeric>
#include "XdmfHeavyDataController.hpp"
#include "XdmfSystemUtils.hpp"
XdmfHeavyDataController::XdmfHeavyDataController(const std::string & filePath,
const std::string & dataSetPath,
const unsigned int size,
const boost::shared_ptr<const XdmfArrayType> type) :
const boost::shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & dimensions) :
mDataSetPath(dataSetPath),
mDimensions(dimensions),
mFilePath(XdmfSystemUtils::getRealPath(filePath)),
mSize(size),
mType(type)
{
}
......@@ -45,6 +47,11 @@ XdmfHeavyDataController::getDataSetPath() const
return mDataSetPath;
}
std::vector<unsigned int> XdmfHeavyDataController::getDimensions() const
{
return mDimensions;
}
std::string
XdmfHeavyDataController::getFilePath() const
{
......@@ -54,7 +61,10 @@ XdmfHeavyDataController::getFilePath() const
unsigned int
XdmfHeavyDataController::getSize() const
{
return mSize;
return std::accumulate(mDimensions.begin(),
mDimensions.end(),
1,
std::multiplies<int>());
}
boost::shared_ptr<const XdmfArrayType>
......
......@@ -31,6 +31,7 @@ class XdmfArrayType;
// Includes
#include <boost/shared_ptr.hpp>
#include <string>
#include <vector>
#include "XdmfCore.hpp"
/**
......@@ -60,6 +61,14 @@ class XDMFCORE_EXPORT XdmfHeavyDataController {
*/
std::string getDataSetPath() const;
/**
* Get the dimensions of the heavy data set owned by this controller.
*
* @return a vector containing the size in each dimension of the heavy data
* set owned by this controller.
*/
std::vector<unsigned int> getDimensions() const;
/**
* Get the absolute path to the heavy data file on disk where the
* data set owned by this controller resides.
......@@ -104,13 +113,13 @@ class XDMFCORE_EXPORT XdmfHeavyDataController {
XdmfHeavyDataController(const std::string & filePath,
const std::string & dataSetPath,
const unsigned int size,
const boost::shared_ptr<const XdmfArrayType> type);
const boost::shared_ptr<const XdmfArrayType> type,
const std::vector<unsigned int> & dimensions);
std::string mDataSetPath;
std::string mFilePath;
unsigned int mSize;
boost::shared_ptr<const XdmfArrayType> mType;
const std::string mDataSetPath;
const std::vector<unsigned int> mDimensions;
const std::string mFilePath;
const boost::shared_ptr<const XdmfArrayType> mType;
private:
......
......@@ -8,11 +8,21 @@ int main(int, char *)
boost::shared_ptr<XdmfHDF5Controller> controller =
XdmfHDF5Controller::New("output.h5",
"/foo/data1",
10,
XdmfArrayType::Int8());
XdmfArrayType::Int8(),
std::vector<unsigned int>(1, 0),
std::vector<unsigned int>(1, 1),
std::vector<unsigned int>(1, 10));
assert(controller->getDataSetPath().compare("/foo/data1") == 0);
assert(controller->getFilePath().compare(XdmfSystemUtils::getRealPath("output.h5")) == 0);
assert(controller->getSize() == 10);
assert(controller->getType() == XdmfArrayType::Int8());
/*
boost::shared_ptr<XdmfHDF5Controller> hyperslabController = XdmfHDF5Controller::New("DislocSegment_10x10x20_8K.h5", "/Attribute/Node/GlobalNodeId", XdmfArrayType::Int32(), std::vector<unsigned int>(1, 0), std::vector<unsigned int>(1, 1), std::vector<unsigned int>(1, 1533));
boost::shared_ptr<XdmfArray> array = XdmfArray::New();
array->setHeavyDataController(hyperslabController);
array->read();
std::cout << array->getValuesString() << std::endl;
*/
return 0;
}
......@@ -23,7 +23,6 @@ int main(int, char *)
boost::shared_ptr<XdmfHeavyDataController> secondController =
array->getHeavyDataController();
std::string secondPath = secondController->getDataSetPath();
assert(firstController != secondController);
assert(firstPath.compare(secondPath) != 0);
//
......@@ -34,7 +33,6 @@ int main(int, char *)
boost::shared_ptr<XdmfHeavyDataController> thirdController =
array->getHeavyDataController();
std::string thirdPath = thirdController->getDataSetPath();
assert(secondController == thirdController);
assert(secondPath.compare(thirdPath) == 0);
array->pushBack(3);
......@@ -42,7 +40,6 @@ int main(int, char *)
boost::shared_ptr<XdmfHeavyDataController> fourthController =
array->getHeavyDataController();
std::string fourthPath = fourthController->getDataSetPath();
assert(thirdController == fourthController);
assert(thirdPath.compare(fourthPath) == 0);
array->erase(0);
......@@ -51,7 +48,6 @@ int main(int, char *)
boost::shared_ptr<XdmfHeavyDataController> fifthController =
array->getHeavyDataController();
std::string fifthPath = fifthController->getDataSetPath();
assert(fourthController == fifthController);
assert(fourthPath.compare(fifthPath) == 0);
//
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment