Commit 9c5aaf4b authored by Kenneth Leiter's avatar Kenneth Leiter

ENH: Add write modes to HDF5Writer. It can now overwrite (write in the same

location previously written to) and append (write data to end of previously written
dataset).
parent ef98c862
......@@ -64,6 +64,25 @@ private:
const unsigned int mValuesStride;
};
class XdmfArray::Erase : public boost::static_visitor <void> {
public:
Erase(const unsigned int index) :
mIndex(index)
{
}
template<typename T>
void operator()(const boost::shared_ptr<std::vector<T> > & array) const
{
array->erase(array->begin() + mIndex);
}
private:
const unsigned int mIndex;
};
class XdmfArray::GetCapacity : public boost::static_visitor <unsigned int> {
public:
......@@ -366,6 +385,14 @@ XdmfArray::~XdmfArray()
std::string XdmfArray::ItemTag = "DataItem";
void XdmfArray::clear()
{
if(mHaveArray)
{
return boost::apply_visitor(Clear(), mArray);
}
}
void XdmfArray::copyValues(const unsigned int startIndex, const boost::shared_ptr<const XdmfArray> values, const unsigned int valuesStartIndex, const unsigned int numValues, const unsigned int arrayStride, const unsigned int valuesStride)
{
if(mHaveArrayPointer)
......@@ -383,11 +410,11 @@ void XdmfArray::copyValues(const unsigned int startIndex, const boost::shared_pt
boost::apply_visitor(CopyArrayValues(startIndex, valuesStartIndex, numValues, arrayStride, valuesStride), mArray, values->mArray);
}
void XdmfArray::clear()
void XdmfArray::erase(const unsigned int index)
{
if(mHaveArray)
{
return boost::apply_visitor(Clear(), mArray);
return boost::apply_visitor(Erase(index), mArray);
}
}
......
......@@ -82,6 +82,11 @@ public:
*/
void clear();
/**
* Remove a value from this array.
*/
void erase(const unsigned int index);
/**
* Get the capacity of this array (the number of values this array can store without reallocation).
*
......@@ -311,6 +316,7 @@ private:
template <typename T>
class CopyValues;
class Erase;
class GetCapacity;
class GetHDF5Type;
class GetPrecision;
......
......@@ -21,7 +21,7 @@ XdmfDataItem::~XdmfDataItem()
boost::shared_ptr<XdmfArray> XdmfDataItem::getArray()
{
return mArray;
return boost::const_pointer_cast<XdmfArray>(static_cast<const XdmfDataItem &>(*this).getArray());
}
boost::shared_ptr<const XdmfArray> XdmfDataItem::getArray() const
......
......@@ -27,12 +27,7 @@ void XdmfDomain::insert(const boost::shared_ptr<XdmfGrid> grid)
boost::shared_ptr<XdmfGrid> XdmfDomain::getGrid(const unsigned int index)
{
if(index >= mGrids.size())
{
assert(false);
// Out of range --- should we throw exceptions?
}
return mGrids[index];
return boost::const_pointer_cast<XdmfGrid>(static_cast<const XdmfDomain &>(*this).getGrid(index));
}
boost::shared_ptr<const XdmfGrid> XdmfDomain::getGrid(const unsigned int index) const
......
......@@ -27,6 +27,11 @@ XdmfGrid::~XdmfGrid()
std::string XdmfGrid::ItemTag = "Grid";
boost::shared_ptr<XdmfAttribute> XdmfGrid::getAttribute(const unsigned int index)
{
return boost::const_pointer_cast<XdmfAttribute>(static_cast<const XdmfGrid &>(*this).getAttribute(index));
}
boost::shared_ptr<const XdmfAttribute> XdmfGrid::getAttribute(const unsigned int index) const
{
if(index >= mAttributes.size())
{
......@@ -35,12 +40,12 @@ boost::shared_ptr<XdmfAttribute> XdmfGrid::getAttribute(const unsigned int index
return mAttributes[index];
}
boost::shared_ptr<const XdmfAttribute> XdmfGrid::getAttribute(const unsigned int index) const
boost::shared_ptr<XdmfAttribute> XdmfGrid::getAttribute(const std::string & attributeName)
{
return boost::const_pointer_cast<XdmfAttribute>(static_cast<const XdmfGrid &>(*this).getAttribute(index));
return boost::const_pointer_cast<XdmfAttribute>(static_cast<const XdmfGrid &>(*this).getAttribute(attributeName));
}
boost::shared_ptr<XdmfAttribute> XdmfGrid::getAttribute(const std::string & attributeName)
boost::shared_ptr<const XdmfAttribute> XdmfGrid::getAttribute(const std::string & attributeName) const
{
for(std::vector<boost::shared_ptr<XdmfAttribute> >::const_iterator iter = mAttributes.begin(); iter != mAttributes.end(); ++iter)
{
......@@ -52,19 +57,14 @@ boost::shared_ptr<XdmfAttribute> XdmfGrid::getAttribute(const std::string & attr
return boost::shared_ptr<XdmfAttribute>();
}
boost::shared_ptr<const XdmfAttribute> XdmfGrid::getAttribute(const std::string & attributeName) const
{
return boost::const_pointer_cast<XdmfAttribute>(static_cast<const XdmfGrid &>(*this).getAttribute(attributeName));
}
boost::shared_ptr<XdmfGeometry> XdmfGrid::getGeometry()
{
return mGeometry;
return boost::const_pointer_cast<XdmfGeometry>(static_cast<const XdmfGrid &>(*this).getGeometry());
}
boost::shared_ptr<const XdmfGeometry> XdmfGrid::getGeometry() const
{
return boost::const_pointer_cast<XdmfGeometry>(static_cast<const XdmfGrid &>(*this).getGeometry());
return mGeometry;
}
std::map<std::string, std::string> XdmfGrid::getItemProperties() const
......@@ -95,6 +95,11 @@ unsigned int XdmfGrid::getNumberOfSets() const
}
boost::shared_ptr<XdmfSet> XdmfGrid::getSet(const unsigned int index)
{
return boost::const_pointer_cast<XdmfSet>(static_cast<const XdmfGrid &>(*this).getSet(index));
}
boost::shared_ptr<const XdmfSet> XdmfGrid::getSet(const unsigned int index) const
{
if(index >= mSets.size())
{
......@@ -103,19 +108,14 @@ boost::shared_ptr<XdmfSet> XdmfGrid::getSet(const unsigned int index)
return mSets[index];
}
boost::shared_ptr<const XdmfSet> XdmfGrid::getSet(const unsigned int index) const
{
return boost::const_pointer_cast<XdmfSet>(static_cast<const XdmfGrid &>(*this).getSet(index));
}
boost::shared_ptr<XdmfTopology> XdmfGrid::getTopology()
{
return mTopology;
return boost::const_pointer_cast<XdmfTopology>(static_cast<const XdmfGrid &>(*this).getTopology());
}
boost::shared_ptr<const XdmfTopology> XdmfGrid::getTopology() const
{
return boost::const_pointer_cast<XdmfTopology>(static_cast<const XdmfGrid &>(*this).getTopology());
return mTopology;
}
void XdmfGrid::insert(const boost::shared_ptr<XdmfAttribute> attribute)
......
......@@ -24,11 +24,7 @@ std::string XdmfGridCollection::ItemTag = "Grid";
boost::shared_ptr<XdmfGrid> XdmfGridCollection::getGrid(const unsigned int index)
{
if(index >= mGrids.size())
{
assert(false);
}
return mGrids[index];
return boost::const_pointer_cast<XdmfGrid>(static_cast<const XdmfGridCollection &>(*this).getGrid(index));
}
boost::shared_ptr<const XdmfGrid> XdmfGridCollection::getGrid(const unsigned int index) const
......
......@@ -6,18 +6,18 @@
#include "XdmfArray.hpp"
#include "XdmfHDF5Controller.hpp"
XdmfHDF5Controller::XdmfHDF5Controller(const std::string & hdf5DataSetPath, const unsigned int precision, const unsigned int size, const std::string & type) :
XdmfHDF5Controller::XdmfHDF5Controller(const std::string & dataSetPath, const unsigned int precision, const unsigned int size, const std::string & type) :
mPrecision(precision),
mSize(size),
mType(type)
{
size_t colonLocation = hdf5DataSetPath.find(":");
size_t colonLocation = dataSetPath.find(":");
if(colonLocation != std::string::npos)
{
mHDF5FilePath = hdf5DataSetPath.substr(0, colonLocation);
mHDF5FilePath = dataSetPath.substr(0, colonLocation);
if(colonLocation + 1 != mHDF5FilePath.size())
{
mDataSetName = hdf5DataSetPath.substr(colonLocation + 1, hdf5DataSetPath.size());
mDataSetName = dataSetPath.substr(colonLocation + 1, dataSetPath.size());
}
else
{
......@@ -34,6 +34,11 @@ XdmfHDF5Controller::~XdmfHDF5Controller()
{
}
std::string XdmfHDF5Controller::getDataSetName() const
{
return mDataSetName;
}
std::string XdmfHDF5Controller::getDataSetPath() const
{
std::stringstream toReturn;
......@@ -41,6 +46,11 @@ std::string XdmfHDF5Controller::getDataSetPath() const
return toReturn.str();
}
std::string XdmfHDF5Controller::getHDF5FilePath() const
{
return mHDF5FilePath;
}
unsigned int XdmfHDF5Controller::getPrecision() const
{
return mPrecision;
......
......@@ -23,12 +23,19 @@ public:
/**
* Create a new controller for an hdf5 data set on disk.
*/
static boost::shared_ptr<XdmfHDF5Controller> New(const std::string & hdf5DataSetPath, const unsigned int precision, const unsigned int size, const std::string & type)
static boost::shared_ptr<XdmfHDF5Controller> New(const std::string & dataSetPath, const unsigned int precision, const unsigned int size, const std::string & type)
{
boost::shared_ptr<XdmfHDF5Controller> p(new XdmfHDF5Controller(hdf5DataSetPath, precision, size, type));
boost::shared_ptr<XdmfHDF5Controller> p(new XdmfHDF5Controller(dataSetPath, precision, size, type));
return p;
}
/**
* Get the name of the data set owned by this controller.
*
* @return a std::string containing the name of the data set.
*/
std::string getDataSetName() const;
/**
* Get the path to the hdf5 data set on disk owned by this controller.
*
......@@ -36,6 +43,13 @@ public:
*/
std::string getDataSetPath() const;
/**
* Get the path to the hdf5 file on disk where the data set owned by this controller resides.
*
* @return a std::string containing the path to the hdf5 file.
*/
std::string getHDF5FilePath() const;
/**
* Get the precision of the hdf5 data set owned by this controller.
*
......@@ -66,7 +80,7 @@ public:
protected:
XdmfHDF5Controller(const std::string & hdf5DataSetPath, const unsigned int precision, const unsigned int size, const std::string & type);
XdmfHDF5Controller(const std::string & dataSetPath, const unsigned int precision, const unsigned int size, const std::string & type);
virtual ~XdmfHDF5Controller();
private:
......
......@@ -16,19 +16,20 @@ public:
XdmfHDF5WriterImpl(const std::string & hdf5FilePath) :
mHDF5FilePath(hdf5FilePath),
mLastWrittenDataSet("")
mLastWrittenDataSet(""),
mDataSetId(0),
mMode(Default)
{
};
~XdmfHDF5WriterImpl()
{
};
int mDataSetId;
std::string mHDF5FilePath;
std::string mLastWrittenDataSet;
static int mDataSetId;
Mode mMode;
};
int XdmfHDF5Writer::XdmfHDF5WriterImpl::mDataSetId = 0;
class XdmfHDF5Writer::GetHDF5Type : public boost::static_visitor <hid_t> {
public:
......@@ -111,6 +112,11 @@ std::string XdmfHDF5Writer::getLastWrittenDataSet() const
return mImpl->mLastWrittenDataSet;
}
void XdmfHDF5Writer::setMode(const Mode mode)
{
mImpl->mMode = mode;
}
void XdmfHDF5Writer::visit(XdmfArray & array, const boost::shared_ptr<XdmfBaseVisitor> visitor)
{
hid_t datatype = -1;
......@@ -131,8 +137,19 @@ void XdmfHDF5Writer::visit(XdmfArray & array, const boost::shared_ptr<XdmfBaseVi
if(datatype != -1)
{
std::string hdf5FilePath = mImpl->mHDF5FilePath;
std::stringstream dataSetName;
dataSetName << "Data" << mImpl->mDataSetId;
if((mImpl->mMode == Overwrite || mImpl->mMode == Append) && array.mHDF5Controller)
{
// Write to the previous dataset
dataSetName << array.mHDF5Controller->getDataSetName();
hdf5FilePath = array.mHDF5Controller->getHDF5FilePath();
}
else
{
dataSetName << "Data" << mImpl->mDataSetId;
}
// Open a hdf5 dataset and write to it on disk.
herr_t status;
......@@ -145,22 +162,63 @@ void XdmfHDF5Writer::visit(XdmfArray & array, const boost::shared_ptr<XdmfBaseVi
H5Eget_auto(0, &old_func, &old_client_data);
H5Eset_auto2(0, NULL, NULL);
if(H5Fis_hdf5(mImpl->mHDF5FilePath.c_str()) > 0)
if(H5Fis_hdf5(hdf5FilePath.c_str()) > 0)
{
hdf5Handle = H5Fopen(mImpl->mHDF5FilePath.c_str(), H5F_ACC_RDWR, H5P_DEFAULT);
hdf5Handle = H5Fopen(hdf5FilePath.c_str(), H5F_ACC_RDWR, H5P_DEFAULT);
}
else
{
hdf5Handle = H5Fcreate(mImpl->mHDF5FilePath.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
hdf5Handle = H5Fcreate(hdf5FilePath.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
}
hid_t dataset = H5Dopen(hdf5Handle, dataSetName.str().c_str(), H5P_DEFAULT);
hid_t dataspace = H5S_ALL;
hid_t memspace = H5S_ALL;
if(dataset < 0)
{
hid_t dataspace = H5Screate_simple(1, &size, NULL);
dataset = H5Dcreate(hdf5Handle, dataSetName.str().c_str(), datatype, dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
hsize_t unlimited = H5S_UNLIMITED;
memspace = H5Screate_simple(1, &size, &unlimited);
hid_t property = H5Pcreate(H5P_DATASET_CREATE);
hsize_t chunkSize = 1024;
status = H5Pset_chunk(property, 1, &chunkSize);
dataset = H5Dcreate(hdf5Handle, dataSetName.str().c_str(), datatype, memspace, H5P_DEFAULT, property, H5P_DEFAULT);
status = H5Pclose(property);
}
else
{
// Need to resize dataset to fit new data
if(mImpl->mMode == Append)
{
// Get size of old dataset
dataspace = H5Dget_space(dataset);
hssize_t datasize = H5Sget_simple_extent_npoints(dataspace);
status = H5Sclose(dataspace);
// Resize to fit size of old and new data.
hsize_t newSize = size + datasize;
status = H5Dset_extent(dataset, &newSize);
// Select hyperslab to write to.
memspace = H5Screate_simple(1, &size, NULL);
dataspace = H5Dget_space(dataset);
hsize_t start = datasize;
status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, &start, NULL, &size, NULL) ;
}
else
{
status = H5Dset_extent(dataset, &size);
}
}
status = H5Dwrite(dataset, datatype, memspace, dataspace, H5P_DEFAULT, array.getValuesPointer());
if(dataspace != H5S_ALL)
{
status = H5Sclose(dataspace);
}
status = H5Dwrite(dataset, datatype, H5S_ALL, H5S_ALL, H5P_DEFAULT, array.getValuesPointer());
if(memspace != H5S_ALL)
{
status = H5Sclose(memspace);
}
status = H5Dclose(dataset);
status = H5Fclose(hdf5Handle);
......@@ -168,13 +226,17 @@ void XdmfHDF5Writer::visit(XdmfArray & array, const boost::shared_ptr<XdmfBaseVi
H5Eset_auto2(0, old_func, old_client_data);
std::stringstream writtenDataSet;
writtenDataSet << mImpl->mHDF5FilePath << ":" << dataSetName.str();
writtenDataSet << hdf5FilePath << ":" << dataSetName.str();
mImpl->mLastWrittenDataSet = writtenDataSet.str();
mImpl->mDataSetId++;
boost::shared_ptr<XdmfHDF5Controller> newDataSetController = XdmfHDF5Controller::New(writtenDataSet.str(), array.getPrecision(),
// Attach a new controller to the array if needed.
if(mImpl->mMode == Default || !array.mHDF5Controller)
{
boost::shared_ptr<XdmfHDF5Controller> newDataSetController = XdmfHDF5Controller::New(writtenDataSet.str(), array.getPrecision(),
array.getSize(), array.getType());
array.setHDF5Controller(newDataSetController);
array.setHDF5Controller(newDataSetController);
mImpl->mDataSetId++;
}
}
}
......@@ -12,13 +12,26 @@ class XdmfArray;
*
* XdmfHDF5Writer traverses an Xdmf graph structure and writes data stored in XdmfArrays to HDF5.
* Writing begins by calling the accept() operation on any XdmfItem and supplying this writer
* as the parameter.
* as the parameter. The writer will write all XdmfArrays under the XdmfItem to an hdf5 file on
* disk. It will also attach an XdmfHDF5Controller to all XdmfArrays that it writes to disk.
*
* There are three modes of operation for this writer:
* Default - All initialized XdmfArrays are written to new hdf5 datasets irregardless of whether they
* are attached to another hdf5 dataset on disk via an XdmfHDF5Controller.
* Overwrite - If an initialized XdmfArray is attached to an hdf5 dataset via an XdmfHDF5Contoller the writer
* will write values to that location, overwriting all previous written values.
* Append - If an initialized XdmfArray is attached to an hdf5 dataset via an XdmfHDF5Controller the writer
* will append the values to the end of the dataset on disk.
*/
class XdmfHDF5Writer : public XdmfVisitor,
public Loki::Visitor<XdmfArray> {
public:
enum Mode {
Default, Overwrite, Append
};
template <typename T> friend void boost::checked_delete(T * x);
/**
......@@ -32,6 +45,8 @@ public:
return p;
}
void setMode(const Mode mode);
/**
* Get the path to the last written data set written by this writer.
*
......
......@@ -69,6 +69,16 @@ void XdmfWriter::closeFile()
xmlCleanupParser();
}
boost::shared_ptr<XdmfHDF5Writer> XdmfWriter::getHDF5Writer()
{
return boost::const_pointer_cast<XdmfHDF5Writer>(static_cast<const XdmfWriter &>(*this).getHDF5Writer());
}
boost::shared_ptr<const XdmfHDF5Writer> XdmfWriter::getHDF5Writer() const
{
return mImpl->mHDF5Writer;
}
unsigned int XdmfWriter::getLightDataLimit() const
{
return mImpl->mLightDataLimit;
......
......@@ -50,6 +50,20 @@ public:
return p;
}
/**
* Get the hdf5 writer that this XdmfWriter uses to write heavy data to disk.
*
* @return the requested hdf5 writer.
*/
boost::shared_ptr<XdmfHDF5Writer> getHDF5Writer();
/**
* Get the hdf5 writer that this XdmfWriter uses to write heavy data to disk (const version).
*
* @return the requested hdf5 writer.
*/
boost::shared_ptr<const XdmfHDF5Writer> getHDF5Writer() const;
/**
* Get the number of values that this writer writes to light data (XML) before switching to a heavy data format.
*
......
#include "XdmfArray.hpp"
#include "XdmfHDF5Controller.hpp"
#include "XdmfHDF5Writer.hpp"
#include "XdmfTestDataGenerator.hpp"
int main(int argc, char* argv[])
{
boost::shared_ptr<XdmfArray> array = XdmfArray::New();
array->pushBack(0);
array->pushBack(1);
array->pushBack(2);
//
// Default operation - Always write to separate datasets.
//
assert(array->getHDF5Controller() == NULL);
boost::shared_ptr<XdmfHDF5Writer> writer = XdmfHDF5Writer::New("hdf5WriterTest.h5");
array->accept(writer);
boost::shared_ptr<XdmfHDF5Controller> firstController = array->getHDF5Controller();
std::string firstPath = firstController->getDataSetPath();
array->accept(writer);
boost::shared_ptr<XdmfHDF5Controller> secondController = array->getHDF5Controller();
std::string secondPath = secondController->getDataSetPath();
assert(firstController != secondController);
assert(firstPath.compare(secondPath) != 0);
//
// Overwrite operation - Always write to the same datasets.
//
writer->setMode(XdmfHDF5Writer::Overwrite);
array->accept(writer);
boost::shared_ptr<XdmfHDF5Controller> thirdController = array->getHDF5Controller();
std::string thirdPath = thirdController->getDataSetPath();
assert(secondController == thirdController);
assert(secondPath.compare(thirdPath) == 0);
array->pushBack(3);
array->accept(writer);
boost::shared_ptr<XdmfHDF5Controller> fourthController = array->getHDF5Controller();
std::string fourthPath = fourthController->getDataSetPath();
assert(thirdController == fourthController);
assert(thirdPath.compare(fourthPath) == 0);
boost::shared_ptr<XdmfGrid> grid = XdmfTestDataGenerator::createHexahedron();
array->erase(0);
array->erase(0);
array->accept(writer);
boost::shared_ptr<XdmfHDF5Controller> fifthController = array->getHDF5Controller();
std::string fifthPath = fifthController->getDataSetPath();
assert(fourthController == fifthController);
assert(fourthPath.compare(fifthPath) == 0);
boost::shared_ptr<XdmfDomain> domain = XdmfDomain::New();
domain->insert(grid);
domain->accept(writer);
//
// Append operation - Append data to same dataset.
//
writer->setMode(XdmfHDF5Writer::Append);
// Append 4 times
array->accept(writer);
array->accept(writer);
array->accept(writer);
array->accept(writer);
assert(array->getSize() == 2);
array->read();
assert(array->getSize() == 10);
for(int i=0; i<5; ++i)
{
assert(array->getValueCopy<int>(i*2) == 2);
assert(array->getValueCopy<int>(i*2 + 1) == 3);
}
return 0;
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment