Updates will be applied April 15th at 12pm EDT (UTC-0400). GitLab could be a little slow between 12 - 12:45pm EDT.

Commit 9f1c72b8 authored by Kenneth Leiter's avatar Kenneth Leiter

ENH: Add Mode to XdmfWriter. Default mode ensures that all heavy data...

ENH: Add Mode to XdmfWriter.  Default mode ensures that all heavy data referenced by the XML file being written
exists in a single heavy data file.  A DistributedHeavyData mode can be set to allow heavy data referenced by
the XML file to exist in more than one hdf5 file.
parent 40a27f65
...@@ -14,8 +14,8 @@ XdmfHDF5Controller::XdmfHDF5Controller(const std::string & dataSetPath, const un ...@@ -14,8 +14,8 @@ XdmfHDF5Controller::XdmfHDF5Controller(const std::string & dataSetPath, const un
size_t colonLocation = dataSetPath.find(":"); size_t colonLocation = dataSetPath.find(":");
if(colonLocation != std::string::npos) if(colonLocation != std::string::npos)
{ {
mHDF5FilePath = dataSetPath.substr(0, colonLocation); mFilePath = dataSetPath.substr(0, colonLocation);
if(colonLocation + 1 != mHDF5FilePath.size()) if(colonLocation + 1 != mFilePath.size())
{ {
mDataSetName = dataSetPath.substr(colonLocation + 1, dataSetPath.size()); mDataSetName = dataSetPath.substr(colonLocation + 1, dataSetPath.size());
} }
...@@ -42,13 +42,13 @@ std::string XdmfHDF5Controller::getDataSetName() const ...@@ -42,13 +42,13 @@ std::string XdmfHDF5Controller::getDataSetName() const
std::string XdmfHDF5Controller::getDataSetPath() const std::string XdmfHDF5Controller::getDataSetPath() const
{ {
std::stringstream toReturn; std::stringstream toReturn;
toReturn << mHDF5FilePath << ":" << mDataSetName; toReturn << mFilePath << ":" << mDataSetName;
return toReturn.str(); return toReturn.str();
} }
std::string XdmfHDF5Controller::getHDF5FilePath() const std::string XdmfHDF5Controller::getFilePath() const
{ {
return mHDF5FilePath; return mFilePath;
} }
unsigned int XdmfHDF5Controller::getSize() const unsigned int XdmfHDF5Controller::getSize() const
...@@ -63,7 +63,7 @@ boost::shared_ptr<const XdmfArrayType> XdmfHDF5Controller::getType() const ...@@ -63,7 +63,7 @@ boost::shared_ptr<const XdmfArrayType> XdmfHDF5Controller::getType() const
void XdmfHDF5Controller::read(XdmfArray * const array) void XdmfHDF5Controller::read(XdmfArray * const array)
{ {
hid_t hdf5Handle = H5Fopen(mHDF5FilePath.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); hid_t hdf5Handle = H5Fopen(mFilePath.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
hid_t dataset = H5Dopen(hdf5Handle, mDataSetName.c_str(), H5P_DEFAULT); hid_t dataset = H5Dopen(hdf5Handle, mDataSetName.c_str(), H5P_DEFAULT);
hid_t dataspace = H5Dget_space(dataset); hid_t dataspace = H5Dget_space(dataset);
hssize_t numVals = H5Sget_simple_extent_npoints(dataspace); hssize_t numVals = H5Sget_simple_extent_npoints(dataspace);
......
...@@ -49,7 +49,7 @@ public: ...@@ -49,7 +49,7 @@ public:
* *
* @return a std::string containing the path to the hdf5 file. * @return a std::string containing the path to the hdf5 file.
*/ */
std::string getHDF5FilePath() const; std::string getFilePath() const;
/** /**
* Get the size of the hdf5 data set owned by this controller. * Get the size of the hdf5 data set owned by this controller.
...@@ -82,7 +82,7 @@ private: ...@@ -82,7 +82,7 @@ private:
void operator=(const XdmfHDF5Controller & hdf5Controller); // Not implemented. void operator=(const XdmfHDF5Controller & hdf5Controller); // Not implemented.
std::string mDataSetName; std::string mDataSetName;
std::string mHDF5FilePath; std::string mFilePath;
unsigned int mSize; unsigned int mSize;
boost::shared_ptr<const XdmfArrayType> mType; boost::shared_ptr<const XdmfArrayType> mType;
}; };
......
...@@ -15,7 +15,7 @@ class XdmfHDF5Writer::XdmfHDF5WriterImpl { ...@@ -15,7 +15,7 @@ class XdmfHDF5Writer::XdmfHDF5WriterImpl {
public: public:
XdmfHDF5WriterImpl(const std::string & hdf5FilePath) : XdmfHDF5WriterImpl(const std::string & hdf5FilePath) :
mHDF5FilePath(hdf5FilePath), mFilePath(hdf5FilePath),
mLastWrittenDataSet(""), mLastWrittenDataSet(""),
mDataSetId(0), mDataSetId(0),
mMode(Default) mMode(Default)
...@@ -25,7 +25,7 @@ public: ...@@ -25,7 +25,7 @@ public:
{ {
}; };
int mDataSetId; int mDataSetId;
std::string mHDF5FilePath; std::string mFilePath;
std::string mLastWrittenDataSet; std::string mLastWrittenDataSet;
Mode mMode; Mode mMode;
}; };
...@@ -107,11 +107,21 @@ XdmfHDF5Writer::~XdmfHDF5Writer() ...@@ -107,11 +107,21 @@ XdmfHDF5Writer::~XdmfHDF5Writer()
std::cout << "Deleted XdmfHDF5Writer " << this << std::endl; std::cout << "Deleted XdmfHDF5Writer " << this << std::endl;
} }
std::string XdmfHDF5Writer::getFilePath() const
{
return mImpl->mFilePath;
}
std::string XdmfHDF5Writer::getLastWrittenDataSet() const std::string XdmfHDF5Writer::getLastWrittenDataSet() const
{ {
return mImpl->mLastWrittenDataSet; return mImpl->mLastWrittenDataSet;
} }
XdmfHDF5Writer::Mode XdmfHDF5Writer::getMode() const
{
return mImpl->mMode;
}
void XdmfHDF5Writer::setMode(const Mode mode) void XdmfHDF5Writer::setMode(const Mode mode)
{ {
mImpl->mMode = mode; mImpl->mMode = mode;
...@@ -137,14 +147,14 @@ void XdmfHDF5Writer::visit(XdmfArray & array, const boost::shared_ptr<XdmfBaseVi ...@@ -137,14 +147,14 @@ void XdmfHDF5Writer::visit(XdmfArray & array, const boost::shared_ptr<XdmfBaseVi
if(datatype != -1) if(datatype != -1)
{ {
std::string hdf5FilePath = mImpl->mHDF5FilePath; std::string hdf5FilePath = mImpl->mFilePath;
std::stringstream dataSetName; std::stringstream dataSetName;
if((mImpl->mMode == Overwrite || mImpl->mMode == Append) && array.mHDF5Controller) if((mImpl->mMode == Overwrite || mImpl->mMode == Append) && array.mHDF5Controller)
{ {
// Write to the previous dataset // Write to the previous dataset
dataSetName << array.mHDF5Controller->getDataSetName(); dataSetName << array.mHDF5Controller->getDataSetName();
hdf5FilePath = array.mHDF5Controller->getHDF5FilePath(); hdf5FilePath = array.mHDF5Controller->getFilePath();
} }
else else
{ {
......
...@@ -45,7 +45,12 @@ public: ...@@ -45,7 +45,12 @@ public:
return p; return p;
} }
void setMode(const Mode mode); /**
* Get the path to the hdf5 file on disk this writer is writing to.
*
* @return a std::string containing the path to the hdf5 file on disk this writer is writing to.
*/
std::string getFilePath() const;
/** /**
* Get the path to the last written data set written by this writer. * Get the path to the last written data set written by this writer.
...@@ -54,6 +59,20 @@ public: ...@@ -54,6 +59,20 @@ public:
*/ */
std::string getLastWrittenDataSet() const; std::string getLastWrittenDataSet() const;
/**
* Get the Mode of operation for this writer.
*
* @return the Mode of operation for this writer.
*/
Mode getMode() const;
/**
* Set the mode of operation for this writer.
*
* @param mode the Mode of operation for this writer.
*/
void setMode(const Mode mode);
/** /**
* Write an XdmfArray to HDF5. * Write an XdmfArray to HDF5.
* *
......
...@@ -5,6 +5,7 @@ ...@@ -5,6 +5,7 @@
#include <sstream> #include <sstream>
#include "XdmfArray.hpp" #include "XdmfArray.hpp"
#include "XdmfItem.hpp" #include "XdmfItem.hpp"
#include "XdmfHDF5Controller.hpp"
#include "XdmfHDF5Writer.hpp" #include "XdmfHDF5Writer.hpp"
#include "XdmfWriter.hpp" #include "XdmfWriter.hpp"
...@@ -18,18 +19,22 @@ public: ...@@ -18,18 +19,22 @@ public:
XdmfWriterImpl(const std::string & xmlFilePath, const boost::shared_ptr<XdmfHDF5Writer> hdf5Writer) : XdmfWriterImpl(const std::string & xmlFilePath, const boost::shared_ptr<XdmfHDF5Writer> hdf5Writer) :
mHDF5Writer(hdf5Writer), mHDF5Writer(hdf5Writer),
mLightDataLimit(100), mLightDataLimit(100),
mXMLFilePath(xmlFilePath), mMode(Default),
mTraverseLevel(0) mTraverseLevel(0),
mXMLCurrentNode(NULL),
mXMLDocument(NULL),
mXMLFilePath(xmlFilePath)
{ {
}; };
~XdmfWriterImpl() ~XdmfWriterImpl()
{ {
}; };
boost::shared_ptr<XdmfHDF5Writer> mHDF5Writer; boost::shared_ptr<XdmfHDF5Writer> mHDF5Writer;
unsigned int mTraverseLevel;
unsigned int mLightDataLimit; unsigned int mLightDataLimit;
xmlDocPtr mXMLDocument; Mode mMode;
unsigned int mTraverseLevel;
xmlNodePtr mXMLCurrentNode; xmlNodePtr mXMLCurrentNode;
xmlDocPtr mXMLDocument;
std::string mXMLFilePath; std::string mXMLFilePath;
}; };
...@@ -79,11 +84,21 @@ boost::shared_ptr<const XdmfHDF5Writer> XdmfWriter::getHDF5Writer() const ...@@ -79,11 +84,21 @@ boost::shared_ptr<const XdmfHDF5Writer> XdmfWriter::getHDF5Writer() const
return mImpl->mHDF5Writer; return mImpl->mHDF5Writer;
} }
std::string XdmfWriter::getFilePath() const
{
return mImpl->mXMLFilePath;
}
unsigned int XdmfWriter::getLightDataLimit() const unsigned int XdmfWriter::getLightDataLimit() const
{ {
return mImpl->mLightDataLimit; return mImpl->mLightDataLimit;
} }
XdmfWriter::Mode XdmfWriter::getMode() const
{
return mImpl->mMode;
}
void XdmfWriter::openFile() void XdmfWriter::openFile()
{ {
mImpl->mXMLDocument = xmlNewDoc((xmlChar*)"1.0"); mImpl->mXMLDocument = xmlNewDoc((xmlChar*)"1.0");
...@@ -96,9 +111,20 @@ void XdmfWriter::setLightDataLimit(const unsigned int numValues) ...@@ -96,9 +111,20 @@ void XdmfWriter::setLightDataLimit(const unsigned int numValues)
mImpl->mLightDataLimit = numValues; mImpl->mLightDataLimit = numValues;
} }
void XdmfWriter::setMode(const Mode mode)
{
mImpl->mMode = mode;
}
void XdmfWriter::visit(XdmfArray & array, const boost::shared_ptr<XdmfBaseVisitor> visitor) void XdmfWriter::visit(XdmfArray & array, const boost::shared_ptr<XdmfBaseVisitor> visitor)
{ {
std::stringstream xmlTextValues; std::stringstream xmlTextValues;
if(array.getHDF5Controller() && array.getHDF5Controller()->getFilePath().compare(mImpl->mHDF5Writer->getFilePath()) != 0 && mImpl->mMode == Default)
{
array.read();
}
if(array.getHDF5Controller() || array.getSize() > mImpl->mLightDataLimit) if(array.getHDF5Controller() || array.getSize() > mImpl->mLightDataLimit)
{ {
mImpl->mHDF5Writer->visit(array, mImpl->mHDF5Writer); mImpl->mHDF5Writer->visit(array, mImpl->mHDF5Writer);
......
...@@ -11,10 +11,16 @@ class XdmfHDF5Writer; ...@@ -11,10 +11,16 @@ class XdmfHDF5Writer;
/** /**
* @brief Traverse the Xdmf graph and write light and heavy data stored to disk. * @brief Traverse the Xdmf graph and write light and heavy data stored to disk.
* *
* XdmfWriter visits each node of an Xdmf graph structure and writes the data to disk. Writing begins by calling the * XdmfWriter visits each node of an Xdmf graph structure and writes data to disk. Writing begins by calling the
* accept() operation on any XdmfItem and supplying this writer as the parameter. The XdmfItem as well as all children * accept() operation on any XdmfItem and supplying this writer as the parameter. The XdmfItem as well as all children
* attached to the XdmfItem are written to disk. Heavy data is written to HDF5 format using the XdmfHDF5Writer and light * attached to the XdmfItem are written to disk. Heavy data is written to HDF5 format using the XdmfHDF5Writer and light
* data is written to XML. * data is written to XML.
*
* By default, the XdmfWriter writes all heavy data to a single heavy data file specified by the XdmfHDF5Writer.
* If a dataset is encountered that resides in a different heavy data file on disk, the dataset is read from disk and written
* to the new hdf5 file. If this is undesired, the XdmfWriter can be set to DistributedHeavyData mode in which the writer
* will automatically reference any hdf5 dataset even if it resides in a different file than the one currently being written to.
* written.
*/ */
class XdmfWriter : public XdmfVisitor, class XdmfWriter : public XdmfVisitor,
public Loki::Visitor<XdmfArray> { public Loki::Visitor<XdmfArray> {
...@@ -23,6 +29,10 @@ public: ...@@ -23,6 +29,10 @@ public:
virtual ~XdmfWriter(); virtual ~XdmfWriter();
enum Mode {
Default, DistributedHeavyData
};
/** /**
* Create a new XdmfWriter to write Xdmf data to disk. This will create its own hdf5 writer based on the xmlFileName. * Create a new XdmfWriter to write Xdmf data to disk. This will create its own hdf5 writer based on the xmlFileName.
* For example, if supplied "output.xmf" the created hdf5 writer would write to file "output.h5". * For example, if supplied "output.xmf" the created hdf5 writer would write to file "output.h5".
...@@ -50,6 +60,13 @@ public: ...@@ -50,6 +60,13 @@ public:
return p; return p;
} }
/**
* Get the path to the XML file on disk this writer is writing to.
*
* @return a std::string containing the path to the XML file on disk this writer is writing to.
*/
std::string getFilePath() const;
/** /**
* Get the hdf5 writer that this XdmfWriter uses to write heavy data to disk. * Get the hdf5 writer that this XdmfWriter uses to write heavy data to disk.
* *
...@@ -71,6 +88,13 @@ public: ...@@ -71,6 +88,13 @@ public:
*/ */
unsigned int getLightDataLimit() const; unsigned int getLightDataLimit() const;
/**
* Get the Mode of operation for this writer.
*
* @return the Mode of operation for this writer.
*/
Mode getMode() const;
/** /**
* Set the number of values that this writer writes to light data (XML) before switching to a heavy data format. * Set the number of values that this writer writes to light data (XML) before switching to a heavy data format.
* *
...@@ -78,6 +102,13 @@ public: ...@@ -78,6 +102,13 @@ public:
*/ */
void setLightDataLimit(const unsigned int numValues); void setLightDataLimit(const unsigned int numValues);
/**
* Set the mode of operation for this writer.
*
* @param mode the Mode of operation for this writer.
*/
void setMode(const Mode mode);
/** /**
* Write an XdmfArray to disk * Write an XdmfArray to disk
* *
......
...@@ -21,6 +21,7 @@ int main(int argc, char* argv[]) ...@@ -21,6 +21,7 @@ int main(int argc, char* argv[])
boost::shared_ptr<XdmfDomain> readDomain = boost::shared_dynamic_cast<XdmfDomain>(reader->read("TestXdmfReader1.xmf")); boost::shared_ptr<XdmfDomain> readDomain = boost::shared_dynamic_cast<XdmfDomain>(reader->read("TestXdmfReader1.xmf"));
boost::shared_ptr<XdmfWriter> writer2 = XdmfWriter::New("TestXdmfReader2.xmf"); boost::shared_ptr<XdmfWriter> writer2 = XdmfWriter::New("TestXdmfReader2.xmf");
writer2->setMode(XdmfWriter::DistributedHeavyData);
readDomain->accept(writer2); readDomain->accept(writer2);
// Compare two files for equality // Compare two files for equality
......
...@@ -5,7 +5,9 @@ ...@@ -5,7 +5,9 @@
int main(int argc, char* argv[]) int main(int argc, char* argv[])
{ {
boost::shared_ptr<XdmfWriter> writer = XdmfWriter::New("output.xmf"); boost::shared_ptr<XdmfWriter> writer = XdmfWriter::New("output.xmf");
assert(writer->getFilePath().compare("output.xmf") == 0);
writer->setLightDataLimit(10); writer->setLightDataLimit(10);
assert(writer->getLightDataLimit() == 10);
boost::shared_ptr<XdmfGrid> grid = XdmfTestDataGenerator::createHexahedron(); boost::shared_ptr<XdmfGrid> grid = XdmfTestDataGenerator::createHexahedron();
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment