ENH: Add XdmfHDF5Controller so that heavy data can be written and released...

ENH: Add XdmfHDF5Controller so that heavy data can be written and released before light data is written.
Modify XdmfWriter to accept an XdmfHDF5Writer to use for heavy data writing and allow users to specify
XML file name to write to.  Add a test that performs the write release write release workflow.

CMakeLists.txt

@@ -42,6 +42,7 @@ set(XdmfSources
 	XdmfGeometry
 	XdmfGeometryType
 	XdmfGrid
+	XdmfHDF5Controller
 	XdmfHDF5Writer
 	XdmfItem
 	XdmfItemProperty

Xdmf.i

@@ -14,6 +14,7 @@ swig -v -c++ -python -o XdmfPython.cpp Xdmf.i
 	#include <XdmfGeometry.hpp>
 	#include <XdmfGeometryType.hpp>
 	#include <XdmfGrid.hpp>
+  #include <XdmfHDF5Controller.hpp>
 	#include <XdmfHDF5Writer.hpp>
 	#include <XdmfItem.hpp>
 	#include <XdmfItemProperty.hpp>
@@ -53,6 +54,7 @@ namespace boost {
 %template(XdmfDomainPtr) boost::shared_ptr<XdmfDomain>;
 %template(XdmfGeometryPtr) boost::shared_ptr<XdmfGeometry>;
 %template(XdmfGridPtr) boost::shared_ptr<XdmfGrid>;
+%template(XdmfHDF5ControllerPtr) boost::shared_ptr<XdmfHDF5Controller>;
 %template(XdmfHDF5WriterPtr) boost::shared_ptr<XdmfHDF5Writer>;
 %template(XdmfItemPtr) boost::shared_ptr<XdmfItem>;
 %template(XdmfObjPtr) boost::shared_ptr<XdmfObject>;
@@ -84,6 +86,7 @@ SWIG_SHARED_PTR_DERIVED(XdmfGeometry, XdmfItem);
 SWIG_SHARED_PTR_DERIVED(XdmfGeometry, XdmfObject);
 SWIG_SHARED_PTR_DERIVED(XdmfGrid, XdmfItem);
 SWIG_SHARED_PTR_DERIVED(XdmfGrid, XdmfObject);
+SWIG_SHARED_PTR_DERIVED(XdmfHDF5Controller, XdmfObject);
 SWIG_SHARED_PTR_DERIVED(XdmfHDF5Writer, XdmfVisitor);
 SWIG_SHARED_PTR_DERIVED(XdmfHDF5Writer, Loki::BaseVisitor);
 SWIG_SHARED_PTR_DERIVED(XdmfHDF5Writer, XdmfObject);
@@ -103,6 +106,7 @@ SWIG_SHARED_PTR_DERIVED(XdmfWriter, XdmfObject);
 %include XdmfItemProperty.hpp
 %include XdmfVisitor.hpp
 
+%include XdmfHDF5Controller.hpp
 %include XdmfHDF5Writer.hpp
 %include XdmfWriter.hpp
 

XdmfArray.cpp

@@ -6,6 +6,7 @@
  */
 
 #include "XdmfArray.hpp"
+#include "XdmfHDF5Controller.hpp"
 #include "XdmfVisitor.hpp"
 #include <sstream>
 
@@ -346,6 +347,7 @@ XdmfArray::XdmfArray() :
 	mHaveArray(false),
 	mHaveArrayPointer(false),
 	mArrayPointerNumValues(0),
+	mHDF5Controller(boost::shared_ptr<XdmfHDF5Controller>()),
 	mTmpReserveSize(0)
 {
 	std::cout << "Created Array " << this << std::endl;
@@ -390,6 +392,16 @@ unsigned int XdmfArray::getCapacity() const
 	return 0;
 }
 
+boost::shared_ptr<XdmfHDF5Controller> XdmfArray::getHDF5Controller()
+{
+	return mHDF5Controller;
+}
+
+const boost::shared_ptr<const XdmfHDF5Controller> XdmfArray::getHDF5Controller() const
+{
+	return mHDF5Controller;
+}
+
 std::map<std::string, std::string> XdmfArray::getItemProperties() const
 {
 	std::map<std::string, std::string> arrayProperties;
@@ -419,6 +431,10 @@ unsigned int XdmfArray::getPrecision() const
 	{
 		return boost::apply_visitor(GetPrecision(), mArrayPointer);
 	}
+	else if(mHDF5Controller)
+	{
+		return mHDF5Controller->getPrecision();
+	}
 	return 0;
 }
 
@@ -432,6 +448,10 @@ unsigned int XdmfArray::getSize() const
 	{
 		return mArrayPointerNumValues;
 	}
+	else if(mHDF5Controller)
+	{
+		return mHDF5Controller->getSize();
+	}
 	return 0;
 }
 
@@ -445,6 +465,10 @@ std::string XdmfArray::getType() const
 	{
 		return boost::apply_visitor(GetType(), mArrayPointer);
 	}
+	else if(mHDF5Controller)
+	{
+		return mHDF5Controller->getType();
+	}
 	return "";
 }
 
@@ -482,6 +506,12 @@ void XdmfArray::internalizeArrayPointer()
 	}
 }
 
+void XdmfArray::release()
+{
+	releaseArray();
+	releaseArrayPointer();
+}
+
 void XdmfArray::releaseArray()
 {
 	boost::shared_ptr<std::vector<char> > emptyArray;
@@ -496,12 +526,6 @@ void XdmfArray::releaseArrayPointer()
 	mHaveArrayPointer = false;
 }
 
-void XdmfArray::releaseData()
-{
-	releaseArray();
-	releaseArrayPointer();
-}
-
 void XdmfArray::reserve(const unsigned int size)
 {
 	if(mHaveArrayPointer)
@@ -518,6 +542,11 @@ void XdmfArray::reserve(const unsigned int size)
 	}
 }
 
+void XdmfArray::setHDF5Controller(boost::shared_ptr<XdmfHDF5Controller> hdf5Controller)
+{
+	mHDF5Controller = hdf5Controller;
+}
+
 void XdmfArray::swap(boost::shared_ptr<XdmfArray> array)
 {
 	ArrayVariant tmpArray = array->mArray;

XdmfArray.hpp

 #ifndef XDMFARRAY_HPP_
 #define XDMFARRAY_HPP_
 
+// Forward Declarations
+class XdmfHDF5Controller;
+
 // Includes
 #include "XdmfItem.hpp"
 #include <boost/shared_array.hpp>
@@ -89,6 +92,20 @@ public:
 
 	std::string getItemTag() const;
 
+	/**
+	 * Get the hdf5 controller attached to this XdmfArray.
+	 *
+	 * @return the hdf5 controller attached to this XdmfArray.
+	 */
+	boost::shared_ptr<XdmfHDF5Controller> getHDF5Controller();
+
+	/**
+	 * Get the hdf5 controller attached to this XdmfArray (const version).
+	 *
+	 * @return the hdf5 controller attached to this XdmfArray.
+	 */
+	const boost::shared_ptr<const XdmfHDF5Controller> getHDF5Controller() const;
+
 	/**
 	 * Get the precision, in bytes, of the data type of this array.
 	 *
@@ -132,7 +149,7 @@ public:
 	 *
 	 * @return a void pointer to the first value stored in this array.
 	 */
-	virtual const void* const getValuesPointer() const;
+	virtual const void * const getValuesPointer() const;
 
 	/**
 	 * Get the values stored in this array as a string.
@@ -152,7 +169,7 @@ public:
 	/**
 	 * Release all data held by this XdmfArray.
 	 */
-	void releaseData();
+	void release();
 
 	/**
 	 * Set the capacity of the XdmfArray to at least size.
@@ -171,6 +188,13 @@ public:
 	template<typename T>
 	void resize(const unsigned int numValues, const T & val);
 
+	/**
+	 * Attach an hdf5 controller to this XdmfArray.
+	 *
+	 * @param hdf5Controller the hdf5 controller to attach to this XdmfArray.
+	 */
+	void setHDF5Controller(boost::shared_ptr<XdmfHDF5Controller> hdf5Controller);
+
 	/**
 	 * Sets the values of this array to the values stored in the arrayPointer array.  No copy is made.  Modifications to the array are
 	 * not permitted through the XdmfArray API.  Any calls through the XdmfArray API to modify the array (i.e. any non-const function)
@@ -310,6 +334,7 @@ private:
 	unsigned int mArrayPointerNumValues;
 	bool mHaveArray;
 	bool mHaveArrayPointer;
+	boost::shared_ptr<XdmfHDF5Controller> mHDF5Controller;
 	unsigned int mTmpReserveSize;
 };
 

XdmfHDF5Controller.cpp

+// Kenneth Leiter
+// Xdmf Smart Pointer Test
+
+#include "XdmfHDF5Controller.hpp"
+
+XdmfHDF5Controller::XdmfHDF5Controller(const std::string & dataSetPath, const int precision,
+	const int size, const std::string & type) :
+		mDataSetPath(dataSetPath),
+		mPrecision(precision),
+		mSize(size),
+		mType(type)
+{
+}
+
+XdmfHDF5Controller::~XdmfHDF5Controller()
+{
+}
+
+std::string XdmfHDF5Controller::getDataSetPath() const
+{
+	return mDataSetPath;
+}
+
+int XdmfHDF5Controller::getPrecision() const
+{
+	return mPrecision;
+}
+
+int XdmfHDF5Controller::getSize() const
+{
+	return mSize;
+}
+
+std::string XdmfHDF5Controller::getType() const
+{
+	return mType;
+}

XdmfHDF5Controller.hpp

+#ifndef XDMFHDF5CONTROLLER_HPP_
+#define XDMFHDF5CONTROLLER_HPP_
+
+// Includes
+#include <string>
+#include "XdmfObject.hpp"
+
+/**
+ * @brief Couples an XdmfArray with HDF5 data stored on disk.
+ *
+ * Serves as an interface between data stored in XdmfArrays and data stored on disk.  When an Xdmf file is read from or
+ * written to disk an XdmfHDF5Controller is attached to XdmfArrays.  This allows data to be released
+ * from memory but still be accessible or have its location written to light data.
+ */
+class XdmfHDF5Controller : public XdmfObject {
+
+public:
+
+	template <typename T> friend void boost::checked_delete(T * x);
+
+	/**
+	 * Create a new controller for an hdf5 data set on disk.
+	 *
+	 * @param dataSetPath the path to the hdf5 data set on disk.
+	 */
+	static boost::shared_ptr<XdmfHDF5Controller> New(const std::string & dataSetPath, const int precision,
+			const int size, const std::string & type)
+	{
+		boost::shared_ptr<XdmfHDF5Controller> p(new XdmfHDF5Controller(dataSetPath, precision, size, type));
+		return p;
+	}
+
+	/**
+	 * Get the path to the hdf5 data set on disk owned by this controller.
+	 *
+	 * @return a std::string to the hdf5 data set location on disk.
+	 */
+	std::string getDataSetPath() const;
+
+	/**
+	 * Get the precision of the hdf5 data set owned by this controller.
+	 *
+	 * @return a int containing the precision of the hdf5 data set.
+	 */
+	int getPrecision() const;
+
+	/**
+	 * Get the size of the hdf5 data set owned by this controller.
+	 *
+	 * @return a int containing the size of the hdf5 data set.
+	 */
+	int getSize() const;
+
+	/**
+	 * Get the data type of the hdf5 data set owned by this controller.
+	 *
+	 * @return a std::string containing the name of the data type of the hdf5 data set.
+	 */
+	std::string getType() const;
+
+protected:
+
+	XdmfHDF5Controller(const std::string & dataSetPath, const int precision, const int size,
+			const std::string & type);
+	virtual ~XdmfHDF5Controller();
+
+private:
+
+	XdmfHDF5Controller(const XdmfHDF5Controller & hdf5Controller);  // Not implemented.
+	void operator=(const XdmfHDF5Controller & hdf5Controller);  // Not implemented.
+
+	std::string mDataSetPath;
+	int mPrecision;
+	int mSize;
+	std::string mType;
+};
+
+#endif /* XDMFHDF5CONTROLLER_HPP_ */

XdmfHDF5Writer.cpp

@@ -4,7 +4,7 @@
 #include <hdf5.h>
 #include <sstream>
 #include "XdmfArray.hpp"
-#include "XdmfItem.hpp"
+#include "XdmfHDF5Controller.hpp"
 #include "XdmfHDF5Writer.hpp"
 
 /**
@@ -14,7 +14,7 @@ class XdmfHDF5Writer::XdmfHDF5WriterImpl {
 
 public:
 
-	XdmfHDF5WriterImpl(std::string & hdf5FilePath) :
+	XdmfHDF5WriterImpl(const std::string & hdf5FilePath) :
 		mHDF5Handle(H5Fcreate(hdf5FilePath.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)),
 		mHeavyFileName(hdf5FilePath),
 		mLastWrittenDataSet("")
@@ -97,7 +97,7 @@ public:
 	}
 };
 
-XdmfHDF5Writer::XdmfHDF5Writer(std::string & hdf5FilePath) :
+XdmfHDF5Writer::XdmfHDF5Writer(const std::string & hdf5FilePath) :
 	mImpl(new XdmfHDF5WriterImpl(hdf5FilePath))
 {
 	std::cout << "Created XdmfHDF5Writer " << this << std::endl;
@@ -117,6 +117,8 @@ std::string XdmfHDF5Writer::getLastWrittenDataSet() const
 void XdmfHDF5Writer::visit(XdmfArray & array, boost::shared_ptr<Loki::BaseVisitor> visitor)
 {
 	hid_t datatype = -1;
+	mImpl->mLastWrittenDataSet = "";
+
 	if(array.mHaveArray)
 	{
 		datatype = boost::apply_visitor(GetHDF5Type(), array.mArray);
@@ -125,6 +127,10 @@ void XdmfHDF5Writer::visit(XdmfArray & array, boost::shared_ptr<Loki::BaseVisito
 	{
 		datatype = boost::apply_visitor(GetHDF5Type(), array.mArrayPointer);
 	}
+	else if(array.mHDF5Controller)
+	{
+		mImpl->mLastWrittenDataSet = array.mHDF5Controller->getDataSetPath();
+	}
 
 	if(datatype != -1)
 	{
@@ -140,13 +146,12 @@ void XdmfHDF5Writer::visit(XdmfArray & array, boost::shared_ptr<Loki::BaseVisito
 		status = H5Dclose(dataset);
 		status = H5Sclose(dataspace);
 
-		std::stringstream writtenDataSetName;
-		writtenDataSetName << mImpl->mHeavyFileName << ":" << dataSetName.str();
-		mImpl->mLastWrittenDataSet = writtenDataSetName.str();
+		std::stringstream writtenDataSetPath;
+		writtenDataSetPath << mImpl->mHeavyFileName << ":" << dataSetName.str();
+		mImpl->mLastWrittenDataSet = writtenDataSetPath.str();
 		mImpl->mDataSetId++;
-	}
-	else
-	{
-		mImpl->mLastWrittenDataSet = "";
+
+		boost::shared_ptr<XdmfHDF5Controller> newDataSetController = XdmfHDF5Controller::New(writtenDataSetPath.str(), array.getPrecision(), array.getSize(), array.getType());
+		array.setHDF5Controller(newDataSetController);
 	}
 }

XdmfHDF5Writer.hpp

@@ -20,7 +20,13 @@ class XdmfHDF5Writer : public XdmfVisitor,
 public:
 
 	template <typename T> friend void boost::checked_delete(T * x);
-	static boost::shared_ptr<XdmfHDF5Writer> New(std::string hdf5FilePath)
+
+	/**
+	 * Construct XdmfHDF5Writer
+	 *
+	 * @param hdf5FilePath the location of the hdf5 file to output to on disk.
+	 */
+	static boost::shared_ptr<XdmfHDF5Writer> New(const std::string & hdf5FilePath)
 	{
 		boost::shared_ptr<XdmfHDF5Writer> p(new XdmfHDF5Writer(hdf5FilePath));
 		return p;
@@ -43,12 +49,7 @@ public:
 
 protected:
 
-	/**
-	 * Construct XdmfHDF5Writer
-	 *
-	 * @param hdf5FilePath the location of the hdf5 file to output to on disk.
-	 */
-	XdmfHDF5Writer(std::string & hdf5FilePath);
+	XdmfHDF5Writer(const std::string & hdf5FilePath);
 	virtual ~XdmfHDF5Writer();
 
 private:

XdmfWriter.cpp

@@ -15,11 +15,12 @@ class XdmfWriter::XdmfWriterImpl {
 
 public:
 
-	XdmfWriterImpl() :
-		mHDF5Writer(XdmfHDF5Writer::New("output.h5")),
+	XdmfWriterImpl(const std::string & xmlFilePath, boost::shared_ptr<XdmfHDF5Writer> hdf5Writer) :
+		mHDF5Writer(hdf5Writer),
 		mLightDataLimit(100),
 		mXMLDocument(xmlNewDoc((xmlChar*)"1.0")),
-		mXMLCurrentNode(xmlNewNode(NULL, (xmlChar*)"Xdmf"))
+		mXMLCurrentNode(xmlNewNode(NULL, (xmlChar*)"Xdmf")),
+		mXMLFilePath(xmlFilePath)
 	{
 		xmlDocSetRootElement(mXMLDocument, mXMLCurrentNode);
 	};
@@ -30,15 +31,31 @@ public:
 		xmlCleanupParser();
 	};
 	boost::shared_ptr<XdmfHDF5Writer> mHDF5Writer;
-	std::string mHeavyFileName;
 	unsigned int mLightDataLimit;
 	xmlDocPtr mXMLDocument;
 	xmlNodePtr mXMLCurrentNode;
-
+	std::string mXMLFilePath;
 };
 
-XdmfWriter::XdmfWriter() :
-	mImpl(new XdmfWriterImpl())
+XdmfWriter::XdmfWriter(const std::string & xmlFilePath)
+{
+	std::stringstream heavyFileName;
+	size_t extension = xmlFilePath.rfind(".");
+	if(extension != std::string::npos)
+	{
+		heavyFileName << xmlFilePath.substr(0, extension) << ".h5";
+	}
+	else
+	{
+		heavyFileName << xmlFilePath << ".h5";
+	}
+	boost::shared_ptr<XdmfHDF5Writer> hdf5Writer = XdmfHDF5Writer::New(heavyFileName.str());
+	mImpl = new XdmfWriterImpl(xmlFilePath, hdf5Writer);
+	std::cout << "Created XdmfWriter " << this << std::endl;
+}
+
+XdmfWriter::XdmfWriter(const std::string & xmlFilePath, boost::shared_ptr<XdmfHDF5Writer> hdf5Writer) :
+	mImpl(new XdmfWriterImpl(xmlFilePath, hdf5Writer))
 {
 	std::cout << "Created XdmfWriter " << this << std::endl;
 }
@@ -67,7 +84,7 @@ void XdmfWriter::visit(XdmfArray & array, boost::shared_ptr<Loki::BaseVisitor> v
 	mImpl->mXMLCurrentNode = mImpl->mXMLCurrentNode->children;
 
 	std::stringstream xmlTextValues;
-	if(array.getSize() > mImpl->mLightDataLimit)
+	if(array.getHDF5Controller() || array.getSize() > mImpl->mLightDataLimit)
 	{
 		mImpl->mHDF5Writer->visit(array, mImpl->mHDF5Writer);
 		xmlTextValues << mImpl->mHDF5Writer->getLastWrittenDataSet();

XdmfWriter.hpp

@@ -3,6 +3,7 @@
 
 // Forward Declarations
 class XdmfArray;
+class XdmfHDF5Writer;
 
 // Includes
 #include "XdmfVisitor.hpp"
@@ -20,7 +21,34 @@ class XdmfWriter : public XdmfVisitor,
 
 public:
 
-	XdmfNewMacro(XdmfWriter);
+	template <typename T> friend void boost::checked_delete(T * x);
+
+	/**
+	 * Create a new XdmfWriter to write Xdmf data to disk.  This will create its own hdf5 writer based on the xmlFileName.
+	 * For example, if supplied "output.xmf" the created hdf5 writer would write to file "output.h5".
+	 *
+	 * @param xmlFilePath the path to the xml file to write to.
+	 * @return the new XdmfWriter.
+	 */
+	static boost::shared_ptr<XdmfWriter> New(const std::string & xmlFilePath)
+	{
+		boost::shared_ptr<XdmfWriter> p(new XdmfWriter(xmlFilePath));
+		return p;
+	}
+
+	/**
+	 * Create a new XdmfWriter to write Xdmf data to disk.  This will utilize the supplied hdf5Writer to write any
+	 * heavy data to disk.
+	 *
+	 * @param xmlFilePath the path to the xml file to write to.
+	 * @param hdf5Writer the heavy data writer to use when writing.
+	 * @return the new XdmfWriter.
+	 */
+	static boost::shared_ptr<XdmfWriter> New(const std::string & xmlFilePath, boost::shared_ptr<XdmfHDF5Writer> hdf5Writer)
+	{
+		boost::shared_ptr<XdmfWriter> p(new XdmfWriter(xmlFilePath, hdf5Writer));
+		return p;
+	}
 
 	/**
 	 * Get the number of values that this writer writes to light data (XML) before switching to a heavy data format.
@@ -54,7 +82,8 @@ public:
 
 protected:
 
-	XdmfWriter();
+	XdmfWriter(const std::string & xmlFilePath);
+	XdmfWriter(const std::string & xmlFilePath, boost::shared_ptr<XdmfHDF5Writer> hdf5Writer);
 	virtual ~XdmfWriter();
 
 private:

tests/Cxx/CMakeLists.txt

@@ -8,6 +8,7 @@ set(XdmfCxxTests
 	TestXdmfTopology
 	TestXdmfVisitorValueCounter
 	TestXdmfWriter
+	TestXdmfWriterHDF5ThenXML
 )
 
 foreach(test ${XdmfCxxTests})

tests/Cxx/TestXdmfArray.cpp

@@ -194,7 +194,7 @@ int main(int argc, char* argv[])
 	/**
 	 * Swap values from a shared vector
 	 */
-	array7->releaseData();
+	array7->release();
 	array7->swap(values2);
 	assert(storedValues2->size() == 0);
 	assert(array7->getSize() == 6);
@@ -204,7 +204,7 @@ int main(int argc, char* argv[])
 	/**
 	 * Swap values from an XdmfArray (with copy)
 	 */
-	array7->releaseData();
+	array7->release();
 	array7->swap(array4);
 	assert(array4->getSize() == 0);
 	assert(array7->getSize() == 3);
@@ -214,7 +214,7 @@ int main(int argc, char* argv[])
 	 */
 	array5->setValues(&values[1], 3, 0);
 	assert(array5->getSize() == 3);
-	array7->releaseData();
+	array7->release();
 	array7->swap(array5);
 	assert(array5->getSize() == 0);
 	assert(array7->getSize() == 3);

tests/Cxx/TestXdmfWriter.cpp

@@ -4,7 +4,7 @@
 
 int main(int argc, char* argv[])
 {
-	boost::shared_ptr<XdmfWriter> writer = XdmfWriter::New();
+	boost::shared_ptr<XdmfWriter> writer = XdmfWriter::New("output.xmf");
 	writer->setLightDataLimit(10);
 
 	boost::shared_ptr<XdmfGrid> grid = XdmfTestDataGenerator::createHexahedron();

tests/Cxx/TestXdmfWriterHDF5ThenXML.cxx

+#include "XdmfHDF5Writer.hpp"
+#include "XdmfWriter.hpp"
+
+#include "XdmfTestDataGenerator.hpp"
+
+int main(int argc, char* argv[])
+{
+	boost::shared_ptr<XdmfGrid> grid = XdmfTestDataGenerator::createHexahedron();
+
+	// First write and release heavy data
+	boost::shared_ptr<XdmfHDF5Writer> hdf5Writer = XdmfHDF5Writer::New("output.h5");
+	grid->getGeometry()->accept(hdf5Writer);
+	grid->getGeometry()->getArray()->release();
+
+	grid->getTopology()->accept(hdf5Writer);
+	grid->getTopology()->getArray()->release();
+
+	for(int i=0; i<grid->getNumberOfAttributes(); ++i)
+	{
+		grid->getAttribute(i)->accept(hdf5Writer);
+		grid->getAttribute(i)->getArray()->release();
+	}
+
+	// Now insert into domain and write light data
+	boost::shared_ptr<XdmfDomain> domain = XdmfDomain::New();
+	domain->insert(grid);
+
+	boost::shared_ptr<XdmfWriter> writer = XdmfWriter::New("output.xmf", hdf5Writer);
+	writer->setLightDataLimit(10);
+	domain->accept(writer);
+
+	return 0;
+}