Commit 4b1c1d0b authored by Andrew J. Burns (Cont's avatar Andrew J. Burns (Cont

changes for hdf5 file splitting and combining xmf files

parent f91048a3
......@@ -38,11 +38,18 @@
* Xdmf supports the following attribute centers:
*
* Example of use:
* //Assuming that exampleAttribute is a shared pointer to an XdmfAttribute with a set center
* if (exampleAttribute->getCenter() == XdmfAttributeCenter::Grid())
* {
* //do whatever is to be done if the center is grid
* }
*
* C++
*
* @dontinclude ExampleXdmfAttribute.cpp
* @skipline exampleCenter
* @until }
*
* Python
*
* @dontinclude XdmfExampleAttribute.py
* @skipline exampleCenter
* @until do
*
* Grid
* Cell
......
......@@ -39,11 +39,17 @@
*
* Example of use:
*
* //Assuming that exampleAttribute is a shared pointer to an XdmfAttribute that has had its type set
* if (exampleAttribute->getType() == XdmfAttributeType:Scalar())
* {
* //do whatever is to be done if the attribute is a scalar
* }
* C++
*
* @dontinclude ExampleXdmfAttribute.cpp
* @skipline exampleType
* @until }
*
* Python
*
* @dontinclude XdmfExampleAttribute.py
* @skipline exampleType
* @until do
*
* Xdmf supports the following attribute types:
* NoAttributeType
......
......@@ -39,11 +39,17 @@
*
* Example of use:
*
* //Assuming that exampleGeometry is a shared pointer to an XdmfGeometry with its type set
* if (exampleGeometry->getType() == XdmfGeometry::XYZ())
* {
* //do whatever is to be done if the geometry is xyz
* }
* C++
*
* @dontinclude ExampleXdmfGeometryType.cpp
* @skipline getType
* @until }
*
* Python
*
* @dontinclude XdmfExampleGeometryType.py
* @skipline getType
* @until do
*
* Xdmf supports the following geometry types:
* NoGeometryType
......
......@@ -39,11 +39,17 @@
*
* Example of use:
*
* //Assuming that exampleCollection is a shared pointer to an XmdfGridCollection with a type that has been set
* if (exampleCollection->getType() == XdmfGridCollectionType::Temporal())
* {
* //do whatever is to be done if the grid collection is temporal
* }
* C++
*
* @dontinclude ExampleXdmfGridCollection.cpp
* @skipline exampleType
* @until }
*
* Python
*
* @dontinclude XdmfExampleGridCollection.py
* @skipline exampleType
* @until do
*
* Xdmf supports the following collection types:
* NoCollectionType
......
......@@ -208,9 +208,23 @@ XdmfMap::populateItem(const std::map<std::string, std::string> & itemProperties,
}
}
else {
mRemoteTaskIdsController = arrayVector[0]->getHeavyDataController();
mLocalNodeIdsController = arrayVector[1]->getHeavyDataController();
mRemoteLocalNodeIdsController = arrayVector[2]->getHeavyDataController();
//this needs to be changed to account for controllers being a vector now
mRemoteTaskIdsControllers.clear();
for (int i = 0; i < arrayVector[0]->getNumberHeavyDataControllers(); i++)
{
mRemoteTaskIdsControllers.push_back(arrayVector[0]->getHeavyDataController(i));
}
mLocalNodeIdsControllers.clear();
for (int i = 0; i < arrayVector[1]->getNumberHeavyDataControllers(); i++)
{
mLocalNodeIdsControllers.push_back(arrayVector[1]->getHeavyDataController(i));
}
mRemoteLocalNodeIdsControllers.clear();
for (int i = 0; i < arrayVector[2]->getNumberHeavyDataControllers(); i++)
{
mRemoteLocalNodeIdsControllers.push_back(arrayVector[2]->getHeavyDataController(i));
}
}
}
}
......@@ -218,14 +232,30 @@ XdmfMap::populateItem(const std::map<std::string, std::string> & itemProperties,
void
XdmfMap::read()
{
if(mLocalNodeIdsController &&
mRemoteTaskIdsController &&
mRemoteLocalNodeIdsController) {
if(!(mLocalNodeIdsController->getSize() ==
mRemoteTaskIdsController->getSize() &&
mLocalNodeIdsController->getSize() ==
mRemoteLocalNodeIdsController->getSize())) {
if(mLocalNodeIdsControllers.size() > 0 &&
mRemoteTaskIdsControllers.size() > 0 &&
mRemoteLocalNodeIdsControllers.size() > 0) {
int localNodeCount = 0;
for (int i = 0; i< mLocalNodeIdsControllers.size(); i++)
{
localNodeCount += mLocalNodeIdsControllers[i]->getSize();
}
int remoteTaskCount = 0;
for (int i = 0; i< mRemoteTaskIdsControllers.size(); i++)
{
remoteTaskCount += mRemoteTaskIdsControllers[i]->getSize();
}
int remoteNodeCount = 0;
for (int i = 0; i< mRemoteLocalNodeIdsControllers.size(); i++)
{
remoteNodeCount += mRemoteLocalNodeIdsControllers[i]->getSize();
}
if(!(localNodeCount ==
remoteTaskCount &&
localNodeCount ==
remoteNodeCount)){
XdmfError::message(XdmfError::FATAL,
"Arrays must be of equal size in XdmfMap::read");
}
......@@ -234,9 +264,27 @@ XdmfMap::read()
shared_ptr<XdmfArray> localNodeIds = XdmfArray::New();
shared_ptr<XdmfArray> remoteLocalNodeIds = XdmfArray::New();
mRemoteTaskIdsController->read(remoteTaskIds.get());
mLocalNodeIdsController->read(localNodeIds.get());
mRemoteLocalNodeIdsController->read(remoteLocalNodeIds.get());
mRemoteTaskIdsControllers[0]->read(remoteTaskIds.get());
for (int i = 1; i < mRemoteTaskIdsControllers.size(); i++)
{
shared_ptr<XdmfArray> tempArray = XdmfArray::New();
mRemoteTaskIdsControllers[i]->read(tempArray.get());
remoteTaskIds->insert(remoteTaskIds->getSize(), tempArray, 0, tempArray->getSize());
}
mLocalNodeIdsControllers[0]->read(localNodeIds.get());
for (int i = 1; i < mLocalNodeIdsControllers.size(); i++)
{
shared_ptr<XdmfArray> tempArray = XdmfArray::New();
mLocalNodeIdsControllers[i]->read(tempArray.get());
localNodeIds->insert(localNodeIds->getSize(), tempArray, 0, tempArray->getSize());
}
mRemoteLocalNodeIdsControllers[0]->read(remoteLocalNodeIds.get());
for (int i = 1; i < mRemoteLocalNodeIdsControllers.size(); i++)
{
shared_ptr<XdmfArray> tempArray = XdmfArray::New();
mRemoteLocalNodeIdsControllers[i]->read(tempArray.get());
remoteLocalNodeIds->insert(remoteLocalNodeIds->getSize(), tempArray, 0, tempArray->getSize());
}
for(unsigned int i=0; i<remoteTaskIds->getSize(); ++i) {
const unsigned int remoteTaskId = remoteTaskIds->getValue<task_id>(i);
......@@ -248,6 +296,7 @@ XdmfMap::read()
}
}
void
XdmfMap::release()
{
......@@ -255,20 +304,35 @@ XdmfMap::release()
}
void
XdmfMap::setHeavyDataControllers(shared_ptr<XdmfHeavyDataController> remoteTaskIdsController,
shared_ptr<XdmfHeavyDataController> localNodeIdsController,
shared_ptr<XdmfHeavyDataController> remoteLocalNodeIdsController)
XdmfMap::setHeavyDataControllers(std::vector<shared_ptr<XdmfHeavyDataController> > remoteTaskIdsControllers,
std::vector<shared_ptr<XdmfHeavyDataController> > localNodeIdsControllers,
std::vector<shared_ptr<XdmfHeavyDataController> > remoteLocalNodeIdsControllers)
{
if(!(localNodeIdsController->getSize() ==
remoteTaskIdsController->getSize() &&
localNodeIdsController->getSize() ==
remoteLocalNodeIdsController->getSize()))
int localNodeCount = 0;
for (int i = 0; i< localNodeIdsControllers.size(); i++)
{
localNodeCount += localNodeIdsControllers[i]->getSize();
}
int remoteTaskCount = 0;
for (int i = 0; i< remoteTaskIdsControllers.size(); i++)
{
remoteTaskCount += remoteTaskIdsControllers[i]->getSize();
}
int remoteNodeCount = 0;
for (int i = 0; i< remoteLocalNodeIdsControllers.size(); i++)
{
remoteNodeCount += remoteLocalNodeIdsControllers[i]->getSize();
}
if(!(localNodeCount ==
remoteTaskCount &&
localNodeCount ==
remoteNodeCount))
XdmfError::message(XdmfError::FATAL,
"Arrays must be of equal size in "
"XdmfMap::setHeavyDataControllers");
mRemoteTaskIdsController = remoteTaskIdsController;
mLocalNodeIdsController = localNodeIdsController;
mRemoteLocalNodeIdsController = remoteLocalNodeIdsController;
mRemoteTaskIdsControllers = remoteTaskIdsControllers;
mLocalNodeIdsControllers = localNodeIdsControllers;
mRemoteLocalNodeIdsControllers = remoteLocalNodeIdsControllers;
}
void
......@@ -311,15 +375,37 @@ XdmfMap::traverse(const shared_ptr<XdmfBaseVisitor> visitor)
}
}
remoteTaskIds->setHeavyDataController(mRemoteTaskIdsController);
localNodeIds->setHeavyDataController(mLocalNodeIdsController);
remoteLocalNodeIds->setHeavyDataController(mRemoteLocalNodeIdsController);
for (int i = 0; i < mRemoteTaskIdsControllers.size(); i++)
{
remoteTaskIds->insert(mRemoteTaskIdsControllers[i]);
}
for (int i = 0; i < mLocalNodeIdsControllers.size(); i++)
{
localNodeIds->insert(mLocalNodeIdsControllers[i]);
}
for (int i = 0; i < mRemoteLocalNodeIdsControllers.size(); i++)
{
remoteLocalNodeIds->insert(mRemoteLocalNodeIdsControllers[i]);
}
remoteTaskIds->accept(visitor);
localNodeIds->accept(visitor);
remoteLocalNodeIds->accept(visitor);
mLocalNodeIdsController = localNodeIds->getHeavyDataController();
mRemoteTaskIdsController = remoteTaskIds->getHeavyDataController();
mRemoteLocalNodeIdsController = remoteLocalNodeIds->getHeavyDataController();
mLocalNodeIdsControllers.clear();
mRemoteTaskIdsControllers.clear();
mRemoteLocalNodeIdsControllers.clear();
for (int i = 0; i < remoteTaskIds->getNumberHeavyDataControllers(); i++)
{
mRemoteTaskIdsControllers.push_back(remoteTaskIds->getHeavyDataController(i));
}
for (int i = 0; i < localNodeIds->getNumberHeavyDataControllers(); i++)
{
mLocalNodeIdsControllers.push_back(localNodeIds->getHeavyDataController(i));
}
for (int i = 0; i < remoteLocalNodeIds->getNumberHeavyDataControllers(); i++)
{
mRemoteLocalNodeIdsControllers.push_back(remoteLocalNodeIds->getHeavyDataController(i));
}
}
......@@ -306,17 +306,17 @@ public:
* @skipline hdf5FilePath
* @until setHeavyDataController
*
* @param remoteTaskIdsController an XdmfHeavyDataController to the remote
* @param remoteTaskIdsControllers a vector of XdmfHeavyDataControllers to the remote
* task ids dataset.
* @param localNodeIdsController an XdmfHeavyDataController to the local
* @param localNodeIdsControllers a vector of XdmfHeavyDataControllers to the local
* node ids dataset.
* @param remoteLocalNodeIdsController an XdmfHeavyDataController to the
* @param remoteLocalNodeIdsControllers a vector of XdmfHeavyDataControllers to the
* remote local node ids dataset.
*/
void
setHeavyDataControllers(shared_ptr<XdmfHeavyDataController> remoteTaskIdsController,
shared_ptr<XdmfHeavyDataController> localNodeIdsController,
shared_ptr<XdmfHeavyDataController> remoteLocalNodeIdsController);
setHeavyDataControllers(std::vector<shared_ptr<XdmfHeavyDataController> > remoteTaskIdsControllers,
std::vector<shared_ptr<XdmfHeavyDataController> > localNodeIdsControllers,
std::vector<shared_ptr<XdmfHeavyDataController> > remoteLocalNodeIdsControllers);
/**
* Set the boundary communicator map.
......@@ -380,12 +380,12 @@ private:
XdmfMap(const XdmfMap & map); // Not implemented.
void operator=(const XdmfMap & map); // Not implemented.
shared_ptr<XdmfHeavyDataController> mLocalNodeIdsController;
std::vector<shared_ptr<XdmfHeavyDataController> > mLocalNodeIdsControllers;
// remoteTaskId | localNodeId | remoteLocalNodeId
std::map<task_id, node_id_map > mMap;
std::string mName;
shared_ptr<XdmfHeavyDataController> mRemoteLocalNodeIdsController;
shared_ptr<XdmfHeavyDataController> mRemoteTaskIdsController;
std::vector<shared_ptr<XdmfHeavyDataController> > mRemoteLocalNodeIdsControllers;
std::vector<shared_ptr<XdmfHeavyDataController> > mRemoteTaskIdsControllers;
};
......
......@@ -14,11 +14,17 @@
*
* Example of use:
*
* //Assuming that exampleSet is a shared pointer to an XdmfSet with its type set
* if (exampleSet->getType() == XdmfSetType::Node())
* {
* //Do whatever is to be done if the set is a node
* }
* C++
*
* @dontinclude ExampleXdmfSet.cpp
* @skipline exampleType
* @until }
*
* Python
*
* @dontinclude XdmfExampleSet.py
* @skipline exampleType
* @until do
*
* Xdmf supports the following set types:
* NoSetType
......
......@@ -39,11 +39,17 @@
*
* Example of use:
*
* //Assuming that exampleTopology is a shared pointer to an XdmfTopology with its type set
* if (exampleTopology->getType() == XdmfTopologyType::Triangle())
* {
* //Do whatever is to be done if the type is Triangle
* }
* C++
*
* @dontinclude ExampleXdmfTopologyType.cpp
* @skipline getType
* @until }
*
* Python
*
* @dontinclude XdmfExampleTopologyType.py
* @skipline getType
* @until do
*
* Xdmf supports the following topology types:
* NoTopologyType
......
This diff is collapsed.
......@@ -117,6 +117,7 @@ public:
virtual ~XdmfArray();
LOKI_DEFINE_VISITABLE(XdmfArray, XdmfItem);
XDMF_CHILDREN(XdmfArray, XdmfHeavyDataController, HeavyDataController, Name);
static const std::string ItemTag;
/**
......@@ -245,46 +246,6 @@ public:
*/
std::string getDimensionsString() const;
/**
* Get the heavy data controller attached to this array.
*
* Example of use:
*
* C++
*
* @dontinclude ExampleXdmfArray.cpp
* @skipline Assuming
* @skipline exampleController
*
* Python
*
* @dontinclude XdmfExampleArray.py
* @skipline Assuming
* @skipline exampleController
*
* @return the heavy data controller attached to this array.
*/
shared_ptr<XdmfHeavyDataController> getHeavyDataController();
/**
* Get the heavy data controller attached to this array (const
* version).
*
* Example of use:
*
* C++
*
* @dontinclude ExampleXdmfArray.cpp
* @skipline Assuming
* @skipline exampleControllerConst
*
* Python: Doesn't support a constant version of this function
*
* @return the heavy data controller attached to this array.
*/
shared_ptr<const XdmfHeavyDataController>
getHeavyDataController() const;
std::map<std::string, std::string> getItemProperties() const;
std::string getItemTag() const;
......@@ -718,6 +679,84 @@ public:
template <typename T>
void pushBack(const T & value);
/**
* Get the first heavy data controller attached to this array.
*
* Example of use:
*
* C++
*
* @dontinclude ExampleXdmfArray.cpp
* @skipline Assuming
* @skipline exampleController
*
* Python
*
* @dontinclude XdmfExampleArray.py
* @skipline Assuming
* @skipline exampleController
*
* @return the heavy data controller attached to this array.
*/
shared_ptr<XdmfHeavyDataController>
getHeavyDataController();
/**
* Get the first heavy data controller attached to this array. (const version)
*
* Example of use:
*
* C++
*
* @dontinclude ExampleXdmfArray.cpp
* @skipline Assuming
* @skipline exampleControllerConst
*
* Python: Doesn't support a constant version of this function
*
* @return the heavy data controller attached to this array.
*/
shared_ptr<const XdmfHeavyDataController>
getHeavyDataController() const;
/**
* Replace all controllers attached to this array with the controller provided.
*
* Example of use:
*
* C++
*
* @dontinclude ExampleXdmfArray.cpp
* @skipline Assuming
* @skipline exampleController
* @until setHeaveyDataController
*
* Python
*
* @dontinclude XdmfExampleArray.py
* @skipline Assuming
* @skipline exampleController
* @until setHeaveyDataController
*
* @param newController the heavy data controller to attach to this array.
*/
void
setHeavyDataController(shared_ptr<XdmfHeavyDataController> newController);
/**
* Read data from disk into memory.
*
......@@ -796,7 +835,7 @@ public:
* @skipline newSize
* @skipline initArray
* @skipline insert
* @until #
* @until //
* @skipline newSize
* @until 4}
*
......@@ -856,31 +895,6 @@ public:
void resize(const std::vector<unsigned int> & dimensions,
const T & value = 0);
/**
* Attach an heavy data controller to this array.
*
* Example of use:
*
* C++
*
* @dontinclude ExampleXdmfArray.cpp
* @skipline Assuming
* @skipline exampleController
* @until setHeaveyDataController
*
* Python
*
* @dontinclude XdmfExampleArray.py
* @skipline Assuming
* @skipline exampleController
* @until setHeaveyDataController
*
* @param heavyDataController the heavy data controller to attach to
* this array.
*/
void
setHeavyDataController(const shared_ptr<XdmfHeavyDataController> heavyDataController);
/**
* Set the name of the array.
*
......@@ -1135,7 +1149,6 @@ private:
ArrayVariant mArray;
unsigned int mArrayPointerNumValues;
std::vector<unsigned int> mDimensions;
shared_ptr<XdmfHeavyDataController> mHeavyDataController;
std::string mName;
unsigned int mTmpReserveSize;
};
......
......@@ -37,11 +37,18 @@
* static methods in the class, i.e. XdmfArrayType::Int32().
*
* Example of use:
* //Assuming that exampleArray is a shared pointer to an XdmfArray object that has been filled with data
* if (XdmfArrayType::Int8() == exampleArray->getArrayType())
* {
* //do whatever is to be done with in the case that the array type is Int8
* }
*
* C++
*
* @dontinclude ExampleXdmfArrayType.cpp
* @skipline XdmfArrayType
* @until }
*
* Python
*
* @dontinclude XdmfExampleArrayType.py
* @skipline exampleType
* @until do
*
* Xdmf supports the following attribute types:
* Uninitialized
......@@ -85,19 +92,13 @@ public:
*
* C++
*
* @code {.cpp}
* unsigned int dataSize = XdmfArrayType::Int8()->getElementSize();
* //The number of bytes in an Int8 will be stored in the dataSize variable
* @endcode
* @dontinclude ExampleXdmfArrayType.cpp
* @skipline getElementSize
*
* Python
*
* @code {.py}
* dataSize = XdmfArrayType.Int8().getElementSize()
* '''
* The number of bytes in an Int8 will be stored in the dataSize variable
* '''
* @endcode
* @dontinclude XdmfExampleArrayType.py
* @skipline getElementSize
*
* @return the data size, in bytes.
*/
......@@ -110,19 +111,13 @@ public:
*
* C++
*
* @code {.cpp}
* std::string dataName = XdmfArrayType::Int8()->getName();
* //The name of the Int8 data type will be stored in the dataName variable
* @endcode
* @dontinclude ExampleXdmfArrayType.cpp
* @skipline getName
*
* Python
*
* @code {.py}
* dataName = XdmfArrayType.Int8().getName()
* '''
* The name of the Int8 data type will be stored in the dataName variable
* '''
* @endcode
* @dontinclude XdmfExampleArrayType.py
* @skipline getName
*
* @return the name of the data type.
*/
......
......@@ -188,6 +188,8 @@ public:
readSingleNode(const xmlNodePtr currNode,
std::vector<shared_ptr<XdmfItem> > & myItems)
{
std::map<xmlNodePtr, shared_ptr<XdmfItem> >::const_iterator iter =
mXPathMap.find(currNode);
if(iter != mXPathMap.end()) {
......@@ -198,15 +200,21 @@ public:
xmlNodePtr childNode = currNode->children;
if (XdmfArray::ItemTag.compare((char *)currNode->name) == 0) {
unsigned int childContentIndex = 0;
while(childNode != NULL) {
if(childNode->type == XML_TEXT_NODE && childNode->content) {
const char * content = (char*)childNode->content;
// determine if content is whitespace
bool whitespace = true;
const char * contentPtr = content;
//step through to end of pointer
while(contentPtr != NULL) {
//if not a whitespace character, break
if(!isspace(*contentPtr++)) {
whitespace = false;
break;
......@@ -214,15 +222,38 @@ public:
}
if(!whitespace) {
itemProperties.insert(std::make_pair("Content", content));
itemProperties.insert(std::make_pair("XMLDir", mXMLDir));
break;
if (childContentIndex == 0) {
itemProperties.insert(std::make_pair("XMLDir", mXMLDir));
}
//split the content based on "|" characters
size_t barSplit = 0;
std::string splitString(content);
std::string subcontent;
while (barSplit != std::string::npos) {
barSplit = 0;
barSplit = splitString.find_first_of("|", barSplit);
if (barSplit == std::string::npos) {
subcontent = splitString;
}
else {
subcontent = splitString.substr(0, barSplit);
splitString = splitString.substr(barSplit+1);
barSplit++;
}
std::stringstream contentString;
contentString << "Content" << childContentIndex;
itemProperties.insert(std::make_pair(contentString.str(), subcontent));
childContentIndex++;
}
}
}
childNode = childNode->next;
//childContentIndex++;
}
}
xmlAttrPtr currAttribute = currNode->properties;
while(currAttribute != NULL) {
itemProperties.insert(std::make_pair((char *)currAttribute->name,
......
This diff is collapsed.
......@@ -25,6 +25,7 @@
#define XDMFHDF5WRITER_HPP_
// Forward Declarations
class XdmfArray;
class XdmfArrayType;
class XdmfHDF5Controller;
......@@ -77,6 +78,51 @@ public:
virtual ~XdmfHDF5Writer();
/**
* Sets the file size limit of the HDF5 files produced by the writer in MB. Overflow is pushed to a new HDF5 file.
*
* Example of use:
*
* C++
*
* @dontinclude ExampleXdmfHDF5Writer.cpp
* @skipline newPath
* @until setFileSizeLimit
*
* Python
*
* @dontinclude XdmfExampleHDF5Writer.py