Commit 4b1c1d0b authored by Andrew J. Burns (Cont's avatar Andrew J. Burns (Cont

changes for hdf5 file splitting and combining xmf files

parent f91048a3
......@@ -38,11 +38,18 @@
* Xdmf supports the following attribute centers:
*
* Example of use:
* //Assuming that exampleAttribute is a shared pointer to an XdmfAttribute with a set center
* if (exampleAttribute->getCenter() == XdmfAttributeCenter::Grid())
* {
* //do whatever is to be done if the center is grid
* }
*
* C++
*
* @dontinclude ExampleXdmfAttribute.cpp
* @skipline exampleCenter
* @until }
*
* Python
*
* @dontinclude XdmfExampleAttribute.py
* @skipline exampleCenter
* @until do
*
* Grid
* Cell
......
......@@ -39,11 +39,17 @@
*
* Example of use:
*
* //Assuming that exampleAttribute is a shared pointer to an XdmfAttribute that has had its type set
* if (exampleAttribute->getType() == XdmfAttributeType:Scalar())
* {
* //do whatever is to be done if the attribute is a scalar
* }
* C++
*
* @dontinclude ExampleXdmfAttribute.cpp
* @skipline exampleType
* @until }
*
* Python
*
* @dontinclude XdmfExampleAttribute.py
* @skipline exampleType
* @until do
*
* Xdmf supports the following attribute types:
* NoAttributeType
......
......@@ -39,11 +39,17 @@
*
* Example of use:
*
* //Assuming that exampleGeometry is a shared pointer to an XdmfGeometry with its type set
* if (exampleGeometry->getType() == XdmfGeometry::XYZ())
* {
* //do whatever is to be done if the geometry is xyz
* }
* C++
*
* @dontinclude ExampleXdmfGeometryType.cpp
* @skipline getType
* @until }
*
* Python
*
* @dontinclude XdmfExampleGeometryType.py
* @skipline getType
* @until do
*
* Xdmf supports the following geometry types:
* NoGeometryType
......
......@@ -39,11 +39,17 @@
*
* Example of use:
*
* //Assuming that exampleCollection is a shared pointer to an XmdfGridCollection with a type that has been set
* if (exampleCollection->getType() == XdmfGridCollectionType::Temporal())
* {
* //do whatever is to be done if the grid collection is temporal
* }
* C++
*
* @dontinclude ExampleXdmfGridCollection.cpp
* @skipline exampleType
* @until }
*
* Python
*
* @dontinclude XdmfExampleGridCollection.py
* @skipline exampleType
* @until do
*
* Xdmf supports the following collection types:
* NoCollectionType
......
......@@ -208,9 +208,23 @@ XdmfMap::populateItem(const std::map<std::string, std::string> & itemProperties,
}
}
else {
mRemoteTaskIdsController = arrayVector[0]->getHeavyDataController();
mLocalNodeIdsController = arrayVector[1]->getHeavyDataController();
mRemoteLocalNodeIdsController = arrayVector[2]->getHeavyDataController();
//this needs to be changed to account for controllers being a vector now
mRemoteTaskIdsControllers.clear();
for (int i = 0; i < arrayVector[0]->getNumberHeavyDataControllers(); i++)
{
mRemoteTaskIdsControllers.push_back(arrayVector[0]->getHeavyDataController(i));
}
mLocalNodeIdsControllers.clear();
for (int i = 0; i < arrayVector[1]->getNumberHeavyDataControllers(); i++)
{
mLocalNodeIdsControllers.push_back(arrayVector[1]->getHeavyDataController(i));
}
mRemoteLocalNodeIdsControllers.clear();
for (int i = 0; i < arrayVector[2]->getNumberHeavyDataControllers(); i++)
{
mRemoteLocalNodeIdsControllers.push_back(arrayVector[2]->getHeavyDataController(i));
}
}
}
}
......@@ -218,14 +232,30 @@ XdmfMap::populateItem(const std::map<std::string, std::string> & itemProperties,
void
XdmfMap::read()
{
if(mLocalNodeIdsController &&
mRemoteTaskIdsController &&
mRemoteLocalNodeIdsController) {
if(!(mLocalNodeIdsController->getSize() ==
mRemoteTaskIdsController->getSize() &&
mLocalNodeIdsController->getSize() ==
mRemoteLocalNodeIdsController->getSize())) {
if(mLocalNodeIdsControllers.size() > 0 &&
mRemoteTaskIdsControllers.size() > 0 &&
mRemoteLocalNodeIdsControllers.size() > 0) {
int localNodeCount = 0;
for (int i = 0; i< mLocalNodeIdsControllers.size(); i++)
{
localNodeCount += mLocalNodeIdsControllers[i]->getSize();
}
int remoteTaskCount = 0;
for (int i = 0; i< mRemoteTaskIdsControllers.size(); i++)
{
remoteTaskCount += mRemoteTaskIdsControllers[i]->getSize();
}
int remoteNodeCount = 0;
for (int i = 0; i< mRemoteLocalNodeIdsControllers.size(); i++)
{
remoteNodeCount += mRemoteLocalNodeIdsControllers[i]->getSize();
}
if(!(localNodeCount ==
remoteTaskCount &&
localNodeCount ==
remoteNodeCount)){
XdmfError::message(XdmfError::FATAL,
"Arrays must be of equal size in XdmfMap::read");
}
......@@ -234,9 +264,27 @@ XdmfMap::read()
shared_ptr<XdmfArray> localNodeIds = XdmfArray::New();
shared_ptr<XdmfArray> remoteLocalNodeIds = XdmfArray::New();
mRemoteTaskIdsController->read(remoteTaskIds.get());
mLocalNodeIdsController->read(localNodeIds.get());
mRemoteLocalNodeIdsController->read(remoteLocalNodeIds.get());
mRemoteTaskIdsControllers[0]->read(remoteTaskIds.get());
for (int i = 1; i < mRemoteTaskIdsControllers.size(); i++)
{
shared_ptr<XdmfArray> tempArray = XdmfArray::New();
mRemoteTaskIdsControllers[i]->read(tempArray.get());
remoteTaskIds->insert(remoteTaskIds->getSize(), tempArray, 0, tempArray->getSize());
}
mLocalNodeIdsControllers[0]->read(localNodeIds.get());
for (int i = 1; i < mLocalNodeIdsControllers.size(); i++)
{
shared_ptr<XdmfArray> tempArray = XdmfArray::New();
mLocalNodeIdsControllers[i]->read(tempArray.get());
localNodeIds->insert(localNodeIds->getSize(), tempArray, 0, tempArray->getSize());
}
mRemoteLocalNodeIdsControllers[0]->read(remoteLocalNodeIds.get());
for (int i = 1; i < mRemoteLocalNodeIdsControllers.size(); i++)
{
shared_ptr<XdmfArray> tempArray = XdmfArray::New();
mRemoteLocalNodeIdsControllers[i]->read(tempArray.get());
remoteLocalNodeIds->insert(remoteLocalNodeIds->getSize(), tempArray, 0, tempArray->getSize());
}
for(unsigned int i=0; i<remoteTaskIds->getSize(); ++i) {
const unsigned int remoteTaskId = remoteTaskIds->getValue<task_id>(i);
......@@ -248,6 +296,7 @@ XdmfMap::read()
}
}
void
XdmfMap::release()
{
......@@ -255,20 +304,35 @@ XdmfMap::release()
}
void
XdmfMap::setHeavyDataControllers(shared_ptr<XdmfHeavyDataController> remoteTaskIdsController,
shared_ptr<XdmfHeavyDataController> localNodeIdsController,
shared_ptr<XdmfHeavyDataController> remoteLocalNodeIdsController)
XdmfMap::setHeavyDataControllers(std::vector<shared_ptr<XdmfHeavyDataController> > remoteTaskIdsControllers,
std::vector<shared_ptr<XdmfHeavyDataController> > localNodeIdsControllers,
std::vector<shared_ptr<XdmfHeavyDataController> > remoteLocalNodeIdsControllers)
{
if(!(localNodeIdsController->getSize() ==
remoteTaskIdsController->getSize() &&
localNodeIdsController->getSize() ==
remoteLocalNodeIdsController->getSize()))
int localNodeCount = 0;
for (int i = 0; i< localNodeIdsControllers.size(); i++)
{
localNodeCount += localNodeIdsControllers[i]->getSize();
}
int remoteTaskCount = 0;
for (int i = 0; i< remoteTaskIdsControllers.size(); i++)
{
remoteTaskCount += remoteTaskIdsControllers[i]->getSize();
}
int remoteNodeCount = 0;
for (int i = 0; i< remoteLocalNodeIdsControllers.size(); i++)
{
remoteNodeCount += remoteLocalNodeIdsControllers[i]->getSize();
}
if(!(localNodeCount ==
remoteTaskCount &&
localNodeCount ==
remoteNodeCount))
XdmfError::message(XdmfError::FATAL,
"Arrays must be of equal size in "
"XdmfMap::setHeavyDataControllers");
mRemoteTaskIdsController = remoteTaskIdsController;
mLocalNodeIdsController = localNodeIdsController;
mRemoteLocalNodeIdsController = remoteLocalNodeIdsController;
mRemoteTaskIdsControllers = remoteTaskIdsControllers;
mLocalNodeIdsControllers = localNodeIdsControllers;
mRemoteLocalNodeIdsControllers = remoteLocalNodeIdsControllers;
}
void
......@@ -311,15 +375,37 @@ XdmfMap::traverse(const shared_ptr<XdmfBaseVisitor> visitor)
}
}
remoteTaskIds->setHeavyDataController(mRemoteTaskIdsController);
localNodeIds->setHeavyDataController(mLocalNodeIdsController);
remoteLocalNodeIds->setHeavyDataController(mRemoteLocalNodeIdsController);
for (int i = 0; i < mRemoteTaskIdsControllers.size(); i++)
{
remoteTaskIds->insert(mRemoteTaskIdsControllers[i]);
}
for (int i = 0; i < mLocalNodeIdsControllers.size(); i++)
{
localNodeIds->insert(mLocalNodeIdsControllers[i]);
}
for (int i = 0; i < mRemoteLocalNodeIdsControllers.size(); i++)
{
remoteLocalNodeIds->insert(mRemoteLocalNodeIdsControllers[i]);
}
remoteTaskIds->accept(visitor);
localNodeIds->accept(visitor);
remoteLocalNodeIds->accept(visitor);
mLocalNodeIdsController = localNodeIds->getHeavyDataController();
mRemoteTaskIdsController = remoteTaskIds->getHeavyDataController();
mRemoteLocalNodeIdsController = remoteLocalNodeIds->getHeavyDataController();
mLocalNodeIdsControllers.clear();
mRemoteTaskIdsControllers.clear();
mRemoteLocalNodeIdsControllers.clear();
for (int i = 0; i < remoteTaskIds->getNumberHeavyDataControllers(); i++)
{
mRemoteTaskIdsControllers.push_back(remoteTaskIds->getHeavyDataController(i));
}
for (int i = 0; i < localNodeIds->getNumberHeavyDataControllers(); i++)
{
mLocalNodeIdsControllers.push_back(localNodeIds->getHeavyDataController(i));
}
for (int i = 0; i < remoteLocalNodeIds->getNumberHeavyDataControllers(); i++)
{
mRemoteLocalNodeIdsControllers.push_back(remoteLocalNodeIds->getHeavyDataController(i));
}
}
......@@ -306,17 +306,17 @@ public:
* @skipline hdf5FilePath
* @until setHeavyDataController
*
* @param remoteTaskIdsController an XdmfHeavyDataController to the remote
* @param remoteTaskIdsControllers a vector of XdmfHeavyDataControllers to the remote
* task ids dataset.
* @param localNodeIdsController an XdmfHeavyDataController to the local
* @param localNodeIdsControllers a vector of XdmfHeavyDataControllers to the local
* node ids dataset.
* @param remoteLocalNodeIdsController an XdmfHeavyDataController to the
* @param remoteLocalNodeIdsControllers a vector of XdmfHeavyDataControllers to the
* remote local node ids dataset.
*/
void
setHeavyDataControllers(shared_ptr<XdmfHeavyDataController> remoteTaskIdsController,
shared_ptr<XdmfHeavyDataController> localNodeIdsController,
shared_ptr<XdmfHeavyDataController> remoteLocalNodeIdsController);
setHeavyDataControllers(std::vector<shared_ptr<XdmfHeavyDataController> > remoteTaskIdsControllers,
std::vector<shared_ptr<XdmfHeavyDataController> > localNodeIdsControllers,
std::vector<shared_ptr<XdmfHeavyDataController> > remoteLocalNodeIdsControllers);
/**
* Set the boundary communicator map.
......@@ -380,12 +380,12 @@ private:
XdmfMap(const XdmfMap & map); // Not implemented.
void operator=(const XdmfMap & map); // Not implemented.
shared_ptr<XdmfHeavyDataController> mLocalNodeIdsController;
std::vector<shared_ptr<XdmfHeavyDataController> > mLocalNodeIdsControllers;
// remoteTaskId | localNodeId | remoteLocalNodeId
std::map<task_id, node_id_map > mMap;
std::string mName;
shared_ptr<XdmfHeavyDataController> mRemoteLocalNodeIdsController;
shared_ptr<XdmfHeavyDataController> mRemoteTaskIdsController;
std::vector<shared_ptr<XdmfHeavyDataController> > mRemoteLocalNodeIdsControllers;
std::vector<shared_ptr<XdmfHeavyDataController> > mRemoteTaskIdsControllers;
};
......
......@@ -14,11 +14,17 @@
*
* Example of use:
*
* //Assuming that exampleSet is a shared pointer to an XdmfSet with its type set
* if (exampleSet->getType() == XdmfSetType::Node())
* {
* //Do whatever is to be done if the set is a node
* }
* C++
*
* @dontinclude ExampleXdmfSet.cpp
* @skipline exampleType
* @until }
*
* Python
*
* @dontinclude XdmfExampleSet.py
* @skipline exampleType
* @until do
*
* Xdmf supports the following set types:
* NoSetType
......
......@@ -39,11 +39,17 @@
*
* Example of use:
*
* //Assuming that exampleTopology is a shared pointer to an XdmfTopology with its type set
* if (exampleTopology->getType() == XdmfTopologyType::Triangle())
* {
* //Do whatever is to be done if the type is Triangle
* }
* C++
*
* @dontinclude ExampleXdmfTopologyType.cpp
* @skipline getType
* @until }
*
* Python
*
* @dontinclude XdmfExampleTopologyType.py
* @skipline getType
* @until do
*
* Xdmf supports the following topology types:
* NoTopologyType
......
......@@ -31,6 +31,11 @@
#include "XdmfVisitor.hpp"
#include "XdmfError.hpp"
XDMF_CHILDREN_IMPLEMENTATION(XdmfArray,
XdmfHeavyDataController,
HeavyDataController,
Name)
class XdmfArray::Clear : public boost::static_visitor<void> {
public:
......@@ -61,8 +66,7 @@ public:
mArray->mArray);
}
private:
private:
XdmfArray * const mArray;
};
......@@ -488,8 +492,8 @@ public:
unsigned int
operator()(const boost::blank & array) const
{
if(mArray->mHeavyDataController) {
return mArray->mHeavyDataController->getSize();
if(mArray->mHeavyDataControllers.size()>0) {
return mArray->mHeavyDataControllers[0]->getSize();//modify this to compile all controllers
}
return 0;
}
......@@ -522,7 +526,6 @@ XdmfArray::New()
XdmfArray::XdmfArray() :
mArrayPointerNumValues(0),
mHeavyDataController(shared_ptr<XdmfHeavyDataController>()),
mName(""),
mTmpReserveSize(0)
{
......@@ -554,8 +557,16 @@ XdmfArray::erase(const unsigned int index)
shared_ptr<const XdmfArrayType>
XdmfArray::getArrayType() const
{
return boost::apply_visitor(GetArrayType(mHeavyDataController),
mArray);
if (mHeavyDataControllers.size()>0)
{
return boost::apply_visitor(GetArrayType(mHeavyDataControllers[0]),
mArray);
}
else
{
return boost::apply_visitor(GetArrayType(shared_ptr<XdmfHDF5Controller>()),
mArray);
}
}
unsigned int
......@@ -569,8 +580,17 @@ std::vector<unsigned int>
XdmfArray::getDimensions() const
{
if(mDimensions.size() == 0) {
if(!this->isInitialized() && mHeavyDataController) {
return mHeavyDataController->getDimensions();
if(!this->isInitialized() && mHeavyDataControllers.size() > 0) {
std::vector<unsigned int> returnDimensions;
std::vector<unsigned int> tempDimensions;
for (int i = 0; i < mHeavyDataControllers.size(); i++) {
tempDimensions = mHeavyDataControllers[i]->getDimensions();
for (int j =0; i < tempDimensions.size(); i++)
{
returnDimensions.push_back(tempDimensions[i]);
}
}
return returnDimensions;
}
const unsigned int size = this->getSize();
return std::vector<unsigned int>(1, size);
......@@ -586,26 +606,13 @@ XdmfArray::getDimensionsString() const
dimensions.size());
}
shared_ptr<XdmfHeavyDataController>
XdmfArray::getHeavyDataController()
{
return boost::const_pointer_cast<XdmfHeavyDataController>
(static_cast<const XdmfArray &>(*this).getHeavyDataController());
}
shared_ptr<const XdmfHeavyDataController>
XdmfArray::getHeavyDataController() const
{
return mHeavyDataController;
}
std::map<std::string, std::string>
XdmfArray::getItemProperties() const
{
std::map<std::string, std::string> arrayProperties;
if(mHeavyDataController) {
if(mHeavyDataControllers.size()>0) {
arrayProperties.insert(std::make_pair("Format",
mHeavyDataController->getName()));
mHeavyDataControllers[0]->getName()));
}
else {
arrayProperties.insert(std::make_pair("Format", "XML"));
......@@ -660,6 +667,33 @@ XdmfArray::getValuesString() const
mArray);
}
shared_ptr<XdmfHeavyDataController>
XdmfArray::getHeavyDataController()
{
return boost::const_pointer_cast<XdmfHeavyDataController>
(static_cast<const XdmfArray &>(*this).getHeavyDataController(0));
}
shared_ptr<const XdmfHeavyDataController>
XdmfArray::getHeavyDataController() const
{
if (mHeavyDataControllers.size()>0) {
return mHeavyDataControllers[0];
}
else {
return shared_ptr<XdmfHeavyDataController>();
}
}
void
XdmfArray::setHeavyDataController(shared_ptr<XdmfHeavyDataController> newController)
{
//since this is replacing the previous version which was designed to completely replace the controller of the array
//it will clear the current controllers before adding the new one in
mHeavyDataControllers.clear();
mHeavyDataControllers.push_back(newController);
}
void
XdmfArray::initialize(const shared_ptr<const XdmfArrayType> arrayType,
const unsigned int size)
......@@ -758,19 +792,41 @@ XdmfArray::populateItem(const std::map<std::string, std::string> & itemPropertie
const std::vector<shared_ptr<XdmfItem> > & childItems,
const XdmfCoreReader * const reader)
{
//This inserts any XdmfInformation in childItems into the object.
XdmfItem::populateItem(itemProperties, childItems, reader);
const shared_ptr<const XdmfArrayType> arrayType =
XdmfArrayType::New(itemProperties);
std::map<std::string, std::string>::const_iterator content =
itemProperties.find("Content");
itemProperties.find("Content0");
if(content == itemProperties.end()) {
XdmfError::message(XdmfError::FATAL,
"'Content' not found in itemProperties in "
"XdmfArray::populateItem");
}
const std::string & contentVal = content->second;
//this was originally a constant, but the compiler doesn't like creating vectors of constant strings.
std::vector<std::string> contentVals;
contentVals.push_back(content->second);
int contentIndex=1;
bool endOfContent = false;
while(!endOfContent)
{
std::stringstream contentSearch;
contentSearch << "Content" << contentIndex;
content = itemProperties.find(contentSearch.str());
if(content != itemProperties.end()) {
contentVals.push_back(content->second);
}
else {
endOfContent = true;
}
contentIndex++;
}
std::map<std::string, std::string>::const_iterator dimensions =
itemProperties.find("Dimensions");
......@@ -796,52 +852,62 @@ XdmfArray::populateItem(const std::map<std::string, std::string> & itemPropertie
}
const std::string & formatVal = format->second;
if(formatVal.compare("HDF") == 0) {
size_t colonLocation = contentVal.find(":");
if(colonLocation == std::string::npos) {
XdmfError::message(XdmfError::FATAL,
"':' not found in content in "
"XdmfArray::populateItem -- double check an HDF5 "
"data set is specified for the file");
}
std::string hdf5Path = contentVal.substr(0, colonLocation);
const std::string dataSetPath =
contentVal.substr(colonLocation + 1,
contentVal.size() - colonLocation - 1);
// FIXME: for other OS (e.g. windows)
if(hdf5Path.size() > 0 && hdf5Path[0] != '/') {
// Dealing with a relative path for hdf5 location
std::map<std::string, std::string>::const_iterator xmlDir =
itemProperties.find("XMLDir");
if(xmlDir == itemProperties.end()) {
contentIndex = 0;
int contentStep = 2;
while (contentIndex < contentVals.size()) {
size_t colonLocation = contentVals[contentIndex].find(":");
if(colonLocation == std::string::npos) {
XdmfError::message(XdmfError::FATAL,
"'XMLDir' not found in itemProperties in "
"XdmfArray::populateItem");
"':' not found in content in "
"XdmfArray::populateItem -- double check an HDF5 "
"data set is specified for the file");
}
std::string hdf5Path = contentVals[contentIndex].substr(0, colonLocation);
std::string dataSetPath = contentVals[contentIndex].substr(colonLocation+1);
//parse dimensions from the content
std::vector<unsigned int> contentDims;
try {
//this is the string that contains the dimensions
boost::tokenizer<> dimtokens(contentVals[contentIndex+1]);
for(boost::tokenizer<>::const_iterator iter = dimtokens.begin();
iter != dimtokens.end();
++iter) {
contentDims.push_back(atoi((*iter).c_str()));
}
contentStep = 2;//if this works then the dimension content should be skipped over
}
catch (...) {//if it fails then it means that the next content is not a dimension string
//in this case it is assumed that the controller will have dimensions equal to the array
for (int j = 0; j < mDimensions.size(); j++) {
contentDims.push_back(mDimensions[j]);
}
contentStep = 1;
}
std::stringstream newHDF5Path;
newHDF5Path << xmlDir->second << hdf5Path;
hdf5Path = newHDF5Path.str();
mHeavyDataControllers.push_back(
XdmfHDF5Controller::New(hdf5Path,
dataSetPath,
arrayType,
std::vector<unsigned int>(contentDims.size(),
0),
std::vector<unsigned int>(contentDims.size(),
1),
contentDims,
contentDims)
);