Updates will be applied on October 27th between 12pm - 12:45pm EDT (UTC-0400). Gitlab may be slow during the maintenance window.

Commit 4b1c1d0b authored by Andrew J. Burns (Cont's avatar Andrew J. Burns (Cont
Browse files

changes for hdf5 file splitting and combining xmf files

parent f91048a3
......@@ -38,11 +38,18 @@
* Xdmf supports the following attribute centers:
*
* Example of use:
* //Assuming that exampleAttribute is a shared pointer to an XdmfAttribute with a set center
* if (exampleAttribute->getCenter() == XdmfAttributeCenter::Grid())
* {
* //do whatever is to be done if the center is grid
* }
*
* C++
*
* @dontinclude ExampleXdmfAttribute.cpp
* @skipline exampleCenter
* @until }
*
* Python
*
* @dontinclude XdmfExampleAttribute.py
* @skipline exampleCenter
* @until do
*
* Grid
* Cell
......
......@@ -39,11 +39,17 @@
*
* Example of use:
*
* //Assuming that exampleAttribute is a shared pointer to an XdmfAttribute that has had its type set
* if (exampleAttribute->getType() == XdmfAttributeType:Scalar())
* {
* //do whatever is to be done if the attribute is a scalar
* }
* C++
*
* @dontinclude ExampleXdmfAttribute.cpp
* @skipline exampleType
* @until }
*
* Python
*
* @dontinclude XdmfExampleAttribute.py
* @skipline exampleType
* @until do
*
* Xdmf supports the following attribute types:
* NoAttributeType
......
......@@ -39,11 +39,17 @@
*
* Example of use:
*
* //Assuming that exampleGeometry is a shared pointer to an XdmfGeometry with its type set
* if (exampleGeometry->getType() == XdmfGeometry::XYZ())
* {
* //do whatever is to be done if the geometry is xyz
* }
* C++
*
* @dontinclude ExampleXdmfGeometryType.cpp
* @skipline getType
* @until }
*
* Python
*
* @dontinclude XdmfExampleGeometryType.py
* @skipline getType
* @until do
*
* Xdmf supports the following geometry types:
* NoGeometryType
......
......@@ -39,11 +39,17 @@
*
* Example of use:
*
* //Assuming that exampleCollection is a shared pointer to an XmdfGridCollection with a type that has been set
* if (exampleCollection->getType() == XdmfGridCollectionType::Temporal())
* {
* //do whatever is to be done if the grid collection is temporal
* }
* C++
*
* @dontinclude ExampleXdmfGridCollection.cpp
* @skipline exampleType
* @until }
*
* Python
*
* @dontinclude XdmfExampleGridCollection.py
* @skipline exampleType
* @until do
*
* Xdmf supports the following collection types:
* NoCollectionType
......
......@@ -208,9 +208,23 @@ XdmfMap::populateItem(const std::map<std::string, std::string> & itemProperties,
}
}
else {
mRemoteTaskIdsController = arrayVector[0]->getHeavyDataController();
mLocalNodeIdsController = arrayVector[1]->getHeavyDataController();
mRemoteLocalNodeIdsController = arrayVector[2]->getHeavyDataController();
//this needs to be changed to account for controllers being a vector now
mRemoteTaskIdsControllers.clear();
for (int i = 0; i < arrayVector[0]->getNumberHeavyDataControllers(); i++)
{
mRemoteTaskIdsControllers.push_back(arrayVector[0]->getHeavyDataController(i));
}
mLocalNodeIdsControllers.clear();
for (int i = 0; i < arrayVector[1]->getNumberHeavyDataControllers(); i++)
{
mLocalNodeIdsControllers.push_back(arrayVector[1]->getHeavyDataController(i));
}
mRemoteLocalNodeIdsControllers.clear();
for (int i = 0; i < arrayVector[2]->getNumberHeavyDataControllers(); i++)
{
mRemoteLocalNodeIdsControllers.push_back(arrayVector[2]->getHeavyDataController(i));
}
}
}
}
......@@ -218,14 +232,30 @@ XdmfMap::populateItem(const std::map<std::string, std::string> & itemProperties,
void
XdmfMap::read()
{
if(mLocalNodeIdsController &&
mRemoteTaskIdsController &&
mRemoteLocalNodeIdsController) {
if(!(mLocalNodeIdsController->getSize() ==
mRemoteTaskIdsController->getSize() &&
mLocalNodeIdsController->getSize() ==
mRemoteLocalNodeIdsController->getSize())) {
if(mLocalNodeIdsControllers.size() > 0 &&
mRemoteTaskIdsControllers.size() > 0 &&
mRemoteLocalNodeIdsControllers.size() > 0) {
int localNodeCount = 0;
for (int i = 0; i< mLocalNodeIdsControllers.size(); i++)
{
localNodeCount += mLocalNodeIdsControllers[i]->getSize();
}
int remoteTaskCount = 0;
for (int i = 0; i< mRemoteTaskIdsControllers.size(); i++)
{
remoteTaskCount += mRemoteTaskIdsControllers[i]->getSize();
}
int remoteNodeCount = 0;
for (int i = 0; i< mRemoteLocalNodeIdsControllers.size(); i++)
{
remoteNodeCount += mRemoteLocalNodeIdsControllers[i]->getSize();
}
if(!(localNodeCount ==
remoteTaskCount &&
localNodeCount ==
remoteNodeCount)){
XdmfError::message(XdmfError::FATAL,
"Arrays must be of equal size in XdmfMap::read");
}
......@@ -234,9 +264,27 @@ XdmfMap::read()
shared_ptr<XdmfArray> localNodeIds = XdmfArray::New();
shared_ptr<XdmfArray> remoteLocalNodeIds = XdmfArray::New();
mRemoteTaskIdsController->read(remoteTaskIds.get());
mLocalNodeIdsController->read(localNodeIds.get());
mRemoteLocalNodeIdsController->read(remoteLocalNodeIds.get());
mRemoteTaskIdsControllers[0]->read(remoteTaskIds.get());
for (int i = 1; i < mRemoteTaskIdsControllers.size(); i++)
{
shared_ptr<XdmfArray> tempArray = XdmfArray::New();
mRemoteTaskIdsControllers[i]->read(tempArray.get());
remoteTaskIds->insert(remoteTaskIds->getSize(), tempArray, 0, tempArray->getSize());
}
mLocalNodeIdsControllers[0]->read(localNodeIds.get());
for (int i = 1; i < mLocalNodeIdsControllers.size(); i++)
{
shared_ptr<XdmfArray> tempArray = XdmfArray::New();
mLocalNodeIdsControllers[i]->read(tempArray.get());
localNodeIds->insert(localNodeIds->getSize(), tempArray, 0, tempArray->getSize());
}
mRemoteLocalNodeIdsControllers[0]->read(remoteLocalNodeIds.get());
for (int i = 1; i < mRemoteLocalNodeIdsControllers.size(); i++)
{
shared_ptr<XdmfArray> tempArray = XdmfArray::New();
mRemoteLocalNodeIdsControllers[i]->read(tempArray.get());
remoteLocalNodeIds->insert(remoteLocalNodeIds->getSize(), tempArray, 0, tempArray->getSize());
}
for(unsigned int i=0; i<remoteTaskIds->getSize(); ++i) {
const unsigned int remoteTaskId = remoteTaskIds->getValue<task_id>(i);
......@@ -248,6 +296,7 @@ XdmfMap::read()
}
}
void
XdmfMap::release()
{
......@@ -255,20 +304,35 @@ XdmfMap::release()
}
void
XdmfMap::setHeavyDataControllers(shared_ptr<XdmfHeavyDataController> remoteTaskIdsController,
shared_ptr<XdmfHeavyDataController> localNodeIdsController,
shared_ptr<XdmfHeavyDataController> remoteLocalNodeIdsController)
XdmfMap::setHeavyDataControllers(std::vector<shared_ptr<XdmfHeavyDataController> > remoteTaskIdsControllers,
std::vector<shared_ptr<XdmfHeavyDataController> > localNodeIdsControllers,
std::vector<shared_ptr<XdmfHeavyDataController> > remoteLocalNodeIdsControllers)
{
if(!(localNodeIdsController->getSize() ==
remoteTaskIdsController->getSize() &&
localNodeIdsController->getSize() ==
remoteLocalNodeIdsController->getSize()))
int localNodeCount = 0;
for (int i = 0; i< localNodeIdsControllers.size(); i++)
{
localNodeCount += localNodeIdsControllers[i]->getSize();
}
int remoteTaskCount = 0;
for (int i = 0; i< remoteTaskIdsControllers.size(); i++)
{
remoteTaskCount += remoteTaskIdsControllers[i]->getSize();
}
int remoteNodeCount = 0;
for (int i = 0; i< remoteLocalNodeIdsControllers.size(); i++)
{
remoteNodeCount += remoteLocalNodeIdsControllers[i]->getSize();
}
if(!(localNodeCount ==
remoteTaskCount &&
localNodeCount ==
remoteNodeCount))
XdmfError::message(XdmfError::FATAL,
"Arrays must be of equal size in "
"XdmfMap::setHeavyDataControllers");
mRemoteTaskIdsController = remoteTaskIdsController;
mLocalNodeIdsController = localNodeIdsController;
mRemoteLocalNodeIdsController = remoteLocalNodeIdsController;
mRemoteTaskIdsControllers = remoteTaskIdsControllers;
mLocalNodeIdsControllers = localNodeIdsControllers;
mRemoteLocalNodeIdsControllers = remoteLocalNodeIdsControllers;
}
void
......@@ -311,15 +375,37 @@ XdmfMap::traverse(const shared_ptr<XdmfBaseVisitor> visitor)
}
}
remoteTaskIds->setHeavyDataController(mRemoteTaskIdsController);
localNodeIds->setHeavyDataController(mLocalNodeIdsController);
remoteLocalNodeIds->setHeavyDataController(mRemoteLocalNodeIdsController);
for (int i = 0; i < mRemoteTaskIdsControllers.size(); i++)
{
remoteTaskIds->insert(mRemoteTaskIdsControllers[i]);
}
for (int i = 0; i < mLocalNodeIdsControllers.size(); i++)
{
localNodeIds->insert(mLocalNodeIdsControllers[i]);
}
for (int i = 0; i < mRemoteLocalNodeIdsControllers.size(); i++)
{
remoteLocalNodeIds->insert(mRemoteLocalNodeIdsControllers[i]);
}
remoteTaskIds->accept(visitor);
localNodeIds->accept(visitor);
remoteLocalNodeIds->accept(visitor);
mLocalNodeIdsController = localNodeIds->getHeavyDataController();
mRemoteTaskIdsController = remoteTaskIds->getHeavyDataController();
mRemoteLocalNodeIdsController = remoteLocalNodeIds->getHeavyDataController();
mLocalNodeIdsControllers.clear();
mRemoteTaskIdsControllers.clear();
mRemoteLocalNodeIdsControllers.clear();
for (int i = 0; i < remoteTaskIds->getNumberHeavyDataControllers(); i++)
{
mRemoteTaskIdsControllers.push_back(remoteTaskIds->getHeavyDataController(i));
}
for (int i = 0; i < localNodeIds->getNumberHeavyDataControllers(); i++)
{
mLocalNodeIdsControllers.push_back(localNodeIds->getHeavyDataController(i));
}
for (int i = 0; i < remoteLocalNodeIds->getNumberHeavyDataControllers(); i++)
{
mRemoteLocalNodeIdsControllers.push_back(remoteLocalNodeIds->getHeavyDataController(i));
}
}
......@@ -306,17 +306,17 @@ public:
* @skipline hdf5FilePath
* @until setHeavyDataController
*
* @param remoteTaskIdsController an XdmfHeavyDataController to the remote
* @param remoteTaskIdsControllers a vector of XdmfHeavyDataControllers to the remote
* task ids dataset.
* @param localNodeIdsController an XdmfHeavyDataController to the local
* @param localNodeIdsControllers a vector of XdmfHeavyDataControllers to the local
* node ids dataset.
* @param remoteLocalNodeIdsController an XdmfHeavyDataController to the
* @param remoteLocalNodeIdsControllers a vector of XdmfHeavyDataControllers to the
* remote local node ids dataset.
*/
void
setHeavyDataControllers(shared_ptr<XdmfHeavyDataController> remoteTaskIdsController,
shared_ptr<XdmfHeavyDataController> localNodeIdsController,
shared_ptr<XdmfHeavyDataController> remoteLocalNodeIdsController);
setHeavyDataControllers(std::vector<shared_ptr<XdmfHeavyDataController> > remoteTaskIdsControllers,
std::vector<shared_ptr<XdmfHeavyDataController> > localNodeIdsControllers,
std::vector<shared_ptr<XdmfHeavyDataController> > remoteLocalNodeIdsControllers);
/**
* Set the boundary communicator map.
......@@ -380,12 +380,12 @@ private:
XdmfMap(const XdmfMap & map); // Not implemented.
void operator=(const XdmfMap & map); // Not implemented.
shared_ptr<XdmfHeavyDataController> mLocalNodeIdsController;
std::vector<shared_ptr<XdmfHeavyDataController> > mLocalNodeIdsControllers;
// remoteTaskId | localNodeId | remoteLocalNodeId
std::map<task_id, node_id_map > mMap;
std::string mName;
shared_ptr<XdmfHeavyDataController> mRemoteLocalNodeIdsController;
shared_ptr<XdmfHeavyDataController> mRemoteTaskIdsController;
std::vector<shared_ptr<XdmfHeavyDataController> > mRemoteLocalNodeIdsControllers;
std::vector<shared_ptr<XdmfHeavyDataController> > mRemoteTaskIdsControllers;
};
......
......@@ -14,11 +14,17 @@
*
* Example of use:
*
* //Assuming that exampleSet is a shared pointer to an XdmfSet with its type set
* if (exampleSet->getType() == XdmfSetType::Node())
* {
* //Do whatever is to be done if the set is a node
* }
* C++
*
* @dontinclude ExampleXdmfSet.cpp
* @skipline exampleType
* @until }
*
* Python
*
* @dontinclude XdmfExampleSet.py
* @skipline exampleType
* @until do
*
* Xdmf supports the following set types:
* NoSetType
......
......@@ -39,11 +39,17 @@
*
* Example of use:
*
* //Assuming that exampleTopology is a shared pointer to an XdmfTopology with its type set
* if (exampleTopology->getType() == XdmfTopologyType::Triangle())
* {
* //Do whatever is to be done if the type is Triangle
* }
* C++
*
* @dontinclude ExampleXdmfTopologyType.cpp
* @skipline getType
* @until }
*
* Python
*
* @dontinclude XdmfExampleTopologyType.py
* @skipline getType
* @until do
*
* Xdmf supports the following topology types:
* NoTopologyType
......
......@@ -31,6 +31,11 @@
#include "XdmfVisitor.hpp"
#include "XdmfError.hpp"
XDMF_CHILDREN_IMPLEMENTATION(XdmfArray,
XdmfHeavyDataController,
HeavyDataController,
Name)
class XdmfArray::Clear : public boost::static_visitor<void> {
public:
......@@ -61,8 +66,7 @@ public:
mArray->mArray);
}
private:
private:
XdmfArray * const mArray;
};
......@@ -488,8 +492,8 @@ public:
unsigned int
operator()(const boost::blank & array) const
{
if(mArray->mHeavyDataController) {
return mArray->mHeavyDataController->getSize();
if(mArray->mHeavyDataControllers.size()>0) {
return mArray->mHeavyDataControllers[0]->getSize();//modify this to compile all controllers
}
return 0;
}
......@@ -522,7 +526,6 @@ XdmfArray::New()
XdmfArray::XdmfArray() :
mArrayPointerNumValues(0),
mHeavyDataController(shared_ptr<XdmfHeavyDataController>()),
mName(""),
mTmpReserveSize(0)
{
......@@ -554,8 +557,16 @@ XdmfArray::erase(const unsigned int index)
shared_ptr<const XdmfArrayType>
XdmfArray::getArrayType() const
{
return boost::apply_visitor(GetArrayType(mHeavyDataController),
mArray);
if (mHeavyDataControllers.size()>0)
{
return boost::apply_visitor(GetArrayType(mHeavyDataControllers[0]),
mArray);
}
else
{
return boost::apply_visitor(GetArrayType(shared_ptr<XdmfHDF5Controller>()),
mArray);
}
}
unsigned int
......@@ -569,8 +580,17 @@ std::vector<unsigned int>
XdmfArray::getDimensions() const
{
if(mDimensions.size() == 0) {
if(!this->isInitialized() && mHeavyDataController) {
return mHeavyDataController->getDimensions();
if(!this->isInitialized() && mHeavyDataControllers.size() > 0) {
std::vector<unsigned int> returnDimensions;
std::vector<unsigned int> tempDimensions;
for (int i = 0; i < mHeavyDataControllers.size(); i++) {
tempDimensions = mHeavyDataControllers[i]->getDimensions();
for (int j =0; i < tempDimensions.size(); i++)
{
returnDimensions.push_back(tempDimensions[i]);
}
}
return returnDimensions;
}
const unsigned int size = this->getSize();
return std::vector<unsigned int>(1, size);
......@@ -586,26 +606,13 @@ XdmfArray::getDimensionsString() const
dimensions.size());
}
shared_ptr<XdmfHeavyDataController>
XdmfArray::getHeavyDataController()
{
return boost::const_pointer_cast<XdmfHeavyDataController>
(static_cast<const XdmfArray &>(*this).getHeavyDataController());
}
shared_ptr<const XdmfHeavyDataController>
XdmfArray::getHeavyDataController() const
{
return mHeavyDataController;
}
std::map<std::string, std::string>
XdmfArray::getItemProperties() const
{
std::map<std::string, std::string> arrayProperties;
if(mHeavyDataController) {
if(mHeavyDataControllers.size()>0) {
arrayProperties.insert(std::make_pair("Format",
mHeavyDataController->getName()));
mHeavyDataControllers[0]->getName()));
}
else {
arrayProperties.insert(std::make_pair("Format", "XML"));
......@@ -660,6 +667,33 @@ XdmfArray::getValuesString() const
mArray);
}
shared_ptr<XdmfHeavyDataController>
XdmfArray::getHeavyDataController()
{
return boost::const_pointer_cast<XdmfHeavyDataController>
(static_cast<const XdmfArray &>(*this).getHeavyDataController(0));
}
shared_ptr<const XdmfHeavyDataController>
XdmfArray::getHeavyDataController() const
{
if (mHeavyDataControllers.size()>0) {
return mHeavyDataControllers[0];
}
else {
return shared_ptr<XdmfHeavyDataController>();
}
}
void
XdmfArray::setHeavyDataController(shared_ptr<XdmfHeavyDataController> newController)
{
//since this is replacing the previous version which was designed to completely replace the controller of the array
//it will clear the current controllers before adding the new one in
mHeavyDataControllers.clear();
mHeavyDataControllers.push_back(newController);
}
void
XdmfArray::initialize(const shared_ptr<const XdmfArrayType> arrayType,
const unsigned int size)
......@@ -758,19 +792,41 @@ XdmfArray::populateItem(const std::map<std::string, std::string> & itemPropertie
const std::vector<shared_ptr<XdmfItem> > & childItems,
const XdmfCoreReader * const reader)
{
//This inserts any XdmfInformation in childItems into the object.
XdmfItem::populateItem(itemProperties, childItems, reader);
const shared_ptr<const XdmfArrayType> arrayType =
XdmfArrayType::New(itemProperties);
std::map<std::string, std::string>::const_iterator content =
itemProperties.find("Content");
itemProperties.find("Content0");
if(content == itemProperties.end()) {
XdmfError::message(XdmfError::FATAL,
"'Content' not found in itemProperties in "
"XdmfArray::populateItem");
}
const std::string & contentVal = content->second;
//this was originally a constant, but the compiler doesn't like creating vectors of constant strings.
std::vector<std::string> contentVals;
contentVals.push_back(content->second);
int contentIndex=1;
bool endOfContent = false;
while(!endOfContent)
{
std::stringstream contentSearch;
contentSearch << "Content" << contentIndex;
content = itemProperties.find(contentSearch.str());
if(content != itemProperties.end()) {
contentVals.push_back(content->second);
}
else {
endOfContent = true;
}
contentIndex++;
}
std::map<std::string, std::string>::const_iterator dimensions =
itemProperties.find("Dimensions");
......@@ -796,52 +852,62 @@ XdmfArray::populateItem(const std::map<std::string, std::string> & itemPropertie
}
const std::string & formatVal = format->second;
if(formatVal.compare("HDF") == 0) {
size_t colonLocation = contentVal.find(":");
if(colonLocation == std::string::npos) {
XdmfError::message(XdmfError::FATAL,
"':' not found in content in "
"XdmfArray::populateItem -- double check an HDF5 "
"data set is specified for the file");
}
std::string hdf5Path = contentVal.substr(0, colonLocation);
const std::string dataSetPath =
contentVal.substr(colonLocation + 1,
contentVal.size() - colonLocation - 1);
// FIXME: for other OS (e.g. windows)
if(hdf5Path.size() > 0 && hdf5Path[0] != '/') {
// Dealing with a relative path for hdf5 location
std::map<std::string, std::string>::const_iterator xmlDir =
itemProperties.find("XMLDir");
if(xmlDir == itemProperties.end()) {
contentIndex = 0;
int contentStep = 2;
while (contentIndex < contentVals.size()) {
size_t colonLocation = contentVals[contentIndex].find(":");
if(colonLocation == std::string::npos) {
XdmfError::message(XdmfError::FATAL,
"'XMLDir' not found in itemProperties in "
"XdmfArray::populateItem");
"':' not found in content in "
"XdmfArray::populateItem -- double check an HDF5 "
"data set is specified for the file");
}
std::string hdf5Path = contentVals[contentIndex].substr(0, colonLocation);
std::string dataSetPath = contentVals[contentIndex].substr(colonLocation+1);
//parse dimensions from the content
std::vector<unsigned int> contentDims;
try {
//this is the string that contains the dimensions
boost::tokenizer<> dimtokens(contentVals[contentIndex+1]);
for(boost::tokenizer<>::const_iterator iter = dimtokens.begin();
iter != dimtokens.end();
++iter) {
contentDims.push_back(atoi((*iter).c_str()));
}
contentStep = 2;//if this works then the dimension content should be skipped over
}
catch (...) {//if it fails then it means that the next content is not a dimension string
//in this case it is assumed that the controller will have dimensions equal to the array
for (int j = 0; j < mDimensions.size(); j++) {
contentDims.push_back(mDimensions[j]);
}
contentStep = 1;
}
std::stringstream newHDF5Path;
newHDF5Path << xmlDir->second << hdf5Path;
hdf5Path = newHDF5Path.str();
mHeavyDataControllers.push_back(