Commit dda62b58 authored by Kenneth Leiter's avatar Kenneth Leiter
Browse files

Update XdmfPartitioner to more efficiently handle element sets.

Modify XdmfExodusReader to add element blocks as element sets in xdmf file.
Merge with master.
parents 8e14630a 261e07a6
......@@ -43,8 +43,13 @@ XdmfMap::New(const std::vector<shared_ptr<XdmfAttribute> > & globalNodeIds)
std::map<node_id, std::map<task_id, node_id> > globalNodeIdMap;
// fill globalNodeIdMap using globalNodeIds
std::vector<bool> releaseGlobalNodeIds(globalNodeIds.size(), false);
for(unsigned int i=0; i<globalNodeIds.size(); ++i) {
const shared_ptr<XdmfAttribute> currGlobalNodeIds = globalNodeIds[i];
if(!currGlobalNodeIds->isInitialized()) {
currGlobalNodeIds->read();
releaseGlobalNodeIds[i] = true;
}
for(unsigned int j=0; j<currGlobalNodeIds->getSize(); ++j) {
const node_id currGlobalNodeId = currGlobalNodeIds->getValue<node_id>(j);
globalNodeIdMap[currGlobalNodeId][i] = j;
......@@ -74,6 +79,9 @@ XdmfMap::New(const std::vector<shared_ptr<XdmfAttribute> > & globalNodeIds)
}
}
}
if(releaseGlobalNodeIds[i]) {
currGlobalNodeIds->release();
}
}
return returnValue;
......
......@@ -74,6 +74,7 @@ XdmfDSMCommMPI::XdmfDSMCommMPI()
XdmfDSMCommMPI::~XdmfDSMCommMPI()
{
#ifndef OPEN_MPI
if (InterComm != MPI_COMM_NULL) {
int status = MPI_Comm_free(&InterComm);
if (status != MPI_SUCCESS) {
......@@ -96,6 +97,7 @@ XdmfDSMCommMPI::~XdmfDSMCommMPI()
}
}
}
#endif
}
void
......@@ -170,7 +172,7 @@ XdmfDSMCommMPI::ClosePort()
{
if (Id == 0) {
int status;
status = MPI_Open_port(MPI_INFO_NULL, DsmPortName);
status = MPI_Close_port(DsmPortName);
if (status != MPI_SUCCESS) {
try {
std::string message = "Failed to close port ";
......@@ -254,6 +256,7 @@ XdmfDSMCommMPI::Connect()
void
XdmfDSMCommMPI::Disconnect()
{
#ifndef OPEN_MPI
if (InterComm != MPI_COMM_NULL) {
int status = MPI_Comm_free(&InterComm);
if (status != MPI_SUCCESS) {
......@@ -265,6 +268,7 @@ XdmfDSMCommMPI::Disconnect()
}
}
}
#endif
InterComm = MPI_COMM_NULL;
}
......@@ -273,6 +277,7 @@ XdmfDSMCommMPI::DupComm(MPI_Comm comm)
{
if (IntraComm != comm) {
int status;
#ifndef OPEN_MPI
if (IntraComm != MPI_COMM_NULL) {
status = MPI_Comm_free(&IntraComm);
if (status != MPI_SUCCESS) {
......@@ -284,6 +289,7 @@ XdmfDSMCommMPI::DupComm(MPI_Comm comm)
}
}
}
#endif
if (comm != MPI_COMM_NULL) {
status = MPI_Comm_dup(comm, &IntraComm);
if (status != MPI_SUCCESS) {
......@@ -307,6 +313,7 @@ XdmfDSMCommMPI::DupInterComm(MPI_Comm comm)
{
if (InterComm != comm) {
int status;
#ifndef OPEN_MPI
if (InterComm != MPI_COMM_NULL) {
status = MPI_Comm_free(&InterComm);
if (status != MPI_SUCCESS) {
......@@ -318,6 +325,7 @@ XdmfDSMCommMPI::DupInterComm(MPI_Comm comm)
}
}
}
#endif
if (comm != MPI_COMM_NULL) {
status = MPI_Comm_dup(comm, &InterComm);
if (status != MPI_SUCCESS) {
......
......@@ -457,6 +457,7 @@ void XdmfHDF5ControllerDSM::setServerMode(bool newMode)
void XdmfHDF5ControllerDSM::setWorkerComm(MPI_Comm comm)
{
int status;
#ifndef OPEN_MPI
if (mWorkerComm != MPI_COMM_NULL) {
status = MPI_Comm_free(&mWorkerComm);
if (status != MPI_SUCCESS) {
......@@ -468,6 +469,7 @@ void XdmfHDF5ControllerDSM::setWorkerComm(MPI_Comm comm)
}
}
}
#endif
if (comm != MPI_COMM_NULL) {
status = MPI_Comm_dup(comm, &mWorkerComm);
if (status != MPI_SUCCESS) {
......
......@@ -406,6 +406,7 @@ void XdmfHDF5WriterDSM::setServerMode(bool newMode)
void XdmfHDF5WriterDSM::setWorkerComm(MPI_Comm comm)
{
int status;
#ifndef OPEN_MPI
if (mWorkerComm != MPI_COMM_NULL) {
status = MPI_Comm_free(&mWorkerComm);
if (status != MPI_SUCCESS) {
......@@ -417,6 +418,7 @@ void XdmfHDF5WriterDSM::setWorkerComm(MPI_Comm comm)
}
}
}
#endif
if (comm != MPI_COMM_NULL) {
status = MPI_Comm_dup(comm, &mWorkerComm);
if (status != MPI_SUCCESS) {
......
......@@ -90,7 +90,10 @@ int main(int argc, char *argv[])
//#ClosePort begin
if (id == 0)
{
exampleWriter->getServerBuffer()->GetComm()->ClosePort();
}
//#ClosePort end
......
......@@ -40,7 +40,7 @@ int main(int, char **)
std::vector<shared_ptr<XdmfItem> > readItems2 =
reader->read("TestXdmfReader1.xmf", "//Attribute");
assert(readItems2.size() == 6);
assert(readItems2.size() == 8);
shared_ptr<XdmfAttribute> readAttribute =
shared_dynamic_cast<XdmfAttribute>(readItems2[0]);
assert(readAttribute->getName().compare("Nodal Attribute") == 0);
......
......@@ -71,7 +71,6 @@ public:
gridAttribute->setCenter(XdmfAttributeCenter::Grid());
gridAttribute->insert(0, &gridValues[0], 2);
// Add Node Set
shared_ptr<XdmfSet> nodeSet = XdmfSet::New();
int nodeIds[] = {0, 1, 2};
......@@ -79,6 +78,15 @@ public:
nodeSet->setType(XdmfSetType::Node());
nodeSet->insert(0, &nodeIds[0], 3);
// Add Node Set Attribute
double nodeSetAttributeValues[] = {10, 11, 12};
shared_ptr<XdmfAttribute> nodeSetAttribute = XdmfAttribute::New();
nodeSetAttribute->setName("Node Set Attribute");
nodeSetAttribute->setType(XdmfAttributeType::Scalar());
nodeSetAttribute->setCenter(XdmfAttributeCenter::Node());
nodeSetAttribute->insert(0, &nodeSetAttributeValues[0], 3);
nodeSet->insert(nodeSetAttribute);
// Add Time
shared_ptr<XdmfTime> time = XdmfTime::New(100);
grid->setTime(time);
......
......@@ -644,11 +644,32 @@ XdmfExodusReader::read(const std::string & fileName,
ex_close(exodusHandle);
// add block information as sets
unsigned int elementId = 0;
for(int i=0; i<num_elem_blk; ++i) {
const int numberElementsInBlock = numElemsInBlock[i];
shared_ptr<XdmfSet> set = XdmfSet::New();
std::stringstream setName;
setName << "Block " << i;
set->setName(setName.str());
set->setType(XdmfSetType::Cell());
set->initialize(XdmfArrayType::Int32(), numberElementsInBlock);
for(int j=0; j<numberElementsInBlock; ++j) {
set->insert(j, elementId++);
}
toReturn->insert(set);
if(heavyDataWriter) {
set->accept(heavyDataWriter);
set->release();
}
}
delete [] blockIds;
delete [] numElemsInBlock;
delete [] numNodesPerElemInBlock;
delete [] numElemAttrInBlock;
if(heavyDataWriter) {
heavyDataWriter->closeFile();
}
......
......@@ -299,6 +299,7 @@ XdmfPartitioner::partition(const shared_ptr<XdmfUnstructuredGrid> gridToPartitio
gridToPartition->getGeometry();
const shared_ptr<const XdmfGeometryType> geometryType =
geometry->getType();
const unsigned int geometryDimensions = geometryType->getDimensions();
const shared_ptr<XdmfTopology> topology =
gridToPartition->getTopology();
const shared_ptr<const XdmfTopologyType> topologyType =
......@@ -389,341 +390,447 @@ XdmfPartitioner::partition(const shared_ptr<XdmfUnstructuredGrid> gridToPartitio
delete [] metisConnectivityEind;
delete [] nodesPartition;
// For each partition, map global to local node id
std::vector<std::map<unsigned int, unsigned int> > globalToLocalNodeIdMap;
// For each partition, list global element id.
std::vector<std::vector<unsigned int> > globalElementIds;
for(unsigned int i=0; i<numberOfPartitions; ++i) {
std::map<unsigned int, unsigned int> nodeMap;
globalToLocalNodeIdMap.push_back(nodeMap);
std::vector<unsigned int> elementIds;
globalElementIds.push_back(elementIds);
}
// map of global node ids to pair of partition id and local node id
std::vector<std::map<unsigned int, unsigned int> >
nodeIdMap(numNodes);
// map of global element ids to pair of partition id and local element id
std::vector<std::pair<unsigned int, unsigned int> >
elementIdMap(numElements);
// keep count of number of nodes and element per partition, needed to
// generate local node and element ids
std::vector<unsigned int> localNodeCounts(numberOfPartitions, 0);
std::vector<unsigned int> localElementCounts(numberOfPartitions, 0);
// Fill in globalNodeId for each partition
// fill maps
unsigned int totalIndex = 0;
for (int i=0; i<numElements; ++i) {
unsigned int partitionId = elementsPartition[i];
for (unsigned int j=0; j<topologyType->getNodesPerElement(); ++j) {
unsigned int globalNodeId = topology->getValue<unsigned int>(totalIndex);
if (globalToLocalNodeIdMap[partitionId].count(globalNodeId) == 0) {
// Have not seen this node, need to add to map
unsigned int size = globalToLocalNodeIdMap[partitionId].size();
globalToLocalNodeIdMap[partitionId][globalNodeId] = size;
for(int i=0; i<numElements; ++i) {
const unsigned int partitionId = elementsPartition[i];
for(unsigned int j=0; j<nodesPerElement; ++j) {
const unsigned int globalNodeId =
topology->getValue<unsigned int>(totalIndex++);
std::map<unsigned int, unsigned int> & localNodeIds =
nodeIdMap[globalNodeId];
std::map<unsigned int, unsigned int>::const_iterator iter =
localNodeIds.find(partitionId);
if(iter == localNodeIds.end()) {
localNodeIds[partitionId] = localNodeCounts[partitionId];
localNodeCounts[partitionId]++;
}
else {
localNodeIds[partitionId] = iter->second;
}
totalIndex++;
}
globalElementIds[partitionId].push_back(i);
elementIdMap[i] = std::make_pair(partitionId,
localElementCounts[partitionId]);
localElementCounts[partitionId]++;
}
delete [] elementsPartition;
bool generateGlobalNodeIds = !gridToPartition->getAttribute("GlobalNodeId");
shared_ptr<XdmfGridCollection> partitionedGrids =
// create returned partitioned grid
shared_ptr<XdmfGridCollection> partitionedGrid =
XdmfGridCollection::New();
partitionedGrids->setType(XdmfGridCollectionType::Spatial());
partitionedGrid->setType(XdmfGridCollectionType::Spatial());
bool releaseGeometry = false;
if(!geometry->isInitialized()) {
geometry->read();
releaseGeometry = true;
}
// Split geometry and topology into proper partitions
// add unstructured grids to partitionedGrid and initialize topology
// and geometry in each
for(unsigned int i=0; i<numberOfPartitions; ++i) {
std::map<unsigned int, unsigned int> & currNodeMap =
globalToLocalNodeIdMap[i];
std::vector<unsigned int> & currElemIds = globalElementIds[i];
if(currElemIds.size() > 0) {
const unsigned int localElementCount = localElementCounts[i];
const unsigned int localNodeCount = localNodeCounts[i];
std::stringstream name;
name << gridToPartition->getName() << "_" << i;
const shared_ptr<XdmfUnstructuredGrid> partitioned =
const shared_ptr<XdmfUnstructuredGrid> grid =
XdmfUnstructuredGrid::New();
partitioned->setName(name.str());
partitionedGrids->insert(partitioned);
grid->setName(name.str());
partitionedGrid->insert(grid);
shared_ptr<XdmfGeometry> localGeometry = grid->getGeometry();
localGeometry->setType(geometryType);
localGeometry->initialize(geometry->getArrayType(),
localNodeCount * geometryDimensions);
shared_ptr<XdmfTopology> localTopology = grid->getTopology();
localTopology->setType(topologyType);
localTopology->initialize(topology->getArrayType(),
localElementCount * nodesPerElement);
}
// Fill in geometry for this partition
partitioned->getGeometry()->setType(geometryType);
unsigned int numDimensions = geometryType->getDimensions();
partitioned->getGeometry()->initialize(geometry->getArrayType(),
currNodeMap.size() * numDimensions);
bool releaseGeometry = false;
if(!geometry->isInitialized()) {
geometry->read();
releaseGeometry = true;
}
// fill geometry for each partition
for(int i=0; i<numNodes; ++i) {
const std::map<unsigned int, unsigned int> & localNodeIds = nodeIdMap[i];
for(std::map<unsigned int, unsigned int>::const_iterator iter =
currNodeMap.begin();
iter != currNodeMap.end();
++iter) {
partitioned->getGeometry()->insert(iter->second * numDimensions,
localNodeIds.begin(); iter != localNodeIds.end(); ++iter) {
const unsigned int partitionId = iter->first;
const unsigned int localNodeId = iter->second;
const shared_ptr<XdmfUnstructuredGrid> grid =
partitionedGrid->getUnstructuredGrid(partitionId);
const shared_ptr<XdmfGeometry> localGeometry = grid->getGeometry();
localGeometry->insert(localNodeId * geometryDimensions,
geometry,
iter->first * numDimensions,
numDimensions);
i * geometryDimensions,
geometryDimensions);
}
if(heavyDataWriter) {
partitioned->getGeometry()->accept(heavyDataWriter);
partitioned->getGeometry()->release();
}
// Fill in topology for this partition
partitioned->getTopology()->setType(topologyType);
partitioned->getTopology()->initialize(topology->getArrayType(),
currElemIds.size() * topologyType->getNodesPerElement());
unsigned int index = 0;
for(std::vector<unsigned int>::const_iterator iter = currElemIds.begin();
iter != currElemIds.end();
++iter) {
// Translate these global node ids to local node ids
for(unsigned int j=0; j<topologyType->getNodesPerElement(); ++j) {
unsigned int globalNodeId =
currNodeMap[topology->getValue<unsigned int>(*iter * topologyType->getNodesPerElement() + j)];
partitioned->getTopology()->insert(index, &globalNodeId, 1);
index++;
}
if(releaseGeometry) {
geometry->release();
}
// write geometries to disk if possible
if(heavyDataWriter) {
partitioned->getTopology()->accept(heavyDataWriter);
partitioned->getTopology()->release();
}
if (generateGlobalNodeIds) {
shared_ptr<XdmfAttribute> globalNodeIds = XdmfAttribute::New();
globalNodeIds->setName("GlobalNodeId");
globalNodeIds->setType(XdmfAttributeType::GlobalId());
globalNodeIds->setCenter(XdmfAttributeCenter::Node());
globalNodeIds->initialize<unsigned int>(currNodeMap.size());
for(std::map<unsigned int, unsigned int>::const_iterator iter =
currNodeMap.begin();
iter != currNodeMap.end();
++iter) {
globalNodeIds->insert(iter->second, &iter->first, 1);
for(unsigned int i=0; i<numberOfPartitions; ++i) {
const shared_ptr<XdmfUnstructuredGrid> grid =
partitionedGrid->getUnstructuredGrid(i);
const shared_ptr<XdmfGeometry> localGeometry = grid->getGeometry();
if(localGeometry->getSize() > 0) {
localGeometry->accept(heavyDataWriter);
localGeometry->release();
}
partitioned->insert(globalNodeIds);
if(heavyDataWriter) {
globalNodeIds->accept(heavyDataWriter);
globalNodeIds->release();
}
}
// fill topology for each partition
for(int i=0; i<numElements; ++i) {
const std::pair<unsigned int, unsigned int> & partitionElementPair =
elementIdMap[i];
const unsigned int partitionId = partitionElementPair.first;
const unsigned int localElementId = partitionElementPair.second;
const shared_ptr<XdmfUnstructuredGrid> grid =
partitionedGrid->getUnstructuredGrid(partitionId);
const shared_ptr<XdmfTopology> localTopology = grid->getTopology();
for(unsigned int j=0; j<nodesPerElement; ++j) {
const unsigned int globalNodeId =
topology->getValue<unsigned int>(i*nodesPerElement + j);
const unsigned int localNodeId = nodeIdMap[globalNodeId][partitionId];
localTopology->insert(localElementId*nodesPerElement + j, localNodeId);
}
}
if(releaseGeometry) {
gridToPartition->getGeometry()->release();
}
if(releaseTopology) {
gridToPartition->getTopology()->release();
topology->release();
}
// Split attributes into proper partitions
for(unsigned int i=0; i<gridToPartition->getNumberAttributes(); ++i) {
// write topology to disk if possible
if(heavyDataWriter) {
for(unsigned int i=0; i<numberOfPartitions; ++i) {
const shared_ptr<XdmfUnstructuredGrid> grid =
partitionedGrid->getUnstructuredGrid(i);
const shared_ptr<XdmfTopology> localTopology = grid->getTopology();
if(localTopology->getSize() > 0) {
localTopology->accept(heavyDataWriter);
localTopology->release();
}
}
}
const shared_ptr<XdmfAttribute> currAttribute =
// split attributes
const unsigned int numberAttributes = gridToPartition->getNumberAttributes();
for(unsigned int i=0; i<numberAttributes; ++i) {
const shared_ptr<XdmfAttribute> attribute =
gridToPartition->getAttribute(i);
bool releaseAttribute = false;
if(!currAttribute->isInitialized()) {
currAttribute->read();
if(!attribute->isInitialized()) {
attribute->read();
releaseAttribute = true;
}
unsigned int partitionId = 0;
const shared_ptr<const XdmfAttributeCenter> attributeCenter =
attribute->getCenter();
std::vector<shared_ptr<XdmfAttribute> > localAttributes;
localAttributes.reserve(numberOfPartitions);
if(attributeCenter == XdmfAttributeCenter::Grid()) {
// insert into each partition
for(unsigned int j=0; j<numberOfPartitions; ++j) {
std::map<unsigned int, unsigned int> & currNodeMap =
globalToLocalNodeIdMap[j];
std::vector<unsigned int> & currElemIds = globalElementIds[j];
if(currElemIds.size() > 0) {
const shared_ptr<XdmfUnstructuredGrid> partitioned =
partitionedGrids->getUnstructuredGrid(partitionId);
partitionId++;
shared_ptr<XdmfAttribute> createdAttribute =
shared_ptr<XdmfAttribute>();
if(currAttribute->getCenter() == XdmfAttributeCenter::Grid()) {
// Insert into each partition
createdAttribute = currAttribute;
}
else if(currAttribute->getCenter() == XdmfAttributeCenter::Cell()) {
createdAttribute = XdmfAttribute::New();
createdAttribute->setName(currAttribute->getName());
createdAttribute->setCenter(currAttribute->getCenter());
createdAttribute->setType(currAttribute->getType());
unsigned int index = 0;
const unsigned int numberComponents =
currAttribute->getSize() / topology->getNumberElements();
createdAttribute->initialize(currAttribute->getArrayType(),
currElemIds.size() * numberComponents);
for(std::vector<unsigned int>::const_iterator iter =
currElemIds.begin();
iter != currElemIds.end();
++iter) {
createdAttribute->insert(index,
currAttribute,
*iter * numberComponents,
partitionedGrid->getUnstructuredGrid(j)->insert(attribute);
}
localAttributes.push_back(attribute);
}
else if(attributeCenter == XdmfAttributeCenter::Cell()) {
const unsigned int numberComponents = attribute->getSize() / numElements;
for(unsigned int j=0; j<numberOfPartitions; ++j) {
const shared_ptr<XdmfAttribute> localAttribute = XdmfAttribute::New();
localAttribute->setName(attribute->getName());
localAttribute->setCenter(attribute->getCenter());
localAttribute->setType(attribute->getType());
localAttribute->initialize(attribute->getArrayType(),
localElementCounts[j] * numberComponents);
partitionedGrid->getUnstructuredGrid(j)->insert(localAttribute);
localAttributes.push_back(localAttribute);
}
for(int j=0; j<numElements; ++j) {
const std::pair<unsigned int, unsigned int> & partitionElementPair =
elementIdMap[j];
const unsigned int partitionId = partitionElementPair.first;
const unsigned int localElementId = partitionElementPair.second;
const shared_ptr<XdmfAttribute> localAttribute =
localAttributes[partitionId];
localAttribute->insert(localElementId * numberComponents,
attribute,
j * numberComponents,
numberComponents);
index += numberComponents;
}
}
else if(currAttribute->getCenter() == XdmfAttributeCenter::Node()) {
createdAttribute = XdmfAttribute::New();
createdAttribute->setName(currAttribute->getName());
createdAttribute->setCenter(currAttribute->getCenter());
createdAttribute->setType(currAttribute->getType());
createdAttribute->initialize(currAttribute->getArrayType(),
currNodeMap.size());
const unsigned int numberComponents =
currAttribute->getSize() / geometry->getNumberPoints();
else if(attributeCenter == XdmfAttributeCenter::Node()) {
const unsigned int numberComponents = attribute->getSize() / numNodes;
for(unsigned int j=0; j<numberOfPartitions; ++j) {
const shared_ptr<XdmfAttribute> localAttribute = XdmfAttribute::New();
localAttribute->setName(attribute->getName());
localAttribute->setCenter(attribute->getCenter());
localAttribute->setType(attribute->getType());
localAttribute->initialize(attribute->getArrayType(),
localNodeCounts[j] * numberComponents);
partitionedGrid->getUnstructuredGrid(j)->insert(localAttribute);
localAttributes.push_back(localAttribute);
}
for(int j=0; j<numNodes; ++j) {
const std::map<unsigned int, unsigned int> & localNodeIds =
nodeIdMap[j];
for(std::map<unsigned int, unsigned int>::const_iterator iter =
currNodeMap.begin();
iter != currNodeMap.end();
++iter) {
createdAttribute->insert(iter->second * numberComponents,
currAttribute,
iter->first * numberComponents,
localNodeIds.begin(); iter != localNodeIds.end(); ++iter) {
const unsigned int partitionId = iter->first;
const unsigned int localNodeId = iter->second;
const shared_ptr<XdmfAttribute> localAttribute =
localAttributes[partitionId];
localAttribute->insert(localNodeId * numberComponents,
attribute,
j * numberComponents,
numberComponents);
}
}
if(createdAttribute) {
partitioned->insert(createdAttribute);
if(heavyDataWriter) {
if(!createdAttribute->isInitialized()) {
createdAttribute->read();
}
createdAttribute->accept(heavyDataWriter);
createdAttribute->release();
if(heavyDataWriter) {
for(std::vector<shared_ptr<XdmfAttribute> >::const_iterator iter =
localAttributes.begin(); iter != localAttributes.end(); ++iter) {
const shared_ptr<XdmfAttribute> localAttribute = *iter;
if(!localAttribute->isInitialized()) {
localAttribute->read();
}
if(localAttribute->getSize() > 0) {
localAttribute->accept(heavyDataWriter);
localAttribute->release();
}
}
}
if(releaseAttribute) {
currAttribute->release();