diff --git a/core/XdmfCoreReader.cpp b/core/XdmfCoreReader.cpp index 370b40d98a4bc041044dc5aa68d87568f1117d08..b8b859584c5f72239ffe10dcf7c4bb80b7e9c6f0 100644 --- a/core/XdmfCoreReader.cpp +++ b/core/XdmfCoreReader.cpp @@ -182,17 +182,17 @@ public: } else if (xmlStrcmp(currNode->name, (xmlChar*)"Function") == 0) { - // function handling goes here + // Function handling goes here xmlNodePtr childNode = currNode->children; std::string arraySubType = ""; - // get Array Subtype, if any + // Gget Array Subtype, if any xmlAttrPtr currAttribute = currNode->properties; while (currAttribute != NULL) { if (xmlStrcmp(currAttribute->name, (xmlChar*)"Type") == 0 ) { arraySubType = (char*)currAttribute->children->content; break; - // uses the first type found + // Uses the first type found } } @@ -200,7 +200,7 @@ public: while (childNode != NULL) { if (xmlStrcmp(childNode->name, (xmlChar*)"Expression") == 0){ - // store expression + // Store expression xmlAttrPtr childAttribute = childNode->properties; while (childAttribute != NULL) { if(xmlStrcmp(childAttribute->name, (xmlChar*)"Value") == 0) { @@ -224,13 +224,13 @@ public: } } - // two seperate loops to allow for different orders and multiple variable sets + // Two seperate loops to allow for different orders and multiple variable sets childNode = currNode->children; std::map > variableCollection; while (childNode != NULL) { if (xmlStrcmp(childNode->name, (xmlChar*)"Variable") == 0) { - // store child variables + // Store child variables xmlNodePtr childVariable = childNode->children; while (childVariable != NULL) { @@ -246,31 +246,31 @@ public: std::map typeMap; while (childAttribute != NULL) { - // the variable type of the array + // The variable type of the array if (xmlStrcmp(childAttribute->name, (xmlChar*)"DataType") == 0) { typeMap["DataType"] = (char*)childAttribute->children->content; } - // the precision of the variable type (only used for long and double) + // The precision of the variable type (only used for long and double) else if (xmlStrcmp(childAttribute->name, (xmlChar*)"Precision") == 0) { typeMap["Precision"] = (char*)childAttribute->children->content; } - // the key or mapped string for the variable + // The key or mapped string for the variable else if (xmlStrcmp(childAttribute->name, (xmlChar*)"Key") == 0) { childKey = (char*)childAttribute->children->content; } - // text based xml data + // Text based xml data else if (xmlStrcmp(childAttribute->name, (xmlChar*)"Value") == 0) { dataString = (char*)childAttribute->children->content; } - // an x pointer to another XdmfArray + // An x pointer to another XdmfArray else if (xmlStrcmp(childAttribute->name, (xmlChar*)"XPointer") == 0) { childXPointer = childAttribute->children->content; } - // used in conjunction with Xpointers to reference objects in a different file + // Used in conjunction with Xpointers to reference objects in a different file else if (xmlStrcmp(childAttribute->name, (xmlChar*)"href") == 0) { childhref = childAttribute->children->content; } - // path to hdf5 data sets and the dimensions of those sets + // Path to hdf5 data sets and the dimensions of those sets else if (xmlStrcmp(childAttribute->name, (xmlChar*)"hdf5") == 0) { childhdf5 = (char*)childAttribute->children->content; } @@ -315,7 +315,7 @@ public: dataType = XdmfArrayType::Uninitialized(); } - // if xpointer grab item at that location + // If xpointer grab item at that location if (childXPointer) { xmlXPathContextPtr context = mXPathContext; @@ -338,18 +338,18 @@ public: xmlXPathObjectPtr result = xmlXPtrEval(childXPointer, context); if(result && !xmlXPathNodeSetIsEmpty(result->nodesetval)) { for(int i=0; inodesetval->nodeNr; ++i) { - // there should only be one item being returned here - // place into a new vector + // There should only be one item being returned here + // Place into a new vector std::vector > pointedItems; this->readSingleNode(result->nodesetval->nodeTab[i], pointedItems); try { - // try to cast it as an array + // Try to cast it as an array childArray = shared_dynamic_cast(pointedItems[0]); } catch (...) { - // if that doesn't work throw an error + // If that doesn't work throw an error try { - // because we should only be working with arrays + // Because we should only be working with arrays XdmfError::message(XdmfError::FATAL, "Error: Variable not Equivalent to an Array"); } @@ -366,11 +366,11 @@ public: xmlXPathFreeContext(context); } } - // if hdf5 create controllers and attach it + // If hdf5 create controllers and attach it else if (childhdf5.compare("") != 0) { - // parse the hdf5 controllers + // Parse the hdf5 controllers std::vector controllerParts; - // split the content based on "|" characters + // Split the content based on "|" characters size_t barSplit = 0; std::string splitString(childhdf5); std::string subcontent; @@ -389,7 +389,7 @@ public: controllerParts.push_back(subcontent); } - // insert those controllers into the childArray + // Insert those controllers into the childArray int hdf5step = 2; for (unsigned int i = 0; i < controllerParts.size(); i = i + hdf5step) { size_t colonLocation = controllerParts[i].find(":"); @@ -410,18 +410,18 @@ public: std::vector contentDims; if (i + 1 < controllerParts.size()){ - // this is the string that contains the dimensions + // This is the string that contains the dimensions boost::tokenizer<> dimtokens(controllerParts[i + 1]); for(boost::tokenizer<>::const_iterator iter = dimtokens.begin(); iter != dimtokens.end(); ++iter) { contentDims.push_back(atoi((*iter).c_str())); } - hdf5step = 2;// if this works then the dimension content should be skipped over + hdf5step = 2;// If this works then the dimension content should be skipped over } else { - // if it fails then it means that the next content is not a dimension string - // in this case an error should be thrown, formatting error + // If it fails then it means that the next content is not a dimension string + // In this case an error should be thrown, formatting error // because there is no base array to pull dimensions from try { XdmfError::message(XdmfError::FATAL, @@ -441,9 +441,9 @@ public: contentDims)); } } - // if xml parse strait to insert + // If xml parse strait to insert else if (dataString.compare("") != 0) { - // parse the data into tokens + // Parse the data into tokens childArray->initialize(dataType, 0); unsigned int index = 0; boost::char_separator sep(" \t\n"); @@ -453,7 +453,7 @@ public: iter = tokens.begin(); iter != tokens.end(); ++iter, ++index) { - // insert those tokens into the childArray + // Insert those tokens into the childArray childArray->insert(index, *iter); } } @@ -462,13 +462,13 @@ public: iter = tokens.begin(); iter != tokens.end(); ++iter, ++index) { - // insert those tokens into the childArray + // Insert those tokens into the childArray childArray->insert(index, atof((*iter).c_str())); } } } - // parse the value into the array + // Parse the value into the array if (childKey.compare("") != 0){ if (variableCollection.find(childKey) != variableCollection.end()) { try { @@ -504,25 +504,25 @@ public: catch (XdmfError e) { throw e; } - // the properties and children aren't really needed to generate the object, but the factory still requires them. + // The properties and children aren't really needed to generate the object, but the factory still requires them. std::map newArrayProperties; std::vector > newArrayChildren; shared_ptr returnArray = XdmfArray::New(); if (arraySubType.compare("") == 0) { - // if no type is specified an array is generated + // If no type is specified an array is generated arraySubType = "DataItem"; } - // this should generate an item that corresponds to the tag provided, the casting ensures that it is a subtype of array - // using a factory to be able to build things outside of core + // This should generate an item that corresponds to the tag provided, the casting ensures that it is a subtype of array + // Using a factory to be able to build things outside of core returnArray = shared_dynamic_cast(mItemFactory->createItem( arraySubType, newArrayProperties, newArrayChildren)); if (!returnArray) { - // if the specified tag fails to generate an item then reclass as an array + // If the specified tag fails to generate an item then reclass as an array arraySubType = "DataItem"; returnArray = shared_dynamic_cast(mItemFactory->createItem( arraySubType, @@ -557,15 +557,15 @@ public: readSingleNode(const xmlNodePtr currNode, std::vector > & myItems) { - // check to see if the node is already in the Xpath + // Check to see if the node is already in the Xpath std::map >::const_iterator iter = mXPathMap.find(currNode); - // if it is grab it from the previously stored items + // If it is grab it from the previously stored items if(iter != mXPathMap.end()) { myItems.push_back(iter->second); } else { - // otherwise, generate it from the node + // Otherwise, generate it from the node std::map itemProperties; xmlNodePtr childNode = currNode->children; @@ -575,13 +575,13 @@ public: if(childNode->type == XML_TEXT_NODE && childNode->content) { const char * content = (char*)childNode->content; - // determine if content is whitespace + // Determine if content is whitespace bool whitespace = true; const char * contentPtr = content; - // step through to end of pointer + // Step through to end of pointer while(contentPtr != NULL) { - // if not a whitespace character, break + // If not a whitespace character, break if(!isspace(*contentPtr++)) { whitespace = false; break; @@ -593,7 +593,7 @@ public: itemProperties.insert(std::make_pair("XMLDir", mXMLDir)); } - // split the content based on "|" characters + // Split the content based on "|" characters size_t barSplit = 0; std::string splitString(content); std::string subcontent; diff --git a/core/XdmfDSMBuffer.cpp b/core/XdmfDSMBuffer.cpp index be4a9e5021639fd5b0a89b2005d4bb2c52042b2f..986ae2cc99b46894009a01a7d4fc310890b4e419 100644 --- a/core/XdmfDSMBuffer.cpp +++ b/core/XdmfDSMBuffer.cpp @@ -61,46 +61,45 @@ XdmfDSMBuffer::XdmfDSMBuffer() { - this->CommChannel = XDMF_DSM_INTER_COMM; - this->IsServer = true; - this->StartAddress = this->EndAddress = 0; - this->StartServerId = this->EndServerId = -1; - this->Length = 0; - this->TotalLength = 0; - this->BlockLength = 0; - this->Comm = NULL; - this->DataPointer = NULL; - this->IsConnected = false; + this->CommChannel = XDMF_DSM_INTER_COMM; + this->IsServer = true; + this->StartAddress = this->EndAddress = 0; + this->StartServerId = this->EndServerId = -1; + this->Length = 0; + this->TotalLength = 0; + this->BlockLength = 0; + this->Comm = NULL; + this->DataPointer = NULL; + this->IsConnected = false; } XdmfDSMBuffer::~XdmfDSMBuffer() { - if (this->DataPointer) - { - free(this->DataPointer); - } - this->DataPointer = NULL; + if (this->DataPointer) { + free(this->DataPointer); + } + this->DataPointer = NULL; } class XdmfDSMBuffer::CommandMsg { - public: - int Opcode; - int Source; - int Target; - int Address; - int Length; + public: + int Opcode; + int Source; + int Target; + int Address; + int Length; }; class XdmfDSMBuffer::InfoMsg { - public: - int type; - unsigned int length; - unsigned int total_length; - unsigned int block_length; - int start_server_id; - int end_server_id; + public: + int type; + unsigned int length; + unsigned int total_length; + unsigned int block_length; + int start_server_id; + int end_server_id; }; void @@ -108,509 +107,439 @@ XdmfDSMBuffer::ConfigureUniform(XdmfDSMCommMPI *aComm, long aLength, int startId, int endId, long aBlockLength, bool random) { - if (startId < 0) - { - startId = 0; - } - if (endId < 0) - { - endId = aComm->GetIntraSize() - 1; - } - this->SetDsmType(XDMF_DSM_TYPE_UNIFORM_RANGE); - if ((startId == 0) && (endId == aComm->GetIntraSize() - 1)) - { - this->SetDsmType(XDMF_DSM_TYPE_UNIFORM); - } - if (aBlockLength) - { - if (!random) - { - this->SetDsmType(XDMF_DSM_TYPE_BLOCK_CYCLIC); - } - else - { - this->SetDsmType(XDMF_DSM_TYPE_BLOCK_RANDOM); - } - this->SetBlockLength(aBlockLength); - } - this->StartServerId = startId; - this->EndServerId = endId; - this->SetComm(aComm); - if ((aComm->GetId() >= startId) && (aComm->GetId() <= endId) && this->IsServer) - { - try - { - if (aBlockLength) - { - // For optimization we make the DSM length fit to a multiple of block size - this->SetLength(((long)(aLength / aBlockLength)) * aBlockLength); - } - else - { - this->SetLength(aLength); - } - } - catch (XdmfError e) - { - throw e; - } - this->StartAddress = (aComm->GetId() - startId) * aLength; - this->EndAddress = this->StartAddress + aLength - 1; - } - else - { - if (aBlockLength) - { - this->Length = ((long)(aLength / aBlockLength)) * aBlockLength; - } - else - { - this->Length = aLength; - } - } - this->TotalLength = this->GetLength() * (endId - startId + 1); + if (startId < 0) { + startId = 0; + } + if (endId < 0) { + endId = aComm->GetIntraSize() - 1; + } + this->SetDsmType(XDMF_DSM_TYPE_UNIFORM_RANGE); + if ((startId == 0) && (endId == aComm->GetIntraSize() - 1)) { + this->SetDsmType(XDMF_DSM_TYPE_UNIFORM); + } + if (aBlockLength) { + if (!random) { + this->SetDsmType(XDMF_DSM_TYPE_BLOCK_CYCLIC); + } + else { + this->SetDsmType(XDMF_DSM_TYPE_BLOCK_RANDOM); + } + this->SetBlockLength(aBlockLength); + } + this->StartServerId = startId; + this->EndServerId = endId; + this->SetComm(aComm); + if ((aComm->GetId() >= startId) && (aComm->GetId() <= endId) && this->IsServer) { + try { + if (aBlockLength) { + // For optimization we make the DSM length fit to a multiple of block size + this->SetLength(((long)(aLength / aBlockLength)) * aBlockLength); + } + else { + this->SetLength(aLength); + } + } + catch (XdmfError e) { + throw e; + } + this->StartAddress = (aComm->GetId() - startId) * aLength; + this->EndAddress = this->StartAddress + aLength - 1; + } + else { + if (aBlockLength) { + this->Length = ((long)(aLength / aBlockLength)) * aBlockLength; + } + else { + this->Length = aLength; + } + } + this->TotalLength = this->GetLength() * (endId - startId + 1); } bool XdmfDSMBuffer::GetIsConnected() { - return IsConnected; + return IsConnected; } void XdmfDSMBuffer::SetIsConnected(bool newStatus) { - IsConnected = newStatus; + IsConnected = newStatus; } char * XdmfDSMBuffer::GetDataPointer() { - return this->DataPointer; + return this->DataPointer; } int XdmfDSMBuffer::GetDsmType() { - return this->DsmType; + return this->DsmType; } void XdmfDSMBuffer::SetDsmType(int newDsmType) { - this->DsmType = newDsmType; + this->DsmType = newDsmType; } bool XdmfDSMBuffer::GetIsServer() { - return this->IsServer; + return this->IsServer; } void XdmfDSMBuffer::SetIsServer(bool newIsServer) { - this->IsServer = newIsServer; + this->IsServer = newIsServer; } int XdmfDSMBuffer::GetEndAddress() { - return this->EndAddress; + return this->EndAddress; } int XdmfDSMBuffer::GetStartAddress() { - return this->StartAddress; + return this->StartAddress; } int XdmfDSMBuffer::GetStartServerId() { - return this->StartServerId; + return this->StartServerId; } int XdmfDSMBuffer::GetEndServerId() { - return this->EndServerId; + return this->EndServerId; } long XdmfDSMBuffer::GetLength() { - return this->Length; + return this->Length; } long XdmfDSMBuffer::GetTotalLength() { - return this->TotalLength; + return this->TotalLength; } long XdmfDSMBuffer::GetBlockLength() { - return this->BlockLength; + return this->BlockLength; } void XdmfDSMBuffer::SetBlockLength(long newBlock) { - this->BlockLength = newBlock; + this->BlockLength = newBlock; } XdmfDSMCommMPI * XdmfDSMBuffer::GetComm() { - return this->Comm; + return this->Comm; } void XdmfDSMBuffer::SetComm(XdmfDSMCommMPI * newComm) { - this->Comm = newComm; + this->Comm = newComm; } void XdmfDSMBuffer::SetLength(long aLength) { - this->Length = aLength; - if (this->DataPointer) - { - // try to reallocate - // this should not be called in most cases - this->DataPointer = static_cast(realloc(this->DataPointer, this->Length*sizeof(char))); - } - else - { + this->Length = aLength; + if (this->DataPointer) { + // Try to reallocate + // This should not be called in most cases + this->DataPointer = static_cast(realloc(this->DataPointer, this->Length*sizeof(char))); + } + else { #ifdef _WIN32 - this->DataPointer = calloc(this->Length, sizeof(char)); + this->DataPointer = calloc(this->Length, sizeof(char)); #else - posix_memalign((void **)(&this->DataPointer), getpagesize(), this->Length); - memset(this->DataPointer, 0, this->Length); + posix_memalign((void **)(&this->DataPointer), getpagesize(), this->Length); + memset(this->DataPointer, 0, this->Length); #endif - } - - if (this->DataPointer == NULL) - { - std::stringstream message; - message << "Allocation Failed, unable to allocate " << this->Length; - XdmfError::message(XdmfError::FATAL, message.str()); - } + } + + if (this->DataPointer == NULL) { + std::stringstream message; + message << "Allocation Failed, unable to allocate " << this->Length; + XdmfError::message(XdmfError::FATAL, message.str()); + } } void XdmfDSMBuffer::SendCommandHeader(int opcode, int dest, int address, int aLength, int comm) { - int status; - CommandMsg cmd; - memset(&cmd, 0, sizeof(CommandMsg)); - cmd.Opcode = opcode; - cmd.Source = this->Comm->GetId(); - cmd.Target = dest; - cmd.Address = address; - cmd.Length = aLength; - - if (comm == XDMF_DSM_INTRA_COMM) - { - status = MPI_Send(&cmd, sizeof(CommandMsg), MPI_UNSIGNED_CHAR, dest, XDMF_DSM_COMMAND_TAG, static_cast(this->Comm)->GetIntraComm()); - } - else if (comm == XDMF_DSM_INTER_COMM) - { - int interSource = 0; - MPI_Comm_rank(static_cast(this->Comm)->GetInterComm(), &interSource); - cmd.Source = interSource; - status = MPI_Send(&cmd, sizeof(CommandMsg), MPI_UNSIGNED_CHAR, dest, XDMF_DSM_COMMAND_TAG, static_cast(this->Comm)->GetInterComm()); - } - else - {//in this case the comm should be a pointer to an MPI_Comm object - status = MPI_Send(&cmd, sizeof(CommandMsg), MPI_UNSIGNED_CHAR, dest, XDMF_DSM_COMMAND_TAG, comm); - } - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Error: Failed to send command header"); - } - catch (XdmfError e) - { - throw e; - } - } + int status; + CommandMsg cmd; + memset(&cmd, 0, sizeof(CommandMsg)); + cmd.Opcode = opcode; + cmd.Source = this->Comm->GetId(); + cmd.Target = dest; + cmd.Address = address; + cmd.Length = aLength; + + if (comm == XDMF_DSM_INTRA_COMM) { + status = MPI_Send(&cmd, sizeof(CommandMsg), MPI_UNSIGNED_CHAR, dest, XDMF_DSM_COMMAND_TAG, static_cast(this->Comm)->GetIntraComm()); + } + else if (comm == XDMF_DSM_INTER_COMM) { + int interSource = 0; + MPI_Comm_rank(static_cast(this->Comm)->GetInterComm(), &interSource); + cmd.Source = interSource; + status = MPI_Send(&cmd, sizeof(CommandMsg), MPI_UNSIGNED_CHAR, dest, XDMF_DSM_COMMAND_TAG, static_cast(this->Comm)->GetInterComm()); + } + else { + // In this case the comm should be a pointer to an MPI_Comm object + status = MPI_Send(&cmd, sizeof(CommandMsg), MPI_UNSIGNED_CHAR, dest, XDMF_DSM_COMMAND_TAG, comm); + } + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Error: Failed to send command header"); + } + catch (XdmfError e) { + throw e; + } + } } void XdmfDSMBuffer::ReceiveCommandHeader(int *opcode, int *source, int *address, int *aLength, int comm, int remoteSource) { - CommandMsg cmd; - memset(&cmd, 0, sizeof(CommandMsg)); - int status = MPI_ERR_OTHER; - MPI_Status signalStatus; - - if (remoteSource < 0) - { - remoteSource = MPI_ANY_SOURCE; - } - - if (comm == XDMF_DSM_INTRA_COMM) - { - status = MPI_Recv(&cmd, sizeof(CommandMsg), MPI_UNSIGNED_CHAR, remoteSource, XDMF_DSM_COMMAND_TAG, static_cast(this->Comm)->GetIntraComm(), &signalStatus); - } - else if (comm == XDMF_DSM_INTER_COMM) - { - status = MPI_Recv(&cmd, sizeof(CommandMsg), MPI_UNSIGNED_CHAR, remoteSource, XDMF_DSM_COMMAND_TAG, static_cast(this->Comm)->GetInterComm(), &signalStatus); - } - else - {//in this case the integer is probably a pointer to an MPI_Comm object - status = MPI_Recv(&cmd, sizeof(CommandMsg), MPI_UNSIGNED_CHAR, remoteSource, XDMF_DSM_COMMAND_TAG, comm, &signalStatus); - } - - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Error: Failed to receive command header"); - } - catch (XdmfError e) - { - throw e; - } - } - else - { - *opcode = cmd.Opcode; + CommandMsg cmd; + memset(&cmd, 0, sizeof(CommandMsg)); + int status = MPI_ERR_OTHER; + MPI_Status signalStatus; + + if (remoteSource < 0) { + remoteSource = MPI_ANY_SOURCE; + } + + if (comm == XDMF_DSM_INTRA_COMM) { + status = MPI_Recv(&cmd, sizeof(CommandMsg), MPI_UNSIGNED_CHAR, remoteSource, XDMF_DSM_COMMAND_TAG, static_cast(this->Comm)->GetIntraComm(), &signalStatus); + } + else if (comm == XDMF_DSM_INTER_COMM) { + status = MPI_Recv(&cmd, sizeof(CommandMsg), MPI_UNSIGNED_CHAR, remoteSource, XDMF_DSM_COMMAND_TAG, static_cast(this->Comm)->GetInterComm(), &signalStatus); + } + else { + // In this case the integer is probably a pointer to an MPI_Comm object + status = MPI_Recv(&cmd, sizeof(CommandMsg), MPI_UNSIGNED_CHAR, remoteSource, XDMF_DSM_COMMAND_TAG, comm, &signalStatus); + } + + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Error: Failed to receive command header"); + } + catch (XdmfError e) { + throw e; + } + } + else { + *opcode = cmd.Opcode; *source = cmd.Source; - *address = cmd.Address; - *aLength = cmd.Length; - } + *address = cmd.Address; + *aLength = cmd.Length; + } } void XdmfDSMBuffer::SendData(int dest, char * data, int aLength, int tag, int aAddress, int comm) { - int status; - if (comm == XDMF_DSM_INTRA_COMM) - { - status = MPI_Send(data, aLength, MPI_UNSIGNED_CHAR, dest, tag, static_cast(this->Comm)->GetIntraComm()); - } - else if (comm == XDMF_DSM_INTER_COMM) - { - status = MPI_Send(data, aLength, MPI_UNSIGNED_CHAR, dest, tag, static_cast(this->Comm)->GetInterComm()); - } - else - { - status = MPI_Send(data, aLength, MPI_UNSIGNED_CHAR, dest, tag, comm); - } - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Error: Failed to send data"); - } - catch (XdmfError e) - { - throw e; - } - } + int status; + if (comm == XDMF_DSM_INTRA_COMM) { + status = MPI_Send(data, aLength, MPI_UNSIGNED_CHAR, dest, tag, static_cast(this->Comm)->GetIntraComm()); + } + else if (comm == XDMF_DSM_INTER_COMM) { + status = MPI_Send(data, aLength, MPI_UNSIGNED_CHAR, dest, tag, static_cast(this->Comm)->GetInterComm()); + } + else { + status = MPI_Send(data, aLength, MPI_UNSIGNED_CHAR, dest, tag, comm); + } + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Error: Failed to send data"); + } + catch (XdmfError e) { + throw e; + } + } } void XdmfDSMBuffer::ReceiveData(int source, char * data, int aLength, int tag, int aAddress, int comm) { - int status; - MPI_Status signalStatus; - if (comm == XDMF_DSM_INTRA_COMM) - { - status = MPI_Recv(data, aLength, MPI_UNSIGNED_CHAR, source, tag, static_cast(this->Comm)->GetIntraComm(), &signalStatus); - } - else if (comm == XDMF_DSM_INTER_COMM) - { - status = MPI_Recv(data, aLength, MPI_UNSIGNED_CHAR, source, tag, static_cast(this->Comm)->GetInterComm(), &signalStatus); - } - else - { - status = MPI_Recv(data, aLength, MPI_UNSIGNED_CHAR, source, tag, comm, &signalStatus); - } - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Error: Failed to receive data"); - } - catch (XdmfError e) - { - throw e; - } - } + int status; + MPI_Status signalStatus; + if (comm == XDMF_DSM_INTRA_COMM) { + status = MPI_Recv(data, aLength, MPI_UNSIGNED_CHAR, source, tag, static_cast(this->Comm)->GetIntraComm(), &signalStatus); + } + else if (comm == XDMF_DSM_INTER_COMM) { + status = MPI_Recv(data, aLength, MPI_UNSIGNED_CHAR, source, tag, static_cast(this->Comm)->GetInterComm(), &signalStatus); + } + else { + status = MPI_Recv(data, aLength, MPI_UNSIGNED_CHAR, source, tag, comm, &signalStatus); + } + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Error: Failed to receive data"); + } + catch (XdmfError e) { + throw e; + } + } } void XdmfDSMBuffer::SendAcknowledgment(int dest, int data, int tag, int comm) { - int status; - - if (comm == XDMF_DSM_INTRA_COMM) - { - status = MPI_Send(&data, sizeof(int), MPI_UNSIGNED_CHAR, dest, tag, static_cast(this->Comm)->GetIntraComm()); - } - else if (comm == XDMF_DSM_INTER_COMM) - { - status = MPI_Send(&data, sizeof(int), MPI_UNSIGNED_CHAR, dest, tag, static_cast(this->Comm)->GetInterComm()); - } - else - { - status = MPI_Send(&data, sizeof(int), MPI_UNSIGNED_CHAR, dest, tag, comm); - } - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Error: Failed to receive data"); - } - catch (XdmfError e) - { - throw e; - } - } + int status; + + if (comm == XDMF_DSM_INTRA_COMM) { + status = MPI_Send(&data, sizeof(int), MPI_UNSIGNED_CHAR, dest, tag, static_cast(this->Comm)->GetIntraComm()); + } + else if (comm == XDMF_DSM_INTER_COMM) { + status = MPI_Send(&data, sizeof(int), MPI_UNSIGNED_CHAR, dest, tag, static_cast(this->Comm)->GetInterComm()); + } + else { + status = MPI_Send(&data, sizeof(int), MPI_UNSIGNED_CHAR, dest, tag, comm); + } + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Error: Failed to receive data"); + } + catch (XdmfError e) { + throw e; + } + } } void XdmfDSMBuffer::ReceiveAcknowledgment(int source, int &data, int tag, int comm) { - int status; - MPI_Status signalStatus; - if (comm == XDMF_DSM_INTRA_COMM) - { - status = MPI_Recv(&data, sizeof(int), MPI_UNSIGNED_CHAR, source, tag, static_cast(this->Comm)->GetIntraComm(), &signalStatus); - } - else if (comm == XDMF_DSM_INTER_COMM) - { - status = MPI_Recv(&data, sizeof(int), MPI_UNSIGNED_CHAR, source, tag, static_cast(this->Comm)->GetInterComm(), &signalStatus); - } - else - { - status = MPI_Recv(&data, sizeof(int), MPI_UNSIGNED_CHAR, source, tag, comm, &signalStatus); - } - - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Error: Failed to receive data"); - } - catch (XdmfError e) - { - throw e; - } - } + int status; + MPI_Status signalStatus; + if (comm == XDMF_DSM_INTRA_COMM) { + status = MPI_Recv(&data, sizeof(int), MPI_UNSIGNED_CHAR, source, tag, static_cast(this->Comm)->GetIntraComm(), &signalStatus); + } + else if (comm == XDMF_DSM_INTER_COMM) { + status = MPI_Recv(&data, sizeof(int), MPI_UNSIGNED_CHAR, source, tag, static_cast(this->Comm)->GetInterComm(), &signalStatus); + } + else { + status = MPI_Recv(&data, sizeof(int), MPI_UNSIGNED_CHAR, source, tag, comm, &signalStatus); + } + + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Error: Failed to receive data"); + } + catch (XdmfError e) { + throw e; + } + } } void XdmfDSMBuffer::SendInfo() { - InfoMsg dsmInfo; - int status; - - memset(&dsmInfo, 0, sizeof(InfoMsg)); - dsmInfo.type = this->GetDsmType(); - dsmInfo.length = this->GetLength(); - dsmInfo.total_length = this->GetTotalLength(); - dsmInfo.block_length = this->GetBlockLength(); - dsmInfo.start_server_id = this->GetStartServerId(); - dsmInfo.end_server_id = this->GetEndServerId(); - if (this->Comm->GetId() == 0) - { - status = MPI_Send(&dsmInfo, sizeof(InfoMsg), MPI_UNSIGNED_CHAR, 0, XDMF_DSM_EXCHANGE_TAG, static_cast(this->Comm)->GetInterComm()); - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Error: Failed to send info"); - } - catch (XdmfError e) - { - throw e; - } - } - } - status = MPI_Barrier(this->Comm->GetIntraComm()); - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Error: Failed to send info"); - } - catch (XdmfError e) - { - throw e; - } - } + InfoMsg dsmInfo; + int status; + + memset(&dsmInfo, 0, sizeof(InfoMsg)); + dsmInfo.type = this->GetDsmType(); + dsmInfo.length = this->GetLength(); + dsmInfo.total_length = this->GetTotalLength(); + dsmInfo.block_length = this->GetBlockLength(); + dsmInfo.start_server_id = this->GetStartServerId(); + dsmInfo.end_server_id = this->GetEndServerId(); + if (this->Comm->GetId() == 0) { + status = MPI_Send(&dsmInfo, sizeof(InfoMsg), MPI_UNSIGNED_CHAR, 0, XDMF_DSM_EXCHANGE_TAG, static_cast(this->Comm)->GetInterComm()); + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Error: Failed to send info"); + } + catch (XdmfError e) { + throw e; + } + } + } + status = MPI_Barrier(this->Comm->GetIntraComm()); + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Error: Failed to send info"); + } + catch (XdmfError e) { + throw e; + } + } } void XdmfDSMBuffer::ReceiveInfo() { - InfoMsg dsmInfo; - int status; - MPI_Status signalStatus; - - memset(&dsmInfo, 0, sizeof(InfoMsg)); - if (this->Comm->GetId() == 0) - { - status = MPI_Recv(&dsmInfo, sizeof(InfoMsg), MPI_UNSIGNED_CHAR, XDMF_DSM_ANY_SOURCE, XDMF_DSM_EXCHANGE_TAG, static_cast(this->Comm)->GetInterComm(), &signalStatus); - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Error: Failed to receive info"); - } - catch (XdmfError e) - { - throw e; - } - } - } - status = MPI_Bcast(&dsmInfo, sizeof(InfoMsg), MPI_UNSIGNED_CHAR, 0, static_cast(this->Comm)->GetIntraComm()); - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Error: Failed to broadcast info"); - } - catch (XdmfError e) - { - throw e; - } - } - this->SetDsmType(dsmInfo.type); - // We are a client so don't allocate anything but only set a virtual remote length - this->SetLength(dsmInfo.length); - this->TotalLength = dsmInfo.total_length; - this->SetBlockLength(dsmInfo.block_length); - this->StartServerId = dsmInfo.start_server_id; - this->EndServerId = dsmInfo.end_server_id; + InfoMsg dsmInfo; + int status; + MPI_Status signalStatus; + + memset(&dsmInfo, 0, sizeof(InfoMsg)); + if (this->Comm->GetId() == 0) { + status = MPI_Recv(&dsmInfo, sizeof(InfoMsg), MPI_UNSIGNED_CHAR, XDMF_DSM_ANY_SOURCE, XDMF_DSM_EXCHANGE_TAG, static_cast(this->Comm)->GetInterComm(), &signalStatus); + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Error: Failed to receive info"); + } + catch (XdmfError e) { + throw e; + } + } + } + status = MPI_Bcast(&dsmInfo, sizeof(InfoMsg), MPI_UNSIGNED_CHAR, 0, static_cast(this->Comm)->GetIntraComm()); + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Error: Failed to broadcast info"); + } + catch (XdmfError e) { + throw e; + } + } + this->SetDsmType(dsmInfo.type); + // We are a client so don't allocate anything but only set a virtual remote length + this->SetLength(dsmInfo.length); + this->TotalLength = dsmInfo.total_length; + this->SetBlockLength(dsmInfo.block_length); + this->StartServerId = dsmInfo.start_server_id; + this->EndServerId = dsmInfo.end_server_id; } void XdmfDSMBuffer::BroadcastComm(int *comm, int root) { - int status; - - status = MPI_Bcast(comm, sizeof(int), MPI_UNSIGNED_CHAR, root, this->Comm->GetIntraComm()); - if (status != MPI_SUCCESS) - { - try - { - XdmfError(XdmfError::FATAL, "Broadcast of Comm failed"); - } - catch (XdmfError e) - { - throw e; - } - } + int status; + + status = MPI_Bcast(comm, sizeof(int), MPI_UNSIGNED_CHAR, root, this->Comm->GetIntraComm()); + if (status != MPI_SUCCESS) { + try { + XdmfError(XdmfError::FATAL, "Broadcast of Comm failed"); + } + catch (XdmfError e) { + throw e; + } + } } //redefined from H5FDBufferService @@ -671,8 +600,7 @@ XdmfDSMBuffer::BufferService(int *returnOpcode) - // connection is an ID for client or server, - // we can use the communicator ID interchangably, but if the architecture is altered - be careful + // Connection is an ID for client or server, int communicatorId = this->CommChannel; switch(opcode) { @@ -739,102 +667,18 @@ XdmfDSMBuffer::BufferService(int *returnOpcode) // H5FD_DSM_LOCK_ACQUIRE // Comes from client or server depending on communicator -// case XDMF_DSM_LOCK_ACQUIRE: - //is->SendAcknowledgment(who, -1, XDMF_DSM_CLIENT_ACK_TAG, this->CommChannel); -/* // wait for all processes to sync before doing anything - if (this->Comm->ChannelSynced(who, &syncId, communicatorId)) { - // only rank 0 actually handles the lock, the other ranks just mimic it later when they get an acknowledgement - if (this->Comm->GetId() == 0) { - if (this->BufferServiceInternals->BufferLock.Lock(communicatorId)) { - // notify all other server nodes - to update their local locks to match rank 0 - H5FDdsmInt32 numberOfRanks = this->Comm->GetIntraSize(); - for (H5FDdsmInt32 who = 1; who < numberOfRanks; ++who) { - this->SendAcknowledgment(who, communicatorId, H5FD_DSM_SERVER_ACK_TAG, H5FD_DSM_INTRA_COMM); - } - // notify the ranks that made the request - numberOfRanks = (communicatorId==H5FD_DSM_SERVER_ID) ? this->Comm->GetIntraSize() : this->Comm->GetInterSize(); - for (H5FDdsmInt32 who = 0; who < numberOfRanks; ++who) { - this->SendAcknowledgment(who, communicatorId, H5FD_DSM_CLIENT_ACK_TAG, this->CommChannel); - } - } - // we were not given the lock, so go back to listening for anyone - else { - this->CommChannel = H5FD_DSM_ANY_COMM; - // notify all other server nodes that lock request failed and to change communicator - H5FDdsmInt32 numberOfRanks = this->Comm->GetIntraSize(); - for (H5FDdsmInt32 who = 1; who < numberOfRanks; ++who) { - this->SendAcknowledgment(who, -1, H5FD_DSM_SERVER_ACK_TAG, H5FD_DSM_INTRA_COMM); - } - } - } - else { - // all server nodes need to update their local locks to match rank 0 - this->ReceiveAcknowledgment(0, communicatorId, H5FD_DSM_SERVER_ACK_TAG, H5FD_DSM_INTRA_COMM); - // the lock request failed, so we don't give the lock to the requestor - if (communicatorId == -1) { - this->CommChannel = H5FD_DSM_ANY_COMM; - } else { - this->BufferServiceInternals->BufferLock.Lock(communicatorId); - } - } - }*/ -// break; + case XDMF_DSM_LOCK_ACQUIRE: + // Currently unsupported + break; // H5FD_DSM_LOCK_RELEASE // Comes from client or server depending on communicator -// case XDMF_DSM_LOCK_RELEASE: -/* // wait for all processes to sync before doing anything - if (this->Comm->ChannelSynced(who, &syncId, communicatorId)) { - // only rank 0 actually handles the lock, the other ranks just mimic it later when they get an acknowledgement - H5FDdsmInt32 newLockOwner = -1; - if (this->Comm->GetId() == 0) { - // When we release the lock, it may be passed straight to the next owner, - // if this happens, we must inform the other server nodes who the owner is - newLockOwner = this->BufferServiceInternals->BufferLock.Unlock(communicatorId); - H5FDdsmInt32 numberOfRanks = this->Comm->GetIntraSize(); - for (H5FDdsmInt32 who = 1; who < numberOfRanks; ++who) { - this->SendAcknowledgment(who, newLockOwner, H5FD_DSM_SERVER_ACK_TAG, H5FD_DSM_INTRA_COMM); - } - } else { - // all server nodes need to update their local locks to match rank 0 - this->ReceiveAcknowledgment(0, newLockOwner, H5FD_DSM_SERVER_ACK_TAG, H5FD_DSM_INTRA_COMM); - this->BufferServiceInternals->BufferLock.Unlock(communicatorId); - } - - // - // the lock has been released : if the client unlocked, wake up waiting server app thread - // note that a lock count decrease returns the same lock owner, so we don't trigger on that event - // - if (newLockOwner != communicatorId && communicatorId == H5FD_DSM_CLIENT_ID) { - // the address flag holds our unlock status (only treat it when received from client) - this->SignalUnlock(address, H5FD_DSM_FALSE); - } - // - // if it has been taken by another communicator/connection, do what's needed - // - if (newLockOwner == -1) { - this->CommChannel = H5FD_DSM_ANY_COMM; - H5FDdsmDebug("Lock released, Switched to " << H5FDdsmCommToString(this->CommChannel)); - } - else if (newLockOwner != communicatorId) { - this->CommChannel = newLockOwner; - H5FDdsmDebug("Lock retaken, Switched to " << H5FDdsmCommToString(this->CommChannel)); - if (this->Comm->GetId() != 0) { - newLockOwner = this->BufferServiceInternals->BufferLock.Lock(newLockOwner); - } - if (this->Comm->GetId() == 0) { - // notify the ranks that made the original lock request - H5FDdsmInt32 numberOfRanks = (newLockOwner == H5FD_DSM_SERVER_ID) ? this->Comm->GetIntraSize() : this->Comm->GetInterSize(); - for (H5FDdsmInt32 who = 0; who < numberOfRanks; ++who) { - this->SendAcknowledgment(who, newLockOwner, H5FD_DSM_CLIENT_ACK_TAG, this->CommChannel); - } - } - } - }*/ -// break; + case XDMF_DSM_LOCK_RELEASE: + // Currently unsupported + break; // H5FD_DSM_OPCODE_DONE - // Always received from server + // Always received on server case XDMF_DSM_OPCODE_DONE: break; @@ -859,7 +703,7 @@ void XdmfDSMBuffer::SendDone() { try { - if (static_cast(this->Comm)->GetInterComm() == MPI_COMM_NULL)//this only stops the first core controlled by the server + if (static_cast(this->Comm)->GetInterComm() == MPI_COMM_NULL) { for (int i = this->StartServerId; i < this->EndServerId; ++i) { if (i != this->Comm->GetId()){ @@ -882,8 +726,9 @@ XdmfDSMBuffer::SendDone() } void -XdmfDSMBuffer::ProbeCommandHeader(int *comm)//used for finding a comm that has a waiting command, then sets the comm +XdmfDSMBuffer::ProbeCommandHeader(int *comm) { + // Used for finding a comm that has a waiting command, then sets the comm int status = XDMF_DSM_FAIL; MPI_Status signalStatus; @@ -908,11 +753,11 @@ XdmfDSMBuffer::ProbeCommandHeader(int *comm)//used for finding a comm that has a else { if (static_cast(this->Comm)->GetInterComm() != MPI_COMM_NULL) { if (probeComm == static_cast(this->Comm)->GetIntraComm()) { - probeComm = static_cast(this->Comm)->GetInterComm(); - } - else { - probeComm = static_cast(this->Comm)->GetIntraComm(); - } + probeComm = static_cast(this->Comm)->GetInterComm(); + } + else { + probeComm = static_cast(this->Comm)->GetIntraComm(); + } } } } @@ -929,37 +774,38 @@ XdmfDSMBuffer::ProbeCommandHeader(int *comm)//used for finding a comm that has a int -XdmfDSMBuffer::AddressToId(int Address){ - int ServerId = XDMF_DSM_FAIL; - - switch(this->DsmType) { - case XDMF_DSM_TYPE_UNIFORM : - case XDMF_DSM_TYPE_UNIFORM_RANGE : - // All Servers have same length - // This finds out which server the address provided starts on - ServerId = this->StartServerId + (Address / this->Length); - if(ServerId > this->EndServerId ){ - try { - std::stringstream message; - message << "ServerId " << ServerId << " for Address " << Address << " is larger than EndServerId " << this->EndServerId; - XdmfError::message(XdmfError::FATAL, message.str()); - } - catch (XdmfError e) { - throw e; - } - } - break; - default : - // Not Implemented - try { - std::stringstream message; - message << "DsmType " << this->DsmType << " not yet implemented"; - XdmfError::message(XdmfError::FATAL, message.str()); - } - catch (XdmfError e) { - throw e; - } - break; +XdmfDSMBuffer::AddressToId(int Address) +{ + int ServerId = XDMF_DSM_FAIL; + + switch(this->DsmType) { + case XDMF_DSM_TYPE_UNIFORM : + case XDMF_DSM_TYPE_UNIFORM_RANGE : + // All Servers have same length + // This finds out which server the address provided starts on + ServerId = this->StartServerId + (Address / this->Length); + if(ServerId > this->EndServerId ){ + try { + std::stringstream message; + message << "ServerId " << ServerId << " for Address " << Address << " is larger than EndServerId " << this->EndServerId; + XdmfError::message(XdmfError::FATAL, message.str()); + } + catch (XdmfError e) { + throw e; + } + } + break; + default : + // Not Implemented + try { + std::stringstream message; + message << "DsmType " << this->DsmType << " not yet implemented"; + XdmfError::message(XdmfError::FATAL, message.str()); + } + catch (XdmfError e) { + throw e; + } + break; } return(ServerId); } @@ -967,146 +813,144 @@ XdmfDSMBuffer::AddressToId(int Address){ void XdmfDSMBuffer::GetAddressRangeForId(int Id, int *Start, int *End){ switch(this->DsmType) { - case XDMF_DSM_TYPE_UNIFORM : - case XDMF_DSM_TYPE_UNIFORM_RANGE : - // All Servers have same length - // Start index is equal to the id inside the servers times the length of the block per server - // It is the starting index of the server's data block relative to the entire block - *Start = (Id - this->StartServerId) * this->Length; - // End index is simply the start index + the length of the server's data block. - // The range produced is the start of the server's data block to its end. - *End = *Start + Length - 1; - break; - default : - // Not Implemented - try { - std::stringstream message; - message << "DsmType " << this->DsmType << " not yet implemented"; - XdmfError::message(XdmfError::FATAL, message.str()); - } - catch (XdmfError e) { - throw e; - } - break; + case XDMF_DSM_TYPE_UNIFORM : + case XDMF_DSM_TYPE_UNIFORM_RANGE : + // All Servers have same length + // Start index is equal to the id inside the servers times the length of the block per server + // It is the starting index of the server's data block relative to the entire block + *Start = (Id - this->StartServerId) * this->Length; + // End index is simply the start index + the length of the server's data block. + // The range produced is the start of the server's data block to its end. + *End = *Start + Length - 1; + break; + default : + // Not Implemented + try { + std::stringstream message; + message << "DsmType " << this->DsmType << " not yet implemented"; + XdmfError::message(XdmfError::FATAL, message.str()); + } + catch (XdmfError e) { + throw e; + } + break; } } void XdmfDSMBuffer::Get(long Address, long aLength, void *Data) { - int who, MyId = this->Comm->GetId(); - int astart, aend, len; - char *datap = (char *)Data; + int who, MyId = this->Comm->GetId(); + int astart, aend, len; + char *datap = (char *)Data; - // While there is length left - while(aLength) - { - // Figure out what server core the address is located on - who = this->AddressToId(Address); - if(who == XDMF_DSM_FAIL){ - try { - XdmfError::message(XdmfError::FATAL, "Address Error"); - } - catch (XdmfError e) { - throw e; - } - } - // Get the start and end of the block listed - this->GetAddressRangeForId(who, &astart, &aend); - // Determine the amount of data to be written to that core - // Basically, it's how much data will fit from the starting point of the address to the end - len = std::min(aLength, aend - Address + 1); - // If the data is on the core running this code, then the put is simple - if(who == MyId){ - char *dp; - - dp = this->DataPointer; - dp += Address - this->StartAddress; - memcpy(datap, dp, len); - - }else{ - // Otherwise send it to the appropriate core to deal with - int status; - int dataComm = XDMF_DSM_INTRA_COMM; - if (this->Comm->GetInterComm() != MPI_COMM_NULL) { - dataComm = XDMF_DSM_INTER_COMM; - } - try { - this->SendCommandHeader(XDMF_DSM_OPCODE_GET, who, Address - astart, len, dataComm); - } - catch (XdmfError e) { - throw e; - } - try { - this->ReceiveData(who, datap, len, XDMF_DSM_GET_DATA_TAG, Address - astart, dataComm); - } - catch (XdmfError e) { - throw e; - } - } - // Shift all the numbers by the length of the data written - // Until aLength = 0 - aLength -= len; - Address += len; - datap += len; + // While there is length left + while(aLength) { + // Figure out what server core the address is located on + who = this->AddressToId(Address); + if(who == XDMF_DSM_FAIL){ + try { + XdmfError::message(XdmfError::FATAL, "Address Error"); + } + catch (XdmfError e) { + throw e; + } + } + // Get the start and end of the block listed + this->GetAddressRangeForId(who, &astart, &aend); + // Determine the amount of data to be written to that core + // Basically, it's how much data will fit from the starting point of the address to the end + len = std::min(aLength, aend - Address + 1); + // If the data is on the core running this code, then the put is simple + if(who == MyId){ + char *dp; + dp = this->DataPointer; + dp += Address - this->StartAddress; + memcpy(datap, dp, len); + } + else{ + // Otherwise send it to the appropriate core to deal with + int status; + int dataComm = XDMF_DSM_INTRA_COMM; + if (this->Comm->GetInterComm() != MPI_COMM_NULL) { + dataComm = XDMF_DSM_INTER_COMM; + } + try { + this->SendCommandHeader(XDMF_DSM_OPCODE_GET, who, Address - astart, len, dataComm); + } + catch (XdmfError e) { + throw e; + } + try { + this->ReceiveData(who, datap, len, XDMF_DSM_GET_DATA_TAG, Address - astart, dataComm); + } + catch (XdmfError e) { + throw e; + } } + // Shift all the numbers by the length of the data written + // Until aLength = 0 + aLength -= len; + Address += len; + datap += len; + } } void -XdmfDSMBuffer::Put(long Address, long aLength, const void *Data){ - int who, MyId = this->Comm->GetId(); - int astart, aend, len; - char *datap = (char *)Data; - - // While there is length left - while(aLength){ - // Figure out what server core the address is located on - who = this->AddressToId(Address); - if(who == XDMF_DSM_FAIL){ - try { - XdmfError::message(XdmfError::FATAL, "Address Error"); - } - catch (XdmfError e) { - throw e; - } - } - // Get the start and end of the block listed - this->GetAddressRangeForId(who, &astart, &aend); - // Determine the amount of data to be written to that core - // Basically, it's how much data will fit from the starting point of the address to the end - len = std::min(aLength, aend - Address + 1); - // If the data is on the core running this code, then the put is simple - if(who == MyId){ - char *dp; - - dp = this->DataPointer; - dp += Address - this->StartAddress; - memcpy(dp, datap, len); - - }else{ - // Otherwise send it to the appropriate core to deal with - int status; - int dataComm = XDMF_DSM_INTRA_COMM; - if (this->Comm->GetInterComm() != MPI_COMM_NULL) { - dataComm = XDMF_DSM_INTER_COMM; - } - try { - this->SendCommandHeader(XDMF_DSM_OPCODE_PUT, who, Address - astart, len, dataComm); - } - catch (XdmfError e) { - throw e; - } - try { - this->SendData(who, datap, len, XDMF_DSM_PUT_DATA_TAG, Address - astart, dataComm); - } - catch (XdmfError e) { - throw e; - } - } - // Shift all the numbers by the length of the data written - // Until aLength = 0 - aLength -= len; - Address += len; - datap += len; +XdmfDSMBuffer::Put(long Address, long aLength, const void *Data) +{ + int who, MyId = this->Comm->GetId(); + int astart, aend, len; + char *datap = (char *)Data; + + // While there is length left + while(aLength){ + // Figure out what server core the address is located on + who = this->AddressToId(Address); + if(who == XDMF_DSM_FAIL){ + try { + XdmfError::message(XdmfError::FATAL, "Address Error"); + } + catch (XdmfError e) { + throw e; + } + } + // Get the start and end of the block listed + this->GetAddressRangeForId(who, &astart, &aend); + // Determine the amount of data to be written to that core + // Basically, it's how much data will fit from the starting point of the address to the end + len = std::min(aLength, aend - Address + 1); + // If the data is on the core running this code, then the put is simple + if(who == MyId){ + char *dp; + dp = this->DataPointer; + dp += Address - this->StartAddress; + memcpy(dp, datap, len); } + else{ + // Otherwise send it to the appropriate core to deal with + int status; + int dataComm = XDMF_DSM_INTRA_COMM; + if (this->Comm->GetInterComm() != MPI_COMM_NULL) { + dataComm = XDMF_DSM_INTER_COMM; + } + try { + this->SendCommandHeader(XDMF_DSM_OPCODE_PUT, who, Address - astart, len, dataComm); + } + catch (XdmfError e) { + throw e; + } + try { + this->SendData(who, datap, len, XDMF_DSM_PUT_DATA_TAG, Address - astart, dataComm); + } + catch (XdmfError e) { + throw e; + } + } + // Shift all the numbers by the length of the data written + // Until aLength = 0 + aLength -= len; + Address += len; + datap += len; + } } diff --git a/core/XdmfDSMCommMPI.cpp b/core/XdmfDSMCommMPI.cpp index 64a163220cec82ec61c32db274547aee787a71ba..1131e7fe157e19d7acc09e95276a0ddfc9c9d488 100644 --- a/core/XdmfDSMCommMPI.cpp +++ b/core/XdmfDSMCommMPI.cpp @@ -58,348 +58,287 @@ XdmfDSMCommMPI::XdmfDSMCommMPI() { - IntraComm = MPI_COMM_NULL; - Id = -1; - IntraSize = -1; - InterComm = MPI_COMM_NULL; - InterSize = -1; - SetDsmMasterHostName(""); - InterCommType = XDMF_DSM_COMM_MPI; + IntraComm = MPI_COMM_NULL; + Id = -1; + IntraSize = -1; + InterComm = MPI_COMM_NULL; + InterSize = -1; + SetDsmMasterHostName(""); + InterCommType = XDMF_DSM_COMM_MPI; } XdmfDSMCommMPI::~XdmfDSMCommMPI() { - if (InterComm != MPI_COMM_NULL) - { - int status = MPI_Comm_free(&InterComm); - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Failed to free intercomm Comm"); - } - catch (XdmfError e) - { - throw e; - } - } - } - if (IntraComm != MPI_COMM_NULL) - { - int status = MPI_Comm_free(&IntraComm); - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Failed to free intercomm Comm"); - } - catch (XdmfError e) - { - throw e; - } - } - } + if (InterComm != MPI_COMM_NULL) { + int status = MPI_Comm_free(&InterComm); + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Failed to free intercomm Comm"); + } + catch (XdmfError e) { + throw e; + } + } + } + if (IntraComm != MPI_COMM_NULL) { + int status = MPI_Comm_free(&IntraComm); + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Failed to free intercomm Comm"); + } + catch (XdmfError e) { + throw e; + } + } + } } void XdmfDSMCommMPI::SetDsmMasterHostName(const char *hostName) { - strcpy(DsmMasterHostName, hostName); + strcpy(DsmMasterHostName, hostName); } char * XdmfDSMCommMPI::GetDsmMasterHostName() { - return DsmMasterHostName; + return DsmMasterHostName; } int XdmfDSMCommMPI::GetId() { - return this->Id; + return this->Id; } int XdmfDSMCommMPI::GetIntraSize() { - return this->IntraSize; + return this->IntraSize; } int XdmfDSMCommMPI::GetInterSize() { - return this->InterSize; + return this->InterSize; } int XdmfDSMCommMPI::GetInterCommType() { - return this->InterCommType; + return this->InterCommType; } void XdmfDSMCommMPI::Init() { - int size, rank; - if (MPI_Comm_size(this->IntraComm, &size) != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Failed to initialize size"); - } - catch (XdmfError e) - { - throw e; - } - } - if (MPI_Comm_rank(this->IntraComm, &rank) != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Failed to initialize rank"); - } - catch (XdmfError e) - { - throw e; - } - } + int size, rank; + if (MPI_Comm_size(this->IntraComm, &size) != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Failed to initialize size"); + } + catch (XdmfError e) { + throw e; + } + } + if (MPI_Comm_rank(this->IntraComm, &rank) != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Failed to initialize rank"); + } + catch (XdmfError e) { + throw e; + } + } - this->Id = rank; - this->IntraSize = size; + this->Id = rank; + this->IntraSize = size; } -//possibly use setenv/unsetenv/getenv to pass dsm port -//http://linux.die.net/man/3/setenv - void XdmfDSMCommMPI::OpenPort() { - if (Id == 0) - { - int status = MPI_Open_port(MPI_INFO_NULL, DsmMasterHostName); - if (status != MPI_SUCCESS) - { - try - { - std::string message = "Failed to open port "; - message = message + DsmMasterHostName; - XdmfError::message(XdmfError::FATAL, message); - } - catch (XdmfError e) - { - throw e; - } - } - } + if (Id == 0) { + int status = MPI_Open_port(MPI_INFO_NULL, DsmMasterHostName); + if (status != MPI_SUCCESS) { + try { + std::string message = "Failed to open port "; + message = message + DsmMasterHostName; + XdmfError::message(XdmfError::FATAL, message); + } + catch (XdmfError e) { + throw e; + } + } + } } void XdmfDSMCommMPI::ClosePort() { - if (Id == 0) - { - int status; - status = MPI_Open_port(MPI_INFO_NULL, DsmMasterHostName); - if (status != MPI_SUCCESS) - { - try - { - std::string message = "Failed to close port "; - message = message + DsmMasterHostName; - XdmfError::message(XdmfError::FATAL, message); - } - catch (XdmfError e) - { - throw e; - } - } - } + if (Id == 0) { + int status; + status = MPI_Open_port(MPI_INFO_NULL, DsmMasterHostName); + if (status != MPI_SUCCESS) { + try { + std::string message = "Failed to close port "; + message = message + DsmMasterHostName; + XdmfError::message(XdmfError::FATAL, message); + } + catch (XdmfError e) { + throw e; + } + } + } } void XdmfDSMCommMPI::Accept() { - int status = MPI_Comm_accept(DsmMasterHostName, MPI_INFO_NULL, 0, IntraComm, &InterComm); - if (status != MPI_SUCCESS) - { - try - { - std::string message = "Failed to accept port "; - message = message + DsmMasterHostName; - XdmfError::message(XdmfError::FATAL, message); - } - catch (XdmfError e) - { - throw e; - } - } - else - { - MPI_Comm_remote_size(InterComm, &InterSize); - } + int status = MPI_Comm_accept(DsmMasterHostName, MPI_INFO_NULL, 0, IntraComm, &InterComm); + if (status != MPI_SUCCESS) { + try { + std::string message = "Failed to accept port "; + message = message + DsmMasterHostName; + XdmfError::message(XdmfError::FATAL, message); + } + catch (XdmfError e) { + throw e; + } + } + else { + MPI_Comm_remote_size(InterComm, &InterSize); + } } int XdmfDSMCommMPI::Connect() { - if (InterComm != MPI_COMM_NULL) - { - // If the intercomm already exists, no need to connect - // If you want to reset the intercomm, set it to MPI_COMM_NULL before calling this - // using either SetInterComm or Disconnect - return MPI_SUCCESS; - } - else - { - MPI_Errhandler_set(IntraComm, MPI_ERRORS_RETURN); - int status = MPI_Comm_connect(DsmMasterHostName, MPI_INFO_NULL, 0, IntraComm, &InterComm); - MPI_Errhandler_set(IntraComm, MPI_ERRORS_ARE_FATAL); - if (status != MPI_SUCCESS) - { - try - { - std::string message = "Failed to connect to port "; - message = message + DsmMasterHostName; - XdmfError::message(XdmfError::FATAL, message); - } - catch (XdmfError e) - { - throw e; - } - } - else - { - status = MPI_Comm_remote_size(InterComm, &InterSize); - return MPI_SUCCESS; - } - } - return MPI_SUCCESS; + if (InterComm != MPI_COMM_NULL) { + // If the intercomm already exists, no need to connect + // If you want to reset the intercomm, set it to MPI_COMM_NULL before calling this + // using either SetInterComm or Disconnect + return MPI_SUCCESS; + } + else { + MPI_Errhandler_set(IntraComm, MPI_ERRORS_RETURN); + int status = MPI_Comm_connect(DsmMasterHostName, MPI_INFO_NULL, 0, IntraComm, &InterComm); + MPI_Errhandler_set(IntraComm, MPI_ERRORS_ARE_FATAL); + if (status != MPI_SUCCESS) { + try { + std::string message = "Failed to connect to port "; + message = message + DsmMasterHostName; + XdmfError::message(XdmfError::FATAL, message); + } + catch (XdmfError e) { + throw e; + } + } + else { + status = MPI_Comm_remote_size(InterComm, &InterSize); + return MPI_SUCCESS; + } + } + return MPI_SUCCESS; } void XdmfDSMCommMPI::Disconnect() { - if (InterComm != MPI_COMM_NULL) - { - int status = MPI_Comm_free(&InterComm); - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Failed to disconnect Comm"); - } - catch (XdmfError e) - { - throw e; - } - } - } - InterComm = MPI_COMM_NULL; + if (InterComm != MPI_COMM_NULL) { + int status = MPI_Comm_free(&InterComm); + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Failed to disconnect Comm"); + } + catch (XdmfError e) { + throw e; + } + } + } + InterComm = MPI_COMM_NULL; } void XdmfDSMCommMPI::DupComm(MPI_Comm comm) { - if (IntraComm != comm) - { - int status; - if (IntraComm != MPI_COMM_NULL) - { - status = MPI_Comm_free(&IntraComm); - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Failed to disconnect Comm"); - } - catch (XdmfError e) - { - throw e; - } - } - } - if (comm != MPI_COMM_NULL) - { - status = MPI_Comm_dup(comm, &IntraComm); - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Failed to duplicate Comm"); - } - catch (XdmfError e) - { - throw e; - } - } - else - { - status = MPI_Comm_size(IntraComm, &IntraSize); - status = MPI_Comm_rank(IntraComm, &Id); - } - } - } + if (IntraComm != comm) { + int status; + if (IntraComm != MPI_COMM_NULL) { + status = MPI_Comm_free(&IntraComm); + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Failed to disconnect Comm"); + } + catch (XdmfError e) { + throw e; + } + } + } + if (comm != MPI_COMM_NULL) { + status = MPI_Comm_dup(comm, &IntraComm); + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Failed to duplicate Comm"); + } + catch (XdmfError e) { + throw e; + } + } + else { + status = MPI_Comm_size(IntraComm, &IntraSize); + status = MPI_Comm_rank(IntraComm, &Id); + } + } + } } void XdmfDSMCommMPI::DupInterComm(MPI_Comm comm) { - if (InterComm != comm) - { - int status; - if (InterComm != MPI_COMM_NULL) - { - status = MPI_Comm_free(&InterComm); - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Failed to disconnect Comm"); - } - catch (XdmfError e) - { - throw e; - } - } - } - if (comm != MPI_COMM_NULL) - { - status = MPI_Comm_dup(comm, &InterComm); - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Failed to duplicate Comm"); - } - catch (XdmfError e) - { - throw e; - } - } - else - { - status = MPI_Comm_size(InterComm, &InterSize); - if (status != MPI_SUCCESS) - { - MPI_Comm_remote_size(InterComm, &InterSize); - } - } - } - else - { - InterSize = -1; - } - } + if (InterComm != comm) { + int status; + if (InterComm != MPI_COMM_NULL) { + status = MPI_Comm_free(&InterComm); + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Failed to disconnect Comm"); + } + catch (XdmfError e) { + throw e; + } + } + } + if (comm != MPI_COMM_NULL) { + status = MPI_Comm_dup(comm, &InterComm); + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Failed to duplicate Comm"); + } + catch (XdmfError e) { + throw e; + } + } + else { + status = MPI_Comm_size(InterComm, &InterSize); + if (status != MPI_SUCCESS) { + MPI_Comm_remote_size(InterComm, &InterSize); + } + } + } + else { + InterSize = -1; + } + } } MPI_Comm XdmfDSMCommMPI::GetInterComm() { - return InterComm; + return InterComm; } MPI_Comm XdmfDSMCommMPI::GetIntraComm() { - return IntraComm; + return IntraComm; } diff --git a/core/XdmfDSMManager.cpp b/core/XdmfDSMManager.cpp index 9f026c70c49ca3921725fe62380348cd96ab9734..706420fd177786a36b712c47f709ab09d70298bd 100644 --- a/core/XdmfDSMManager.cpp +++ b/core/XdmfDSMManager.cpp @@ -59,52 +59,52 @@ XdmfDSMManager::XdmfDSMManager() { - this->MpiComm = MPI_COMM_NULL; - this->UpdatePiece = 0; - this->UpdateNumPieces = 0; - this->LocalBufferSizeMBytes = 128; - - this->DsmBuffer = NULL; - this->DsmComm = NULL; - this->IsServer = true; - this->DsmType = XDMF_DSM_TYPE_UNIFORM; - this->BlockLength = XDMF_DSM_DEFAULT_BLOCK_LENGTH; - this->InterCommType = XDMF_DSM_COMM_MPI; + this->MpiComm = MPI_COMM_NULL; + this->UpdatePiece = 0; + this->UpdateNumPieces = 0; + this->LocalBufferSizeMBytes = 128; + + this->DsmBuffer = NULL; + this->DsmComm = NULL; + this->IsServer = true; + this->DsmType = XDMF_DSM_TYPE_UNIFORM; + this->BlockLength = XDMF_DSM_DEFAULT_BLOCK_LENGTH; + this->InterCommType = XDMF_DSM_COMM_MPI; } XdmfDSMManager::~XdmfDSMManager() { - this->Destroy(); + this->Destroy(); } XdmfDSMBuffer * XdmfDSMManager::GetDsmBuffer() { - return this->DsmBuffer; + return this->DsmBuffer; } void XdmfDSMManager::SetDsmBuffer(XdmfDSMBuffer * newBuffer) { - this->DsmBuffer = newBuffer; + this->DsmBuffer = newBuffer; } int XdmfDSMManager::GetUpdatePiece() { - return this->UpdatePiece; + return this->UpdatePiece; } int XdmfDSMManager::GetUpdateNumPieces() { - return this->UpdateNumPieces; + return this->UpdateNumPieces; } MPI_Comm XdmfDSMManager::GetMpiComm() { - return this->MpiComm; + return this->MpiComm; } void @@ -122,74 +122,72 @@ XdmfDSMManager::SetMpiComm(MPI_Comm comm) void XdmfDSMManager::SetLocalBufferSizeMBytes(unsigned int newSize) { - this->LocalBufferSizeMBytes = newSize; + this->LocalBufferSizeMBytes = newSize; } unsigned int XdmfDSMManager::GetLocalBufferSizeMBytes() { - return this->LocalBufferSizeMBytes; + return this->LocalBufferSizeMBytes; } void XdmfDSMManager::SetIsServer(bool newStatus) { - this->IsServer = newStatus; + this->IsServer = newStatus; } bool XdmfDSMManager::GetIsServer() { - return this->IsServer; + return this->IsServer; } void XdmfDSMManager::SetDsmType(int newType) { - this->DsmType = newType; + this->DsmType = newType; } int XdmfDSMManager::GetDsmType() { - return this->DsmType; + return this->DsmType; } void XdmfDSMManager::SetBlockLength(long newSize) { - this->BlockLength = newSize; + this->BlockLength = newSize; } long XdmfDSMManager::GetBlockLength() { - return this->BlockLength; + return this->BlockLength; } void XdmfDSMManager::SetInterCommType(int newType) { - this->InterCommType = newType; + this->InterCommType = newType; } int XdmfDSMManager::GetInterCommType() { - return this->InterCommType; + return this->InterCommType; } bool XdmfDSMManager::GetIsConnected() { - if (this->DsmBuffer) - { - return this->DsmBuffer->GetIsConnected(); - } - else - { - return false; - } + if (this->DsmBuffer) { + return this->DsmBuffer->GetIsConnected(); + } + else { + return false; + } } void @@ -200,8 +198,8 @@ XdmfDSMManager::Destroy() if (this->DsmBuffer) { delete this->DsmBuffer; this->DsmBuffer = NULL; - //Will be replaced by an Xdmf version - //H5FD_dsm_set_manager(NULL); + // Will be replaced by an Xdmf version + // H5FD_dsm_set_manager(NULL); } if (this->DsmComm) { delete this->DsmComm; @@ -270,7 +268,6 @@ XdmfDSMManager::Connect(bool persist) int status; if (!(dynamic_cast (this->DsmBuffer)->GetIsConnected())) { - do { try { status = this->DsmBuffer->GetComm()->Connect(); @@ -279,7 +276,7 @@ XdmfDSMManager::Connect(bool persist) throw e; } if (status == MPI_SUCCESS) { - dynamic_cast (this->DsmBuffer)->SetIsConnected(true); + dynamic_cast (this->DsmBuffer)->SetIsConnected(true); try { this->DsmBuffer->ReceiveInfo(); } @@ -289,9 +286,10 @@ XdmfDSMManager::Connect(bool persist) } else { #ifdef _WIN32 - Sleep(1000);//since windows has a different sleep command + Sleep(1000); + // Since windows has a different sleep command #else - sleep(1); + sleep(1); #endif } } while (persist && (status != MPI_SUCCESS)); @@ -301,16 +299,12 @@ XdmfDSMManager::Connect(bool persist) void XdmfDSMManager::Disconnect() { - //disconnecting is done manually - - try - { - this->DsmBuffer->GetComm()->Disconnect(); - } - catch (XdmfError e) - { - throw e; - } - - dynamic_cast (this->DsmBuffer)->SetIsConnected(false); + // Disconnecting is done manually + try { + this->DsmBuffer->GetComm()->Disconnect(); + } + catch (XdmfError e) { + throw e; + } + dynamic_cast (this->DsmBuffer)->SetIsConnected(false); } diff --git a/core/XdmfHDF5ControllerDSM.cpp b/core/XdmfHDF5ControllerDSM.cpp index 33696cf2fbe4ff38468c74f5b7581d06f8bc2c46..8dc07aac1033b9143712d4a4fccbd89d5baafc02 100644 --- a/core/XdmfHDF5ControllerDSM.cpp +++ b/core/XdmfHDF5ControllerDSM.cpp @@ -77,7 +77,7 @@ XdmfHDF5ControllerDSM::New(const std::string & hdf5FilePath, return p; } -//server/ nonthreaded versions +// Server/ nonthreaded versions shared_ptr XdmfHDF5ControllerDSM::New(const std::string & hdf5FilePath, const std::string & dataSetPath, @@ -267,122 +267,118 @@ XdmfHDF5ControllerDSM::XdmfHDF5ControllerDSM(const std::string & hdf5FilePath, mServerMode(true) { - //negative values will be changed to maximum range - if (startCoreIndex < 0) - { - startCoreIndex = 0; - } - if (endCoreIndex < 0) - { - endCoreIndex = mGroupSize - 1; - } - - //ensure start index is less than end index - if (startCoreIndex > endCoreIndex) - { - int tempholder = startCoreIndex; - startCoreIndex = endCoreIndex; - endCoreIndex = tempholder; - } - - mGroupComm = comm; - mStartCoreIndex = startCoreIndex; - mEndCoreIndex = endCoreIndex; - - MPI_Comm_size(comm, &mGroupSize); - MPI_Comm_rank(comm, &mRank); - - MPI_Group workers, dsmgroup, serversplit, servergroup; - - int * ServerIds = (int *)calloc((3), sizeof(int)); - unsigned int index = 0; - for(int i=mStartCoreIndex ; i <= mEndCoreIndex ; ++i) - { - ServerIds[index++] = i; - } - - MPI_Comm_group(comm, &serversplit); - MPI_Group_incl(serversplit, index, ServerIds, &servergroup); - MPI_Comm_create(comm, servergroup, &mServerComm); - MPI_Comm_group(comm, &dsmgroup); - MPI_Group_excl(dsmgroup, index, ServerIds, &workers); - MPI_Comm_create(comm, workers, &mWorkerComm); - cfree(ServerIds); - - //create the manager - - mDSMServerManager = new XdmfDSMManager(); - - mDSMServerManager->SetLocalBufferSizeMBytes(bufferSize); - mDSMServerManager->SetInterCommType(H5FD_DSM_COMM_MPI); - - if (mRank >=mStartCoreIndex && mRank <=mEndCoreIndex) - { - mDSMServerManager->SetMpiComm(mServerComm); - mDSMServerManager->Create(); - } - else - { - mDSMServerManager->SetMpiComm(mWorkerComm); - mDSMServerManager->SetIsServer(false); - mDSMServerManager->Create(mStartCoreIndex, mEndCoreIndex); - } - - XDMF_dsm_set_manager(mDSMServerManager); - - mDSMServerBuffer = mDSMServerManager->GetDsmBuffer(); - - mDSMServerBuffer->GetComm()->DupInterComm(mGroupComm); - mDSMServerBuffer->SetIsConnected(true); - - MPI_Barrier(comm); - - //loop needs to be started before anything can be done to the file, since the service is what sets up the file - - if (mRank < mStartCoreIndex || mRank > mEndCoreIndex) - { - //turn off the server designation - mDSMServerBuffer->SetIsServer(H5FD_DSM_FALSE);//if this is set to false then the buffer will attempt to connect to the intercomm for DSM stuff - mDSMServerManager->SetIsServer(H5FD_DSM_FALSE); - } - else - { - //on cores where memory is set up, start the service loop - //this should iterate infinitely until a value to end the loop is passed - H5FDdsmInt32 returnOpCode; - try - { - mDSMServerBuffer->BufferServiceLoop(&returnOpCode); - } - catch (XdmfError e) - { - throw e; - } - } + // Negative values will be changed to maximum range + if (startCoreIndex < 0) { + startCoreIndex = 0; + } + if (endCoreIndex < 0) { + endCoreIndex = mGroupSize - 1; + } + + // Ensure start index is less than end index + if (startCoreIndex > endCoreIndex) { + int tempholder = startCoreIndex; + startCoreIndex = endCoreIndex; + endCoreIndex = tempholder; + } + + mGroupComm = comm; + mStartCoreIndex = startCoreIndex; + mEndCoreIndex = endCoreIndex; + + MPI_Comm_size(comm, &mGroupSize); + MPI_Comm_rank(comm, &mRank); + + MPI_Group workers, dsmgroup, serversplit, servergroup; + + int * ServerIds = (int *)calloc((3), sizeof(int)); + unsigned int index = 0; + for(int i=mStartCoreIndex ; i <= mEndCoreIndex ; ++i) { + ServerIds[index++] = i; + } + + MPI_Comm_group(comm, &serversplit); + MPI_Group_incl(serversplit, index, ServerIds, &servergroup); + MPI_Comm_create(comm, servergroup, &mServerComm); + MPI_Comm_group(comm, &dsmgroup); + MPI_Group_excl(dsmgroup, index, ServerIds, &workers); + MPI_Comm_create(comm, workers, &mWorkerComm); + cfree(ServerIds); + + // Create the manager + + mDSMServerManager = new XdmfDSMManager(); + + mDSMServerManager->SetLocalBufferSizeMBytes(bufferSize); + mDSMServerManager->SetInterCommType(H5FD_DSM_COMM_MPI); + + if (mRank >=mStartCoreIndex && mRank <=mEndCoreIndex) { + mDSMServerManager->SetMpiComm(mServerComm); + mDSMServerManager->Create(); + } + else { + mDSMServerManager->SetMpiComm(mWorkerComm); + mDSMServerManager->SetIsServer(false); + mDSMServerManager->Create(mStartCoreIndex, mEndCoreIndex); + } + + XDMF_dsm_set_manager(mDSMServerManager); + + mDSMServerBuffer = mDSMServerManager->GetDsmBuffer(); + + mDSMServerBuffer->GetComm()->DupInterComm(mGroupComm); + mDSMServerBuffer->SetIsConnected(true); + + MPI_Barrier(comm); + + // Loop needs to be started before anything can be done to the file, since the service is what sets up the file + + if (mRank < mStartCoreIndex || mRank > mEndCoreIndex) { + // Turn off the server designation + mDSMServerBuffer->SetIsServer(H5FD_DSM_FALSE); + // If this is set to false then the buffer will attempt to connect to the intercomm for DSM stuff + mDSMServerManager->SetIsServer(H5FD_DSM_FALSE); + } + else { + // On cores where memory is set up, start the service loop + // This should iterate infinitely until a value to end the loop is passed + H5FDdsmInt32 returnOpCode; + try { + mDSMServerBuffer->BufferServiceLoop(&returnOpCode); + } + catch (XdmfError e) { + throw e; + } + } } XdmfHDF5ControllerDSM::~XdmfHDF5ControllerDSM() { } -std::string XdmfHDF5ControllerDSM::getName() const +void XdmfHDF5ControllerDSM::deleteManager() { - return "HDFDSM"; + if (mDSMManager != NULL) { + delete mDSMManager; + } + if (mDSMServerManager != NULL) { + delete mDSMServerManager; + } } -H5FDdsmManager * XdmfHDF5ControllerDSM::getManager() +std::string XdmfHDF5ControllerDSM::getName() const { - return mDSMManager; + return "HDFDSM"; } H5FDdsmBuffer * XdmfHDF5ControllerDSM::getBuffer() { - return mDSMBuffer; + return mDSMBuffer; } -XdmfDSMManager * XdmfHDF5ControllerDSM::getServerManager() +H5FDdsmManager * XdmfHDF5ControllerDSM::getManager() { - return mDSMServerManager; + return mDSMManager; } XdmfDSMBuffer * XdmfHDF5ControllerDSM::getServerBuffer() @@ -390,23 +386,28 @@ XdmfDSMBuffer * XdmfHDF5ControllerDSM::getServerBuffer() return mDSMServerBuffer; } -bool XdmfHDF5ControllerDSM::getServerMode() +MPI_Comm XdmfHDF5ControllerDSM::getServerComm() { - return mServerMode; + MPI_Comm returnComm = MPI_COMM_NULL; + int status = MPI_Comm_dup(mServerComm, &returnComm); + return returnComm; } -MPI_Comm XdmfHDF5ControllerDSM::getServerComm() +XdmfDSMManager * XdmfHDF5ControllerDSM::getServerManager() { - MPI_Comm returnComm = MPI_COMM_NULL; - int status = MPI_Comm_dup(mServerComm, &returnComm); - return returnComm; + return mDSMServerManager; +} + +bool XdmfHDF5ControllerDSM::getServerMode() +{ + return mServerMode; } MPI_Comm XdmfHDF5ControllerDSM::getWorkerComm() { - MPI_Comm returnComm = MPI_COMM_NULL; - int status = MPI_Comm_dup(mWorkerComm, &returnComm); - return returnComm; + MPI_Comm returnComm = MPI_COMM_NULL; + int status = MPI_Comm_dup(mWorkerComm, &returnComm); + return returnComm; } void XdmfHDF5ControllerDSM::setManager(XdmfDSMManager * newManager) @@ -416,11 +417,6 @@ void XdmfHDF5ControllerDSM::setManager(XdmfDSMManager * newManager) mDSMServerBuffer = newBuffer; } -void XdmfHDF5ControllerDSM::setBuffer(XdmfDSMBuffer * newBuffer) -{ - mDSMServerBuffer = newBuffer; -} - void XdmfHDF5ControllerDSM::setManager(H5FDdsmManager * newManager) { H5FDdsmBuffer * newBuffer = newManager->GetDsmBuffer(); @@ -428,132 +424,101 @@ void XdmfHDF5ControllerDSM::setManager(H5FDdsmManager * newManager) mDSMBuffer = newBuffer; } -void XdmfHDF5ControllerDSM::setBuffer(H5FDdsmBuffer * newBuffer) +void XdmfHDF5ControllerDSM::setBuffer(XdmfDSMBuffer * newBuffer) { - mDSMBuffer = newBuffer; + mDSMServerBuffer = newBuffer; } -void XdmfHDF5ControllerDSM::setServerMode(bool newMode) +void XdmfHDF5ControllerDSM::setBuffer(H5FDdsmBuffer * newBuffer) { - mServerMode = newMode; + mDSMBuffer = newBuffer; } void XdmfHDF5ControllerDSM::setServerComm(MPI_Comm comm) { - int status; - if (mServerComm != MPI_COMM_NULL) - { - status = MPI_Comm_free(&mServerComm); - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Failed to disconnect Comm"); - } - catch (XdmfError e) - { - throw e; - } - } - } - if (comm != MPI_COMM_NULL) - { - status = MPI_Comm_dup(comm, &mServerComm); - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Failed to duplicate Comm"); - } - catch (XdmfError e) - { - throw e; - } - } - } - mDSMServerBuffer->GetComm()->DupComm(comm); + int status; + if (mServerComm != MPI_COMM_NULL) { + status = MPI_Comm_free(&mServerComm); + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Failed to disconnect Comm"); + } + catch (XdmfError e) { + throw e; + } + } + } + if (comm != MPI_COMM_NULL) { + status = MPI_Comm_dup(comm, &mServerComm); + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Failed to duplicate Comm"); + } + catch (XdmfError e) { + throw e; + } + } + } + mDSMServerBuffer->GetComm()->DupComm(comm); } -void XdmfHDF5ControllerDSM::setWorkerComm(MPI_Comm comm) +void XdmfHDF5ControllerDSM::setServerMode(bool newMode) { - int status; - if (mWorkerComm != MPI_COMM_NULL) - { - status = MPI_Comm_free(&mWorkerComm); - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Failed to disconnect Comm"); - } - catch (XdmfError e) - { - throw e; - } - } - } - if (comm != MPI_COMM_NULL) - { - status = MPI_Comm_dup(comm, &mWorkerComm); - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Failed to duplicate Comm"); - } - catch (XdmfError e) - { - throw e; - } - } - } - mDSMServerBuffer->GetComm()->DupComm(comm); + mServerMode = newMode; } -void XdmfHDF5ControllerDSM::stopDSM() +void XdmfHDF5ControllerDSM::setWorkerComm(MPI_Comm comm) { - //send manually - for (int i = mStartCoreIndex; i <= mEndCoreIndex; ++i) - { - try - { - mDSMServerBuffer->SendCommandHeader(H5FD_DSM_OPCODE_DONE, i, 0, 0, H5FD_DSM_INTER_COMM); - } - catch (XdmfError e) - { - throw e; - } - //originally this was set to the intra comm - //that doesn't work in this instance because it won't reach the server cores - } + int status; + if (mWorkerComm != MPI_COMM_NULL) { + status = MPI_Comm_free(&mWorkerComm); + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Failed to disconnect Comm"); + } + catch (XdmfError e) { + throw e; + } + } + } + if (comm != MPI_COMM_NULL) { + status = MPI_Comm_dup(comm, &mWorkerComm); + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Failed to duplicate Comm"); + } + catch (XdmfError e) { + throw e; + } + } + } + mDSMServerBuffer->GetComm()->DupComm(comm); } -void XdmfHDF5ControllerDSM::restartDSM() +void XdmfHDF5ControllerDSM::stopDSM() { - if (mRank >= mStartCoreIndex && mRank <= mEndCoreIndex) - { - H5FDdsmInt32 returnOpCode; - try - { - mDSMServerBuffer->BufferServiceLoop(&returnOpCode); - } - catch (XdmfError e) - { - throw e; - } - } + // Send manually + for (int i = mStartCoreIndex; i <= mEndCoreIndex; ++i) { + try { + mDSMServerBuffer->SendCommandHeader(H5FD_DSM_OPCODE_DONE, i, 0, 0, H5FD_DSM_INTER_COMM); + } + catch (XdmfError e) { + throw e; + } + } } -void XdmfHDF5ControllerDSM::deleteManager() +void XdmfHDF5ControllerDSM::restartDSM() { - if (mDSMManager != NULL) - { - delete mDSMManager; - } - if (mDSMServerManager != NULL) - { - delete mDSMServerManager; - } + if (mRank >= mStartCoreIndex && mRank <= mEndCoreIndex) { + H5FDdsmInt32 returnOpCode; + try { + mDSMServerBuffer->BufferServiceLoop(&returnOpCode); + } + catch (XdmfError e) { + throw e; + } + } } void XdmfHDF5ControllerDSM::read(XdmfArray * const array) diff --git a/core/XdmfHDF5Writer.cpp b/core/XdmfHDF5Writer.cpp index 38594b8e33ad0e12adde87c4e6dfdd164e9a9cb2..e1ce786009c54b6b12436bd06aa77e06798b7542 100644 --- a/core/XdmfHDF5Writer.cpp +++ b/core/XdmfHDF5Writer.cpp @@ -108,7 +108,7 @@ public: } } else { - //this is where it currently fails + // This is where it currently fails mHDF5Handle = H5Fcreate(filePath.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, @@ -254,7 +254,7 @@ XdmfHDF5Writer::visit(XdmfArray & array, mImpl->mDepth++; std::set::iterator checkWritten = mImpl->mWrittenItems.find(&array); if (checkWritten == mImpl->mWrittenItems.end() || array.getItemTag() == "DataItem") { - //if it has children send the writer to them too. + // If it has children send the writer to them too. try { array.traverse(visitor); } @@ -262,7 +262,7 @@ XdmfHDF5Writer::visit(XdmfArray & array, throw e; } if (array.isInitialized()) { - //only do this if the object has not already been written + // Only do this if the object has not already been written try { this->write(array, H5P_DEFAULT); } @@ -272,7 +272,7 @@ XdmfHDF5Writer::visit(XdmfArray & array, mImpl->mWrittenItems.insert(&array); } } - //if the object has already been written, just end, it already has the data + // If the object has already been written, just end, it already has the data mImpl->mDepth--; if(mImpl->mDepth <= 0) { mImpl->mWrittenItems.clear(); @@ -284,9 +284,9 @@ XdmfHDF5Writer::visit(XdmfItem & item, const shared_ptr visitor) { mImpl->mDepth++; - //This is similar to the algorithm for writing XPaths - //shouldn't be a problem if XPaths are turned off because all this does is avoid writing an object twice - //if it was written once then all instances of the object should have the controller + // This is similar to the algorithm for writing XPaths + // shouldn't be a problem if XPaths are turned off because all this does is avoid writing an object twice + // if it was written once then all instances of the object should have the controller std::set::iterator checkWritten = mImpl->mWrittenItems.find(&item); if (checkWritten == mImpl->mWrittenItems.end()) { mImpl->mWrittenItems.insert(&item); @@ -311,9 +311,10 @@ XdmfHDF5Writer::write(XdmfArray & array, hid_t datatype = -1; bool closeDatatype = false; - const unsigned int hdf5Overhead = 800;//base size of an hdf5 file is 800 + // Base size of an hdf5 file is 800 + const unsigned int hdf5Overhead = 800; - //determining data type + // Determining data type if(array.isInitialized()) { if(array.getArrayType() == XdmfArrayType::Int8()) { datatype = H5T_NATIVE_CHAR; @@ -342,7 +343,8 @@ XdmfHDF5Writer::write(XdmfArray & array, else if(array.getArrayType() == XdmfArrayType::UInt32()) { datatype = H5T_NATIVE_UINT; } - else if(array.getArrayType() == XdmfArrayType::String()) {//strings are a special case as they have mutable size + else if(array.getArrayType() == XdmfArrayType::String()) { + // Strings are a special case as they have mutable size datatype = H5Tcopy(H5T_C_S1); H5Tset_size(datatype, H5T_VARIABLE); closeDatatype = true; @@ -381,12 +383,12 @@ XdmfHDF5Writer::write(XdmfArray & array, std::vector > previousControllers; - //hold the controllers in order to base the new controllers on them + // Hold the controllers in order to base the new controllers on them for(unsigned int i = 0; i < array.getNumberHeavyDataControllers(); ++i) { previousControllers.push_back(array.getHeavyDataController(i)); } - //remove controllers from the array, they will be replaced by the controllers created by this function. + // Remove controllers from the array, they will be replaced by the controllers created by this function. while(array.getNumberHeavyDataControllers() != 0) { array.removeHeavyDataController(array.getNumberHeavyDataControllers() -1); } @@ -394,7 +396,7 @@ XdmfHDF5Writer::write(XdmfArray & array, if (previousControllers.size() == 0) { - //create a temporary controller if the array doesn't have one + // Create a temporary controller if the array doesn't have one try { shared_ptr tempDataController = this->createHDF5Controller(hdf5FilePath, @@ -413,9 +415,11 @@ XdmfHDF5Writer::write(XdmfArray & array, int controllerIndexOffset = 0; - for(unsigned int i = 0; i < previousControllers.size(); ++i)// it is assumed that the array will have at least one controller, if it didn't have one a temporary one was generated + // It is assumed that the array will have at least one controller, if it didn't have one a temporary one was generated + for(unsigned int i = 0; i < previousControllers.size(); ++i) { - if (mMode == Append) {//append only cares about the last controller, so add the rest back in + if (mMode == Append) { + // Append only cares about the last controller, so add the rest back in for (; i < previousControllers.size() - 1; ++i) { array.insert(previousControllers[i]); } @@ -441,13 +445,13 @@ XdmfHDF5Writer::write(XdmfArray & array, bool startedloop = false; unsigned int origFileIndex = mImpl->mFileIndex; while ((mMode == Hyperslab && i < previousControllers.size()) || !startedloop) { - //Hyperslab mode wants to assign all data using the current location without writing until all data sets are determined + // Hyperslab mode wants to assign all data using the current location without writing until all data sets are determined startedloop = true; shared_ptr heavyDataController = previousControllers[i]; - //stats for the data currently stored in the array + // Stats for the data currently stored in the array std::vector dimensions; if (mMode != Hyperslab) { @@ -467,7 +471,8 @@ XdmfHDF5Writer::write(XdmfArray & array, dataSetPath.str(std::string()); dataSetPath << heavyDataController->getDataSetPath(); hdf5FilePath = heavyDataController->getFilePath(); - if(mMode == Hyperslab) {//start, stride, and dataspace dimensions only matter for hyperslab mode + if(mMode == Hyperslab) { + // Start, stride, and dataspace dimensions only matter for hyperslab mode dataspaceDimensions = heavyDataController->getDataspaceDimensions(); start = heavyDataController->getStart(); stride = heavyDataController->getStride(); @@ -478,23 +483,24 @@ XdmfHDF5Writer::write(XdmfArray & array, dataSetPath << "Data" << mDataSetId; } - //check here for if the file would become larger than the limit after the addition. - //then check subsequent files for the same limitation + // Check here for if the file would become larger than the limit after the addition. + // Then check subsequent files for the same limitation - //this is the file splitting algorithm - if (mImpl->mHDF5FileSizeLimit > 0) {//only if the file limit is positive, disabled if 0 or negative + // This is the file splitting algorithm + if (mImpl->mHDF5FileSizeLimit > 0) { + // Only if the file limit is positive, disabled if 0 or negative unsigned int previousDataSize = 0; std::vector previousDimensions; std::vector previousDataSizes; unsigned int amountAlreadyWritten = 0; - //Even though theoretically this could be an infinite loop - //if all possible files with the specified name are produced - //the chances of that happening are small. - //It can handle up to 65535 different files. - //This value may vary depending on the compiler and platform. - //The variable UINT_MAX holds the value in question. - //If all files are take up it will loop until a file opens up since adding past the max causes overflow. + // Even though theoretically this could be an infinite loop + // if all possible files with the specified name are produced + // the chances of that happening are small. + // It can handle up to 65535 different files. + // This value may vary depending on the compiler and platform. + // The variable UINT_MAX holds the value in question. + // If all files are take up it will loop until a file opens up since adding past the max causes overflow. unsigned int containedInController = 1; @@ -511,25 +517,27 @@ XdmfHDF5Writer::write(XdmfArray & array, std::vector partialDataSizes; std::stringstream testFile; - if (mImpl->mFileIndex == 0) {//if sequentially named files need to be created or referenced + if (mImpl->mFileIndex == 0) { + // If sequentially named files need to be created or referenced testFile << checkFileName << "." << checkFileExt; } else { testFile << checkFileName << mImpl->mFileIndex << "." << checkFileExt; } FILE *checkFile = NULL; - unsigned int fileSize = 0;//if the file doesn't exist the size is 0 because there's no data - // get the file stream + unsigned int fileSize = 0; + // If the file doesn't exist the size is 0 because there's no data + // Get the file stream checkFile = fopen(testFile.str().c_str(), "a"); if (checkFile != NULL) { - // set the file pointer to end of file + // Set the file pointer to end of file fseek(checkFile, 0, SEEK_END); - // get the file size, in bytes + // Get the file size, in bytes fileSize = ftell(checkFile); - //if overwrite subtract previous data size. + // If overwrite subtract previous data size. if (mMode == Overwrite || mMode == Hyperslab) { - //find previous data size + // Find previous data size mImpl->openFile(testFile.str(), fapl, mDataSetId); hid_t checkset = H5Dopen(mImpl->mHDF5Handle, @@ -547,7 +555,8 @@ XdmfHDF5Writer::write(XdmfArray & array, fileSize = 0; } else { - fileSize = fileSize - checksize;//remove previous set's size, since it's overwritten + fileSize = fileSize - checksize; + // Remove previous set's size, since it's overwritten } if (fileSize == 0) { fileSize += hdf5Overhead; @@ -565,12 +574,12 @@ XdmfHDF5Writer::write(XdmfArray & array, else if (previousDataSize == 0) { fileSize += hdf5Overhead; } - // close stream and release buffer - //check size to see if it's within range + // Check size to see if it's within range - if (closeDatatype == true) { //closetype is only true if strings are being used, it's set at the beginning when types are checked. - //size needed is equal to the dataspaceDimensions if in hyperslab mode - //otherwise is equal to the size of the written array + if (closeDatatype == true) { + // closeDatatype is only true if strings are being used, it's set at the beginning when types are checked. + // Size needed is equal to the dataspaceDimensions if in hyperslab mode + // Otherwise is equal to the size of the written array unsigned int remainingValues = 0; unsigned int sizeArrayIndex = 0; if (mMode == Hyperslab) { @@ -585,34 +594,38 @@ XdmfHDF5Writer::write(XdmfArray & array, remainingValues += array.getSize(); sizeArrayIndex = amountAlreadyWritten; } - remainingValues -= amountAlreadyWritten;//reduce by number of values already written - if (remainingValues == 0) {//end if no remaining values + remainingValues -= amountAlreadyWritten; + // Reduce by number of values already written + if (remainingValues == 0) { + // End if no remaining values break; } - //if remaining size is less than available space, just write all of what's left - //calculate remaining size + // If remaining size is less than available space, just write all of what's left + // Calculate remaining size unsigned int remainingSize = 0; for (unsigned int j = sizeArrayIndex; j < array.getSize(); ++j) { remainingSize += array.getValue(j).size() * 8; } if (mMode == Hyperslab) { - //size is estimated based on averages + // Size is estimated based on averages remainingSize = (remainingSize / (array.getSize() - sizeArrayIndex)) * remainingValues; } if (remainingSize + previousDataSize + fileSize < mImpl->mHDF5FileSizeLimit*(1024*1024)) { - //if the array hasn't been split + // If the array hasn't been split if (amountAlreadyWritten == 0) { - //just pass all data to the partial vectors - for (unsigned int j = 0; j < dimensions.size(); ++j) {//done using a loop so that data is copied, not referenced + // Just pass all data to the partial vectors + for (unsigned int j = 0; j < dimensions.size(); ++j) { + // Done using a loop so that data is copied, not referenced partialStarts.push_back(start[j]); partialStrides.push_back(stride[j]); partialDimensions.push_back(dimensions[j]); partialDataSizes.push_back(dataspaceDimensions[j]); } } - else {//if the array has been split + else { + // If the array has been split int dimensionIndex = previousDimensions.size() - 1; - //loop previous dimensions in + // Loop previous dimensions in int j = 0; for (j = 0; j < dimensionIndex; ++j) { partialStarts.push_back(start[j]); @@ -626,9 +639,9 @@ XdmfHDF5Writer::write(XdmfArray & array, newStart += stride[j]; } partialStarts.push_back(newStart); - //stride should not change in this algorithm + // Stride should not change in this algorithm partialStrides.push_back(stride[j]); - //total up number of blocks for the higher dimesions and subtract the amount already written + // Total up number of blocks for the higher dimesions and subtract the amount already written unsigned int dimensiontotal = dimensions[j]; unsigned int dataspacetotal = dataspaceDimensions[j]; for (unsigned int k = j + 1; k < dimensions.size(); ++k) { @@ -649,11 +662,11 @@ XdmfHDF5Writer::write(XdmfArray & array, } } else { - //start and stride are not used outside of hyperslab + // Start and stride are not used outside of hyperslab partialStarts.push_back(start[j]); partialStrides.push_back(stride[j]); - //total up number of blocks for the higher dimesions and subtract the amount already written - //since it isn't hyperslab dimensions and dataspacedimensions should be the same + // Total up number of blocks for the higher dimesions and subtract the amount already written + // Since it isn't hyperslab dimensions and dataspacedimensions should be the same unsigned int dimensiontotal = dimensions[j]; for (unsigned int k = j + 1; k < dimensions.size(); ++k) { dimensiontotal *= dimensions[k]; @@ -673,28 +686,29 @@ XdmfHDF5Writer::write(XdmfArray & array, } } } - else {//otherwise, take remaining size and start removing dimensions until the dimension block is less, then take a fraction of the dimension - //calculate the number of values of the data type you're using will fit + else { + // Otherwise, take remaining size and start removing dimensions until the dimension block is less, then take a fraction of the dimension + // Calculate the number of values of the data type you're using will fit unsigned int usableSpace = (mImpl->mHDF5FileSizeLimit*(1024*1024) - fileSize); if (previousDataSize + fileSize > mImpl->mHDF5FileSizeLimit*(1024*1024)) { usableSpace = 0; } usableSpace += hyperslabSize-previousDataSize; - //if the array hasn't been split + // If the array hasn't been split if (amountAlreadyWritten == 0) { - //see if it will fit in the next file - //if it will just go to the next file - //otherwise split it. + // See if it will fit in the next file + // If it will just go to the next file + // Otherwise split it. if (remainingSize + hdf5Overhead > mImpl->mHDF5FileSizeLimit*(1024*1024) && usableSpace > 0) { if (mImpl->mAllowSplitDataSets) { - //figure out the size of the largest block that will fit. + // Figure out the size of the largest block that will fit. unsigned int blockSizeSubtotal = 0; unsigned int dimensionSizeTotal = 1; unsigned int dimensionIndex = 0; unsigned int previousBlockSize = 0; - //find the dimension that was split + // Find the dimension that was split while (dimensionIndex < dataspaceDimensions.size() && blockSizeSubtotal <= usableSpace) { - //this is totally different for strings + // This is totally different for strings dimensionSizeTotal *= dimensions[dimensionIndex]; previousBlockSize = blockSizeSubtotal; blockSizeSubtotal = 0; @@ -711,13 +725,15 @@ XdmfHDF5Writer::write(XdmfArray & array, blockSizeSubtotal += array.getValue(amountAlreadyWritten + k).size(); } dimensionIndex++; - }//It should end on the "blockSizeSubtotal <= usableSpace" statement, the other half is for backup - //move back one dimension so we're working on the dimension that was split, not the one after it + } + // It should end on the "blockSizeSubtotal <= usableSpace" statement, the other half is for backup + // move back one dimension so we're working on the dimension that was split, not the one after it dimensionIndex--; blockSizeSubtotal = previousBlockSize; - //determine how many of those blocks will fit - unsigned int numBlocks = usableSpace / blockSizeSubtotal;//this should be less than the current value for the dimension - //add dimensions as required. + // Determine how many of those blocks will fit + unsigned int numBlocks = usableSpace / blockSizeSubtotal; + // This should be less than the current value for the dimension + // Add dimensions as required unsigned int j = 0; for (; j < dimensionIndex; ++j) { partialStarts.push_back(start[j]); @@ -736,8 +752,9 @@ XdmfHDF5Writer::write(XdmfArray & array, if (dimensions[j] == dataspaceDimensions[j]) {//this is for non-hyperslab and specific cases of hyperslab partialDimensions.push_back(numBlocks); } - else {//for hyperslab in general - //determine how many values from the array will fit into the blocks being used with the dimensions specified + else { + // For hyperslab in general + // Determine how many values from the array will fit into the blocks being used with the dimensions specified unsigned int displacement = numBlocks / stride[j]; if (((int)displacement * (int)stride[j]) + (start[j] % stride[j]) < numBlocks) { displacement++; @@ -746,17 +763,20 @@ XdmfHDF5Writer::write(XdmfArray & array, if (start[j] > numBlocks) { displacement = 0; } - if (dimensions[j] <= displacement) {//if there are less values than there are space for, just write all of them. + if (dimensions[j] <= displacement) { + // If there are less values than there are space for, just write all of them. partialDimensions.push_back(dimensions[j]); } - else {//otherwise write what space allows for + else { + // Otherwise write what space allows for partialDimensions.push_back(displacement); } } } else { - //just pass all data to the partial vectors - for (unsigned int j = 0; j < dimensions.size(); ++j) {//done using a loop so that data is copied, not referenced + // Just pass all data to the partial vectors + for (unsigned int j = 0; j < dimensions.size(); ++j) { + // Done using a loop so that data is copied, not referenced partialStarts.push_back(start[j]); partialStrides.push_back(stride[j]); partialDimensions.push_back(dimensions[j]); @@ -765,20 +785,21 @@ XdmfHDF5Writer::write(XdmfArray & array, } } } - else {//if the array has been split - //This case should not come up often as it requires truly gigantic data sets - //see if the remaining data will fit in the next file - //if yes, skip to it - //if no, split + else { + // If the array has been split + // This case should not come up often as it requires truly gigantic data sets + // See if the remaining data will fit in the next file + // If yes, skip to it + // If no, split if (remainingSize + hdf5Overhead > mImpl->mHDF5FileSizeLimit*(1024*1024) && usableSpace > 0) { - //figure out the size of the largest block that will fit. + // Figure out the size of the largest block that will fit. unsigned int blockSizeSubtotal = 0; unsigned int tempTotal = 0; unsigned int dimensionSizeTotal = 1; unsigned int dimensionIndex = 0; - //find the dimension that was split + // Find the dimension that was split while (dimensionIndex < dataspaceDimensions.size() && blockSizeSubtotal <= usableSpace) { - //this is totally different for strings + // This is totally different for strings dimensionSizeTotal *= dimensions[dimensionIndex]; tempTotal = blockSizeSubtotal; blockSizeSubtotal = 0; @@ -795,8 +816,9 @@ XdmfHDF5Writer::write(XdmfArray & array, blockSizeSubtotal += array.getValue(amountAlreadyWritten + k).size(); } dimensionIndex++; - }//It should end on the "blockSizeSubtotal <= usableSpace" statement, the other half is for backup - //move back one dimension so we're working on the dimension that was split, not the one after it + } + // It should end on the "blockSizeSubtotal <= usableSpace" statement, the other half is for backup + // Move back one dimension so we're working on the dimension that was split, not the one after it dimensionIndex--; blockSizeSubtotal = tempTotal; unsigned int j = 0; @@ -806,23 +828,23 @@ XdmfHDF5Writer::write(XdmfArray & array, partialDimensions.push_back(dimensions[j]); partialDataSizes.push_back(dataspaceDimensions[j]); } - //continue if the block is smaller than the available size + // Continue if the block is smaller than the available size if (blockSizeSubtotal <=usableSpace) { - //find number of blocks that will fit - //this should be less than the current value for the dimension + // Find number of blocks that will fit + // This should be less than the current value for the dimension unsigned int numBlocks = usableSpace / blockSizeSubtotal; - //add dimensions to the partial vectors + // Add dimensions to the partial vectors if (mMode == Hyperslab) { int newStart = (start[j] + stride[j] * previousDimensions[j]) - previousDataSizes[j]; while (newStart < 0) { newStart += stride[j]; } partialStarts.push_back(newStart); - //stride should not change in this algorithm + // Stride should not change in this algorithm partialStrides.push_back(stride[j]); partialDataSizes.push_back(numBlocks); - //determine how many values from the array will fit into the blocks being used - //with the dimensions specified + // Determine how many values from the array will fit into the blocks being used + // with the dimensions specified unsigned int displacement = (numBlocks - newStart) / stride[j]; if (((int)displacement * (int)stride[j]) + (newStart % stride[j]) < numBlocks) { displacement++; @@ -831,27 +853,31 @@ XdmfHDF5Writer::write(XdmfArray & array, if (newStart > (int)numBlocks) { displacement = 0; } - if ((dimensions[j] - previousDimensions[j]) <= displacement) {//if there are less values than there are space for, just write all of them. + if ((dimensions[j] - previousDimensions[j]) <= displacement) { + // If there are less values than there are space for, just write all of them. partialDimensions.push_back(dimensions[j] - previousDimensions[j]); } - else {//otherwise write what space allows for + else { + // Otherwise write what space allows for partialDimensions.push_back(displacement); } } else { - //start and stride are only specified in hyperslab + // Start and stride are only specified in hyperslab partialStarts.push_back(start[j]); partialStrides.push_back(stride[j]); partialDataSizes.push_back(numBlocks); partialDimensions.push_back(numBlocks); } - //place dimensions into previous dimensions for later iterations + // Place dimensions into previous dimensions for later iterations } - else {//if this is larger than usable space, try the next file - //if moving to next file, just do nothing and pass out of the if statement - //but also check if specified file size is too small - if (mImpl->mHDF5FileSizeLimit*(1024*1024) < blockSizeSubtotal) {//this shouldn't ever trigger, but it's good to cover ourselves - //and throw an error if the block size won't work + else { + // If this is larger than usable space, try the next file + // If moving to next file, just do nothing and pass out of the if statement + // but also check if specified file size is too small + if (mImpl->mHDF5FileSizeLimit*(1024*1024) < blockSizeSubtotal) { + // This shouldn't ever trigger, but it's good to cover ourselves + // and throw an error if the block size won't work try { XdmfError::message(XdmfError::FATAL, "Error: Dimension Block size / Maximum File size mismatch.\n"); @@ -863,15 +889,15 @@ XdmfHDF5Writer::write(XdmfArray & array, } } } - //move to next file + // Move to next file mImpl->mFileIndex++; } } else { - //if needed split the written array into smaller arrays based on dimension blocks - //working with strings has a more resource intensive version of this algorithm - //size needed is equal to the dataspaceDimensions if in hyperslab mode - //otherwise is equal to the size of the written array + // If needed split the written array into smaller arrays based on dimension blocks + // Working with strings has a more resource intensive version of this algorithm + // Size needed is equal to the dataspaceDimensions if in hyperslab mode + // otherwise is equal to the size of the written array unsigned int remainingValues = 0; if (mMode == Hyperslab) { remainingValues += 1; @@ -885,26 +911,29 @@ XdmfHDF5Writer::write(XdmfArray & array, remainingValues *= dimensions[j]; } } - remainingValues -= amountAlreadyWritten;//reduce by number of values already written + remainingValues -= amountAlreadyWritten; + // Reduce by number of values already written if (remainingValues == 0) {//end if no remaining values break; } unsigned int dataItemSize = array.getArrayType()->getElementSize(); - //if remaining size is less than available space, just write all of what's left + // If remaining size is less than available space, just write all of what's left if ((remainingValues * dataItemSize) + previousDataSize + fileSize < mImpl->mHDF5FileSizeLimit*(1024*1024)) { - //if the array hasn't been split + // If the array hasn't been split if (amountAlreadyWritten == 0) { - //just pass all data to the partial vectors - for (unsigned int j = 0; j < dimensions.size(); ++j) {//done using a loop so that data is copied, not referenced + // Just pass all data to the partial vectors + for (unsigned int j = 0; j < dimensions.size(); ++j) { + // Done using a loop so that data is copied, not referenced partialStarts.push_back(start[j]); partialStrides.push_back(stride[j]); partialDimensions.push_back(dimensions[j]); partialDataSizes.push_back(dataspaceDimensions[j]); } } - else {//if the array has been split + else { + // If the array has been split int dimensionIndex = previousDimensions.size() - 1; - //loop previous dimensions in + // Loop previous dimensions in int j = 0; for (j = 0; j < dimensionIndex; ++j) { partialStarts.push_back(start[j]); @@ -918,9 +947,9 @@ XdmfHDF5Writer::write(XdmfArray & array, newStart += stride[j]; } partialStarts.push_back(newStart); - //stride should not change in this algorithm + // Stride should not change in this algorithm partialStrides.push_back(stride[j]); - //total up number of blocks for the higher dimesions and subtract the amount already written + // Total up number of blocks for the higher dimesions and subtract the amount already written unsigned int dimensiontotal = dimensions[j]; unsigned int dataspacetotal = dataspaceDimensions[j]; for (unsigned int k = j + 1; k < dimensions.size(); ++k) { @@ -941,11 +970,11 @@ XdmfHDF5Writer::write(XdmfArray & array, } } else { - //start and stride are not used outside of hyperslab + // Start and stride are not used outside of hyperslab partialStarts.push_back(start[j]); partialStrides.push_back(stride[j]); - //total up number of blocks for the higher dimesions and subtract the amount already written - //since it isn't hyperslab dimensions and dataspacedimensions should be the same + // Total up number of blocks for the higher dimesions and subtract the amount already written + // since it isn't hyperslab dimensions and dataspacedimensions should be the same unsigned int dimensiontotal = dimensions[j]; for (unsigned int k = j + 1; k < dimensions.size(); ++k) { dimensiontotal *= dimensions[k]; @@ -965,34 +994,37 @@ XdmfHDF5Writer::write(XdmfArray & array, } } } - else {//otherwise, take remaining size and start removing dimensions until the dimension block is less, then take a fraction of the dimension - //calculate the number of values of the data type you're using will fit + else { + // Otherwise, take remaining size and start removing dimensions until the dimension block is less, then take a fraction of the dimension + // Calculate the number of values of the data type you're using will fit unsigned int usableSpace = (mImpl->mHDF5FileSizeLimit*(1024*1024) - fileSize) / dataItemSize; if (mImpl->mHDF5FileSizeLimit*(1024*1024) < fileSize) { usableSpace = 0; } usableSpace += hyperslabSize-previousDataSize; - //if the array hasn't been split + // If the array hasn't been split if (amountAlreadyWritten == 0) { - //see if it will fit in the next file - //if it will just go to the next file - //otherwise split it. + // See if it will fit in the next file + // If it will just go to the next file + // Otherwise split it. if ((remainingValues * dataItemSize) + hdf5Overhead > mImpl->mHDF5FileSizeLimit*(1024*1024) && usableSpace > 0) { if (mImpl->mAllowSplitDataSets) { - //figure out the size of the largest block that will fit. + // Figure out the size of the largest block that will fit. unsigned int blockSizeSubtotal = 1; unsigned int dimensionIndex = 0; - //find the dimension that was split + // Find the dimension that was split while (dimensionIndex < dataspaceDimensions.size() && blockSizeSubtotal <= usableSpace) { blockSizeSubtotal *= dataspaceDimensions[dimensionIndex]; dimensionIndex++; - }//It should end on the "blockSizeSubtotal <= arrayStartIndex" statement, the other half is for backup - //move back one dimension so we're working on the dimension that was split, not the one after it + } + // It should end on the "blockSizeSubtotal <= arrayStartIndex" statement, the other half is for backup + // Move back one dimension so we're working on the dimension that was split, not the one after it dimensionIndex--; blockSizeSubtotal /= dataspaceDimensions[dimensionIndex]; - //determine how many of those blocks will fit - unsigned int numBlocks = usableSpace / blockSizeSubtotal;//this should be less than the current value for the dimension - //add dimensions as required. + // Determine how many of those blocks will fit + unsigned int numBlocks = usableSpace / blockSizeSubtotal; + // This should be less than the current value for the dimension + // Add dimensions as required. unsigned int j = 0; for (j = 0; j < dimensionIndex; ++j) { partialStarts.push_back(start[j]); @@ -1008,11 +1040,13 @@ XdmfHDF5Writer::write(XdmfArray & array, } partialStrides.push_back(stride[j]); partialDataSizes.push_back(numBlocks); - if (dimensions[j] == dataspaceDimensions[j]) {//this is for non-hyperslab and specific cases of hyperslab + if (dimensions[j] == dataspaceDimensions[j]) { + // This is for non-hyperslab and specific cases of hyperslab partialDimensions.push_back(numBlocks); } - else {//for hyperslab in general - //determine how many values from the array will fit into the blocks being used with the dimensions specified + else { + // For hyperslab in general + // Determine how many values from the array will fit into the blocks being used with the dimensions specified unsigned int displacement = numBlocks / stride[j]; if (((int)displacement * (int)stride[j]) + (start[j] % stride[j]) < numBlocks) { displacement++; @@ -1021,17 +1055,20 @@ XdmfHDF5Writer::write(XdmfArray & array, if (start[j] > numBlocks) { displacement = 0; } - if (dimensions[j] <= displacement) {//if there are less values than there are space for, just write all of them. + if (dimensions[j] <= displacement) { + // If there are less values than there are space for, just write all of them. partialDimensions.push_back(dimensions[j]); } - else {//otherwise write what space allows for + else { + // Otherwise write what space allows for partialDimensions.push_back(displacement); } } } else { - //just pass all data to the partial vectors - for (unsigned int j = 0; j < dimensions.size(); ++j) {//done using a loop so that data is copied, not referenced + // Just pass all data to the partial vectors + for (unsigned int j = 0; j < dimensions.size(); ++j) { + // Done using a loop so that data is copied, not referenced partialStarts.push_back(start[j]); partialStrides.push_back(stride[j]); partialDimensions.push_back(dimensions[j]); @@ -1040,20 +1077,22 @@ XdmfHDF5Writer::write(XdmfArray & array, } } } - else {//if the array has been split - //This case should not come up often as it requires truly gigantic data sets - //see if it will fit in the next file - //if it will just go to the next file - //otherwise split it. + else { + // If the array has been split + // This case should not come up often as it requires truly gigantic data sets + // See if it will fit in the next file + // If it will just go to the next file + // Otherwise split it. if ((remainingValues * dataItemSize) + hdf5Overhead > mImpl->mHDF5FileSizeLimit*(1024*1024) && usableSpace > 0) { unsigned int blockSizeSubtotal = 1; unsigned int dimensionIndex = 0; - //find the dimension that was split + // Find the dimension that was split while (dimensionIndex < dataspaceDimensions.size() && blockSizeSubtotal <= amountAlreadyWritten) { blockSizeSubtotal *= dataspaceDimensions[dimensionIndex]; dimensionIndex++; - }//It should end on the "blockSizeSubtotal <= arrayStartIndex" statement, the other half is for backup - //move back one dimension so we're working on the dimension that was split, not the one after it + } + // It should end on the "blockSizeSubtotal <= arrayStartIndex" statement, the other half is for backup + // Move back one dimension so we're working on the dimension that was split, not the one after it dimensionIndex--; blockSizeSubtotal /= dataspaceDimensions[dimensionIndex]; unsigned int j = 0; @@ -1063,19 +1102,19 @@ XdmfHDF5Writer::write(XdmfArray & array, partialDimensions.push_back(dimensions[j]); partialDataSizes.push_back(dataspaceDimensions[j]); } - //continue if the block is smaller than the available size + // Continue if the block is smaller than the available size if (blockSizeSubtotal <=usableSpace) { - //find number of blocks that will fit - //this should be less than the current value for the dimension + // Find number of blocks that will fit + // This should be less than the current value for the dimension unsigned int numBlocks = usableSpace / blockSizeSubtotal; - //add dimensions to the partial vectors + // Add dimensions to the partial vectors if (mMode == Hyperslab) { int newStart = (start[j] + stride[j] * previousDimensions[j]) - previousDataSizes[j]; while (newStart < 0) { newStart += stride[j]; } partialStarts.push_back(newStart); - //stride should not change in this algorithm + // Stride should not change in this algorithm partialStrides.push_back(stride[j]); partialDataSizes.push_back(numBlocks); //determine how many values from the array will fit into the blocks being used @@ -1088,27 +1127,31 @@ XdmfHDF5Writer::write(XdmfArray & array, if (newStart > (int)numBlocks) { displacement = 0; } - if ((dimensions[j] - previousDimensions[j]) <= displacement) {//if there are less values than there are space for, just write all of them. + if ((dimensions[j] - previousDimensions[j]) <= displacement) { + // If there are less values than there are space for, just write all of them. partialDimensions.push_back(dimensions[j] - previousDimensions[j]); } - else {//otherwise write what space allows for + else { + // Otherwise write what space allows for partialDimensions.push_back(displacement); } } else { - //start and stride are only specified in hyperslab + // Start and stride are only specified in hyperslab partialStarts.push_back(start[j]); partialStrides.push_back(stride[j]); partialDataSizes.push_back(numBlocks); partialDimensions.push_back(numBlocks); } - //place dimensions into previous dimensions for later iterations + // Place dimensions into previous dimensions for later iterations } - else {//if this is larger than usable space, try the next file - //if moving to next file, just do nothing and pass out of the if statement - //but also check if specified file size is too small - if (mImpl->mHDF5FileSizeLimit*(1024*1024) < blockSizeSubtotal) {//this shouldn't ever trigger, but it's good to cover ourselves - //and throw an error if the block size won't work + else { + // If this is larger than usable space, try the next file + // If moving to next file, just do nothing and pass out of the if statement + // but also check if specified file size is too small + if (mImpl->mHDF5FileSizeLimit*(1024*1024) < blockSizeSubtotal) { + // This shouldn't ever trigger, but it's good to cover ourselves + // Throw an error if the block size won't work try { XdmfError::message(XdmfError::FATAL, "Error: Dimension Block size / Maximum File size mismatch.\n"); @@ -1120,18 +1163,21 @@ XdmfHDF5Writer::write(XdmfArray & array, } } } - //move to next file + // Move to next file mImpl->mFileIndex++; } } - if (partialDimensions.size() > 0) {//building the array to be written - int containedInDimensions = 1;//count moved + if (partialDimensions.size() > 0) { + // Building the array to be written + int containedInDimensions = 1; + // Count moved for (unsigned int j = 0 ; j < partialDimensions.size(); ++j) { containedInDimensions *= partialDimensions[j]; } - int containedInPriorDimensions = controllerIndexOffset;//starting index + // Starting index + int containedInPriorDimensions = controllerIndexOffset; int startOffset = 1; for (unsigned int j = 0; j < previousDimensions.size(); ++j) { startOffset *= previousDimensions[j]; @@ -1200,7 +1246,8 @@ XdmfHDF5Writer::write(XdmfArray & array, array.getValues(containedInPriorDimensions, movedData, containedInDimensions); partialArray->insert(0, movedData, containedInDimensions); } - else if (closeDatatype) {//closeDatatype is only true if strings are being used + else if (closeDatatype) { + // closeDatatype is only true if strings are being used partialArray->initialize(XdmfArrayType::String(), 0); for (int j = containedInPriorDimensions; j < containedInPriorDimensions + containedInDimensions; ++j) { partialArray->pushBack(array.getValue(j)); @@ -1220,23 +1267,23 @@ XdmfHDF5Writer::write(XdmfArray & array, if (containedInDimensions + containedInPriorDimensions == dimensionTotal) { controllerIndexOffset += dimensionTotal; } - //for hyperslab the space is controlled by the dataspace dimensions - //so use that since the dimensions should be equal to the dataspace dimensions in all other variations - //total up written data space + // For hyperslab the space is controlled by the dataspace dimensions + // So use that since the dimensions should be equal to the dataspace dimensions in all other variations + // Total up written data space unsigned int writtenDataSpace = 1; for (unsigned int j = 0; j < partialDataSizes.size(); ++j) { writtenDataSpace *= partialDataSizes[j]; } amountAlreadyWritten += writtenDataSpace; - //generate previous dimensions + // Generate previous dimensions if (previousDataSizes.size() == 0) { previousDataSizes = partialDataSizes; previousDimensions = partialDimensions; } else { - //determine if the sizes match - //if they do, add the top values together - //otherwise, compress the higher dimensions and then add them + // Determine if the sizes match + // If they do, add the top values together + // Otherwise, compress the higher dimensions and then add them if (previousDimensions.size() == partialDimensions.size()) { previousDimensions[previousDimensions.size()-1] += partialDimensions[previousDimensions.size()-1]; } @@ -1280,9 +1327,9 @@ XdmfHDF5Writer::write(XdmfArray & array, } if (mMode == Append) { - //if the written filename is different write add the previous controller + // If the written filename is different write add the previous controller if (*(filesWritten.rbegin()) != heavyDataController->getFilePath()) { - //should also be different from previous controller + // Should also be different from previous controller if (filesWritten.size() > 1) { if (*(filesWritten.rbegin()) != *((filesWritten.rbegin())++)) { array.insert(heavyDataController); @@ -1297,9 +1344,9 @@ XdmfHDF5Writer::write(XdmfArray & array, } else { - //otherwise work with the full array + // Otherwise work with the full array shared_ptr partialArray = XdmfArray::New(); - //need to copy by duplicating the contents of the array + // Need to copy by duplicating the contents of the array unsigned int j = controllerIndexOffset; try { @@ -1434,22 +1481,21 @@ XdmfHDF5Writer::write(XdmfArray & array, partialArray->insert(0, movedData, movedSize); j+=movedSize; } - else if (closeDatatype) {//closeDatatype is only true if strings are being used + else if (closeDatatype) { + // closeDatatype is only true if strings are being used partialArray->initialize(XdmfArrayType::String(), 0); - //transfering via loop because the getValues function is not fully tested with strings + // Transfering via loop because the getValues function is not fully tested with strings for (j = controllerIndexOffset; j < controllerIndexOffset + heavyDataController->getSize() && j < array.getSize(); ++j){ partialArray->pushBack(array.getValue(j)); } } -// if (partialArray->getSize()==0) { -// break; -// } arrayOffsetsWritten.push_back(controllerIndexOffset); - controllerIndexOffset = j;//set the offset to the point after the end of the current subset + // Set the offset to the point after the end of the current subset + controllerIndexOffset = j; arraysWritten.push_back(partialArray); filesWritten.push_back(hdf5FilePath); - //also need to push the starts and strides loaded from the HeavyDataController + // Also need to push the starts and strides loaded from the HeavyDataController startsWritten.push_back(start); stridesWritten.push_back(stride); dimensionsWritten.push_back(dimensions); @@ -1473,11 +1519,11 @@ XdmfHDF5Writer::write(XdmfArray & array, std::list::iterator arrayOffsetWalker = arrayOffsetsWritten.begin(); - //loop based on the amount of blocks split from the array. + // Loop based on the amount of blocks split from the array. for (unsigned int writeIndex = 0; writeIndex < arraysWritten.size(); ++writeIndex) { - //this is the section where the data is written to hdf5 - //if you want to change the writer to write to a different data format, do it here + // This is the section where the data is written to hdf5 + // If you want to change the writer to write to a different data format, do it here std::string curFileName = *fileNameWalker; shared_ptr curArray = *arrayWalker; @@ -1489,8 +1535,8 @@ XdmfHDF5Writer::write(XdmfArray & array, bool closeFile = false; - //This is meant to open files if it isn't already opened by the write prior - //If it wasn't open prior to writing it will be closed after writing + // This is meant to open files if it isn't already opened by the write prior + // If it wasn't open prior to writing it will be closed after writing if (mImpl->mOpenFile.compare(curFileName) != 0) { if(mImpl->mHDF5Handle < 0) { closeFile = true; @@ -1514,14 +1560,7 @@ XdmfHDF5Writer::write(XdmfArray & array, H5P_DEFAULT); } - //hid_t checkspace = H5S_ALL; - //checkspace = H5Dget_space(dataset); - //hssize_t checksize = H5Sget_simple_extent_npoints(checkspace); - //if(checkspace != H5S_ALL) { - // status = H5Sclose(checkspace); - //} - - // if default mode find a new data set to write to (keep + // If default mode find a new data set to write to (keep // incrementing dataSetId) if(dataset >=0 && mMode == Default) { while(true) { @@ -1547,10 +1586,11 @@ XdmfHDF5Writer::write(XdmfArray & array, std::vector current_dims(curDataSize.begin(), curDataSize.end()); - if(dataset < 0) {//if the dataset doesn't contain anything + if(dataset < 0) { + // If the dataset doesn't contain anything std::vector maximum_dims(curDimensions.size(), H5S_UNLIMITED); - //create a new dataspace + // Create a new dataspace dataspace = H5Screate_simple(current_dims.size(), ¤t_dims[0], &maximum_dims[0]); @@ -1561,13 +1601,15 @@ XdmfHDF5Writer::write(XdmfArray & array, current_dims.end(), 1, std::multiplies()); - //the Nth root of the chunk size divided by the dimensions added together + // The Nth root of the chunk size divided by the dimensions added together const double factor = std::pow(((double)mImpl->mChunkSize / totalDimensionsSize), - 1.0 / current_dims.size());//The end result is the amount of slots alloted per unit of dimension + 1.0 / current_dims.size()); + // The end result is the amount of slots alloted per unit of dimension std::vector chunk_size(current_dims.begin(), current_dims.end()); - if (mImpl->mChunkSize > 0) {//The chunk size won't do anything unless it's positive + if (mImpl->mChunkSize > 0) { + // The chunk size won't do anything unless it's positive for(std::vector::iterator iter = chunk_size.begin(); iter != chunk_size.end(); ++iter) { *iter = (hsize_t)(*iter * factor); @@ -1578,7 +1620,7 @@ XdmfHDF5Writer::write(XdmfArray & array, } status = H5Pset_chunk(property, current_dims.size(), &chunk_size[0]); - //use that dataspace to create a new dataset + // Use that dataspace to create a new dataset dataset = H5Dcreate(mImpl->mHDF5Handle, dataSetPath.str().c_str(), datatype, @@ -1597,7 +1639,7 @@ XdmfHDF5Writer::write(XdmfArray & array, hssize_t datasize = H5Sget_simple_extent_npoints(dataspace); status = H5Sclose(dataspace); - //reset the datasize if the file or set is different + // Reset the datasize if the file or set is different if (curFileName != previousControllers[i]->getFilePath()) { datasize = 0; } @@ -1719,7 +1761,7 @@ XdmfHDF5Writer::write(XdmfArray & array, status = H5Dclose(dataset); - //this is causing a lot of overhead + // This is causing a lot of overhead if(closeFile) { mImpl->closeFile(); } @@ -1733,11 +1775,12 @@ XdmfHDF5Writer::write(XdmfArray & array, // Attach a new controller to the array shared_ptr newDataController = - shared_ptr();//This generates an empty pointer + shared_ptr(); + //This generates an empty pointer unsigned int newSize; if(mMode == Append) { - //find data size + // Find data size mImpl->openFile(curFileName, fapl, mDataSetId); hid_t checkset = H5Dopen(mImpl->mHDF5Handle, @@ -1775,7 +1818,8 @@ XdmfHDF5Writer::write(XdmfArray & array, } } - if(!newDataController) {//if the controller wasn't generated by append + if(!newDataController) { + // If the controller wasn't generated by append try { newDataController = this->createHDF5Controller(curFileName, diff --git a/core/XdmfHDF5WriterDSM.cpp b/core/XdmfHDF5WriterDSM.cpp index 3aaa5ee259777a08d1dad7c3c8ea9de7bb31adbd..3db0edca7d07ed9a0447bf6c8835745573fd017c 100644 --- a/core/XdmfHDF5WriterDSM.cpp +++ b/core/XdmfHDF5WriterDSM.cpp @@ -118,23 +118,23 @@ XdmfHDF5WriterDSM::XdmfHDF5WriterDSM(const std::string & filePath, mGroupSize(-1), mServerMode(false) { - H5FDdsmManager * newManager = new H5FDdsmManager(); - newManager->SetMpiComm(comm); - newManager->SetLocalBufferSizeMBytes(bufferSize); - newManager->SetIsStandAlone(H5FD_DSM_TRUE); - newManager->Create(); + H5FDdsmManager * newManager = new H5FDdsmManager(); + newManager->SetMpiComm(comm); + newManager->SetLocalBufferSizeMBytes(bufferSize); + newManager->SetIsStandAlone(H5FD_DSM_TRUE); + newManager->Create(); - H5FD_dsm_set_manager(newManager); + H5FD_dsm_set_manager(newManager); - H5FD_dsm_set_options(H5FD_DSM_LOCK_ASYNCHRONOUS); + H5FD_dsm_set_options(H5FD_DSM_LOCK_ASYNCHRONOUS); - H5FDdsmBuffer * newBuffer = newManager->GetDsmBuffer(); - mDSMManager = newManager; - mDSMBuffer = newBuffer; + H5FDdsmBuffer * newBuffer = newManager->GetDsmBuffer(); + mDSMManager = newManager; + mDSMBuffer = newBuffer; } -//the database/nonthreaded version +// The database/nonthreaded version XdmfHDF5WriterDSM::XdmfHDF5WriterDSM(const std::string & filePath, XdmfDSMBuffer * const dsmBuffer, @@ -174,105 +174,96 @@ XdmfHDF5WriterDSM::XdmfHDF5WriterDSM(const std::string & filePath, mDSMBuffer(NULL), mServerMode(true) { - //negative values will be changed to maximum range - if (startCoreIndex < 0) - { - startCoreIndex = 0; - } - if (endCoreIndex < 0) - { - endCoreIndex = mGroupSize - 1; - } - - //ensure start index is less than end index - if (startCoreIndex > endCoreIndex) - { - int tempholder = startCoreIndex; - startCoreIndex = endCoreIndex; - endCoreIndex = tempholder; - } - - mGroupComm = comm; - mStartCoreIndex = startCoreIndex; - mEndCoreIndex = endCoreIndex; - - MPI_Comm_size(comm, &mGroupSize); - MPI_Comm_rank(comm, &mRank); - - MPI_Group workers, dsmgroup, serversplit, servergroup; - - int * ServerIds = (int *)calloc((mEndCoreIndex - mStartCoreIndex + 1), sizeof(int)); - unsigned int index = 0; - for(int i=mStartCoreIndex ; i <= mEndCoreIndex ; ++i) - { - ServerIds[index++] = i; - } - - MPI_Comm_group(comm, &serversplit); - MPI_Group_incl(serversplit, index, ServerIds, &servergroup); - MPI_Comm_create(comm, servergroup, &mServerComm); - MPI_Comm_group(comm, &dsmgroup); - MPI_Group_excl(dsmgroup, index, ServerIds, &workers); - MPI_Comm_create(comm, workers, &mWorkerComm); - cfree(ServerIds); - - //create the manager - - mDSMServerManager = new XdmfDSMManager(); - - mDSMServerManager->SetLocalBufferSizeMBytes(bufferSize); - mDSMServerManager->SetInterCommType(H5FD_DSM_COMM_MPI); - - MPI_Barrier(mGroupComm); - - if (mRank >=mStartCoreIndex && mRank <=mEndCoreIndex) - { - mDSMServerManager->SetMpiComm(mServerComm); - mDSMServerManager->Create(); - } - else - { - mDSMServerManager->SetMpiComm(mWorkerComm); - mDSMServerManager->SetIsServer(false); - mDSMServerManager->Create(mStartCoreIndex, mEndCoreIndex); - } - - XDMF_dsm_set_manager(mDSMServerManager); - - mDSMServerBuffer = mDSMServerManager->GetDsmBuffer(); - - mDSMServerBuffer->GetComm()->DupInterComm(mGroupComm); - mDSMServerBuffer->SetIsConnected(true); - - MPI_Barrier(comm); - - //loop needs to be started before anything can be done to the file, since the service is what sets up the file - - if (mRank < mStartCoreIndex || mRank > mEndCoreIndex) - { - //turn off the server designation - mDSMServerBuffer->SetIsServer(H5FD_DSM_FALSE);//if this is set to false then the buffer will attempt to connect to the intercomm for DSM stuff - mDSMServerManager->SetIsServer(H5FD_DSM_FALSE); - } - else - { - //on cores where memory is set up, start the service loop - //this should iterate infinitely until a value to end the loop is passed - H5FDdsmInt32 returnOpCode; - try - { - mDSMServerBuffer->BufferServiceLoop(&returnOpCode); - } - catch (XdmfError e) - { - throw e; - } - } + // Negative values will be changed to maximum range + if (startCoreIndex < 0) { + startCoreIndex = 0; + } + if (endCoreIndex < 0) { + endCoreIndex = mGroupSize - 1; + } + + // Ensure start index is less than end index + if (startCoreIndex > endCoreIndex) { + int tempholder = startCoreIndex; + startCoreIndex = endCoreIndex; + endCoreIndex = tempholder; + } + + mGroupComm = comm; + mStartCoreIndex = startCoreIndex; + mEndCoreIndex = endCoreIndex; + + MPI_Comm_size(comm, &mGroupSize); + MPI_Comm_rank(comm, &mRank); + + MPI_Group workers, dsmgroup, serversplit, servergroup; + + int * ServerIds = (int *)calloc((mEndCoreIndex - mStartCoreIndex + 1), sizeof(int)); + unsigned int index = 0; + for(int i=mStartCoreIndex ; i <= mEndCoreIndex ; ++i) { + ServerIds[index++] = i; + } + + MPI_Comm_group(comm, &serversplit); + MPI_Group_incl(serversplit, index, ServerIds, &servergroup); + MPI_Comm_create(comm, servergroup, &mServerComm); + MPI_Comm_group(comm, &dsmgroup); + MPI_Group_excl(dsmgroup, index, ServerIds, &workers); + MPI_Comm_create(comm, workers, &mWorkerComm); + cfree(ServerIds); + + // Create the manager + + mDSMServerManager = new XdmfDSMManager(); + + mDSMServerManager->SetLocalBufferSizeMBytes(bufferSize); + mDSMServerManager->SetInterCommType(H5FD_DSM_COMM_MPI); + + MPI_Barrier(mGroupComm); + + if (mRank >=mStartCoreIndex && mRank <=mEndCoreIndex) { + mDSMServerManager->SetMpiComm(mServerComm); + mDSMServerManager->Create(); + } + else { + mDSMServerManager->SetMpiComm(mWorkerComm); + mDSMServerManager->SetIsServer(false); + mDSMServerManager->Create(mStartCoreIndex, mEndCoreIndex); + } + + XDMF_dsm_set_manager(mDSMServerManager); + + mDSMServerBuffer = mDSMServerManager->GetDsmBuffer(); + + mDSMServerBuffer->GetComm()->DupInterComm(mGroupComm); + mDSMServerBuffer->SetIsConnected(true); + + MPI_Barrier(comm); + + // Loop needs to be started before anything can be done to the file, since the service is what sets up the file + + if (mRank < mStartCoreIndex || mRank > mEndCoreIndex) { + // Turn off the server designation + mDSMServerBuffer->SetIsServer(H5FD_DSM_FALSE); + // If this is set to false then the buffer will attempt to connect to the intercomm for DSM communications + mDSMServerManager->SetIsServer(H5FD_DSM_FALSE); + } + else { + // On cores where memory is set up, start the service loop + // This should iterate infinitely until a value to end the loop is passed + H5FDdsmInt32 returnOpCode; + try { + mDSMServerBuffer->BufferServiceLoop(&returnOpCode); + } + catch (XdmfError e) { + throw e; + } + } } XdmfHDF5WriterDSM::~XdmfHDF5WriterDSM() { - + } shared_ptr @@ -308,9 +299,27 @@ XdmfHDF5WriterDSM::createHDF5Controller(const std::string & hdf5FilePath, } } -H5FDdsmManager * XdmfHDF5WriterDSM::getManager() +void XdmfHDF5WriterDSM::deleteManager() { - return mDSMManager; + if (mDSMManager != NULL) + { + delete mDSMManager; + } + if (mDSMServerManager != NULL) + { + closeFile(); + delete mDSMServerManager; + } +} + +void +XdmfHDF5WriterDSM::closeFile() +{ + if(mFAPL >= 0) { + herr_t status = H5Pclose(mFAPL); + mFAPL = -1; + } + XdmfHDF5Writer::closeFile(); } H5FDdsmBuffer * XdmfHDF5WriterDSM::getBuffer() @@ -318,9 +327,9 @@ H5FDdsmBuffer * XdmfHDF5WriterDSM::getBuffer() return mDSMBuffer; } -XdmfDSMManager * XdmfHDF5WriterDSM::getServerManager() +H5FDdsmManager * XdmfHDF5WriterDSM::getManager() { - return mDSMServerManager; + return mDSMManager; } XdmfDSMBuffer * XdmfHDF5WriterDSM::getServerBuffer() @@ -328,6 +337,18 @@ XdmfDSMBuffer * XdmfHDF5WriterDSM::getServerBuffer() return mDSMServerBuffer; } +MPI_Comm XdmfHDF5WriterDSM::getServerComm() +{ + MPI_Comm returnComm; + int status = MPI_Comm_dup(mServerComm, &returnComm); + return returnComm; +} + +XdmfDSMManager * XdmfHDF5WriterDSM::getServerManager() +{ + return mDSMServerManager; +} + bool XdmfHDF5WriterDSM::getServerMode() { return mServerMode; @@ -335,27 +356,25 @@ bool XdmfHDF5WriterDSM::getServerMode() MPI_Comm XdmfHDF5WriterDSM::getWorkerComm() { - MPI_Comm returnComm; - int status = MPI_Comm_dup(mWorkerComm, &returnComm); - return returnComm; + MPI_Comm returnComm; + int status = MPI_Comm_dup(mWorkerComm, &returnComm); + return returnComm; } -MPI_Comm XdmfHDF5WriterDSM::getServerComm() +void XdmfHDF5WriterDSM::setBuffer(H5FDdsmBuffer * newBuffer) { - MPI_Comm returnComm; - int status = MPI_Comm_dup(mServerComm, &returnComm); - return returnComm; + mDSMBuffer = newBuffer; } -void XdmfHDF5WriterDSM::setManager(H5FDdsmManager * newManager) +void XdmfHDF5WriterDSM::setBuffer(XdmfDSMBuffer * newBuffer) { - H5FDdsmBuffer * newBuffer = newManager->GetDsmBuffer(); - mDSMManager = newManager; - mDSMBuffer = newBuffer; + mDSMServerBuffer = newBuffer; } -void XdmfHDF5WriterDSM::setBuffer(H5FDdsmBuffer * newBuffer) +void XdmfHDF5WriterDSM::setManager(H5FDdsmManager * newManager) { + H5FDdsmBuffer * newBuffer = newManager->GetDsmBuffer(); + mDSMManager = newManager; mDSMBuffer = newBuffer; } @@ -366,9 +385,32 @@ void XdmfHDF5WriterDSM::setManager(XdmfDSMManager * newManager) mDSMServerBuffer = newBuffer; } -void XdmfHDF5WriterDSM::setBuffer(XdmfDSMBuffer * newBuffer) +void XdmfHDF5WriterDSM::setServerComm(MPI_Comm comm) { - mDSMServerBuffer = newBuffer; + int status; + if (mServerComm != MPI_COMM_NULL) { + status = MPI_Comm_free(&mServerComm); + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Failed to disconnect Comm"); + } + catch (XdmfError e) { + throw e; + } + } + } + if (comm != MPI_COMM_NULL) { + status = MPI_Comm_dup(comm, &mServerComm); + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Failed to duplicate Comm"); + } + catch (XdmfError e) { + throw e; + } + } + } + mDSMServerBuffer->GetComm()->DupComm(comm); } void XdmfHDF5WriterDSM::setServerMode(bool newMode) @@ -378,131 +420,56 @@ void XdmfHDF5WriterDSM::setServerMode(bool newMode) void XdmfHDF5WriterDSM::setWorkerComm(MPI_Comm comm) { - int status; - if (mWorkerComm != MPI_COMM_NULL) - { - status = MPI_Comm_free(&mWorkerComm); - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Failed to disconnect Comm"); - } - catch (XdmfError e) - { - throw e; - } - } - } - if (comm != MPI_COMM_NULL) - { - status = MPI_Comm_dup(comm, &mWorkerComm); - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Failed to duplicate Comm"); - } - catch (XdmfError e) - { - throw e; - } - } - } - mDSMServerBuffer->GetComm()->DupComm(comm); -} - -void XdmfHDF5WriterDSM::setServerComm(MPI_Comm comm) -{ - int status; - if (mServerComm != MPI_COMM_NULL) - { - status = MPI_Comm_free(&mServerComm); - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Failed to disconnect Comm"); - } - catch (XdmfError e) - { - throw e; - } - } - } - if (comm != MPI_COMM_NULL) - { - status = MPI_Comm_dup(comm, &mServerComm); - if (status != MPI_SUCCESS) - { - try - { - XdmfError::message(XdmfError::FATAL, "Failed to duplicate Comm"); - } - catch (XdmfError e) - { - throw e; - } - } - } - mDSMServerBuffer->GetComm()->DupComm(comm); -} - -void XdmfHDF5WriterDSM::deleteManager() -{ - if (mDSMManager != NULL) - { - delete mDSMManager; - } - if (mDSMServerManager != NULL) - { - closeFile(); - delete mDSMServerManager; - } + int status; + if (mWorkerComm != MPI_COMM_NULL) { + status = MPI_Comm_free(&mWorkerComm); + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Failed to disconnect Comm"); + } + catch (XdmfError e) { + throw e; + } + } + } + if (comm != MPI_COMM_NULL) { + status = MPI_Comm_dup(comm, &mWorkerComm); + if (status != MPI_SUCCESS) { + try { + XdmfError::message(XdmfError::FATAL, "Failed to duplicate Comm"); + } + catch (XdmfError e) { + throw e; + } + } + } + mDSMServerBuffer->GetComm()->DupComm(comm); } void XdmfHDF5WriterDSM::stopDSM() { - //send manually - for (int i = mStartCoreIndex; i <= mEndCoreIndex; ++i) - { - try - { - mDSMServerBuffer->SendCommandHeader(H5FD_DSM_OPCODE_DONE, i, 0, 0, H5FD_DSM_INTER_COMM); - } - catch (XdmfError e) - { - throw e; - } - //originally this was set to the intra comm - //that doesn't work in this instance because it won't reach the server cores - } + // Send manually + for (int i = mStartCoreIndex; i <= mEndCoreIndex; ++i) { + try { + mDSMServerBuffer->SendCommandHeader(H5FD_DSM_OPCODE_DONE, i, 0, 0, H5FD_DSM_INTER_COMM); + } + catch (XdmfError e) { + throw e; + } + } } void XdmfHDF5WriterDSM::restartDSM() { - if (mRank >= mStartCoreIndex && mRank <= mEndCoreIndex) - { - H5FDdsmInt32 returnOpCode; - try - { - mDSMServerBuffer->BufferServiceLoop(&returnOpCode); - } - catch (XdmfError e) - { - throw e; - } - } -} - -void -XdmfHDF5WriterDSM::closeFile() -{ - if(mFAPL >= 0) { - herr_t status = H5Pclose(mFAPL); - mFAPL = -1; + if (mRank >= mStartCoreIndex && mRank <= mEndCoreIndex) { + H5FDdsmInt32 returnOpCode; + try { + mDSMServerBuffer->BufferServiceLoop(&returnOpCode); + } + catch (XdmfError e) { + throw e; + } } - XdmfHDF5Writer::closeFile(); } void diff --git a/core/XdmfWriter.cpp b/core/XdmfWriter.cpp index 0df47eb1361a9cdc20feda1e8e582a28f952bd1a..2aee60a52c2b049ecc26bf1fc17ace2fc303afa8 100644 --- a/core/XdmfWriter.cpp +++ b/core/XdmfWriter.cpp @@ -73,7 +73,7 @@ public: { mXPath.clear(); - //this section writes to file + // This section writes to file std::ofstream fileStream; if(!mStream) { fileStream.open(mXMLFilePath.c_str()); @@ -311,7 +311,8 @@ XdmfWriter::visit(XdmfArray & array, array.getSize() > mImpl->mLightDataLimit) { // Write values to heavy data - try {// this takes about half the time needed + try { + // This takes about half the time needed mImpl->mHeavyDataWriter->visit(array, mImpl->mHeavyDataWriter); } catch (XdmfError e) { @@ -324,17 +325,17 @@ XdmfWriter::visit(XdmfArray & array, array.getHeavyDataController(i)->getFilePath(); size_t index = heavyDataPath.find_last_of("/\\"); if(index != std::string::npos) { - // if path is not a folder + // If path is not a folder // put the directory path into this variable const std::string heavyDataDir = heavyDataPath.substr(0, index + 1); - // if the directory is in the XML File Path + // If the directory is in the XML File Path if(mImpl->mXMLFilePath.find(heavyDataDir) == 0) { heavyDataPath = heavyDataPath.substr(heavyDataDir.size(), heavyDataPath.size() - heavyDataDir.size()); - // pull the file off of the end and place it in the DataPath + // Pull the file off of the end and place it in the DataPath } - // otherwise the full path is required + // Otherwise the full path is required } std::stringstream dimensionStream; for (unsigned int j = 0; j < array.getHeavyDataController(i)->getDimensions().size(); ++j) { @@ -343,7 +344,7 @@ XdmfWriter::visit(XdmfArray & array, dimensionStream << " "; } } - // clear the stream + // Clear the stream valuesStream.str(std::string()); if (array.getNumberHeavyDataControllers() > 1) { valuesStream << heavyDataPath << ":"