From 9075be36b535ccb328c9eba8eee21c629b66fb25 Mon Sep 17 00:00:00 2001 From: "Andrew J. Burns (Cont" Date: Tue, 2 Jul 2013 15:26:49 -0400 Subject: [PATCH] added nonthreaded DSM, inter program connection hasn't been tested --- Xdmf.i | 3 + core/CMakeLists.txt | 8 +- core/XdmfCore.i | 6 + core/XdmfDSMBuffer.cpp | 1112 ++++++++++++++++ core/XdmfDSMBuffer.hpp | 1405 +++++++++++++++++++++ core/XdmfDSMCommMPI.cpp | 405 ++++++ core/XdmfDSMCommMPI.hpp | 669 ++++++++++ core/XdmfDSMDriver.cpp | 1345 ++++++++++++++++++++ core/XdmfDSMDriver.hpp | 127 ++ core/XdmfDSMManager.cpp | 316 +++++ core/XdmfDSMManager.hpp | 812 ++++++++++++ core/XdmfHDF5Controller.cpp | 2 + core/XdmfHDF5ControllerDSM.cpp | 404 +++++- core/XdmfHDF5ControllerDSM.hpp | 598 ++++++++- core/XdmfHDF5Writer.cpp | 8 +- core/XdmfHDF5WriterDSM.cpp | 410 +++++- core/XdmfHDF5WriterDSM.hpp | 529 +++++++- core/XdmfWriter.cpp | 3 - core/tests/Cxx/TestXdmfHDF5WriterDSM.cpp | 4 +- examples/Cxx/DSMLoopTest.cpp | 310 +++++ examples/Cxx/ExampleXdmfDSMNoThread.cpp | 537 ++++++++ examples/Python/XdmfExampleDSMNoThread.py | 402 ++++++ tests/Cxx/HugeWriteArray.cpp | 17 +- tests/Cxx/XdmfPostFixCalc.cpp | 36 +- utils/XdmfUtils.i | 3 + 25 files changed, 9356 insertions(+), 115 deletions(-) create mode 100644 core/XdmfDSMBuffer.cpp create mode 100644 core/XdmfDSMBuffer.hpp create mode 100644 core/XdmfDSMCommMPI.cpp create mode 100644 core/XdmfDSMCommMPI.hpp create mode 100644 core/XdmfDSMDriver.cpp create mode 100644 core/XdmfDSMDriver.hpp create mode 100644 core/XdmfDSMManager.cpp create mode 100644 core/XdmfDSMManager.hpp create mode 100644 examples/Cxx/DSMLoopTest.cpp create mode 100644 examples/Cxx/ExampleXdmfDSMNoThread.cpp create mode 100644 examples/Python/XdmfExampleDSMNoThread.py diff --git a/Xdmf.i b/Xdmf.i index d9397734..90d60ed9 100644 --- a/Xdmf.i +++ b/Xdmf.i @@ -18,6 +18,9 @@ swig -v -c++ -python -o XdmfPython.cpp Xdmf.i #include #include #include + #include + #include + #include #include #include #include diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 814ace15..8cf00884 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -80,7 +80,7 @@ check_cxx_source_compiles(" struct Base { virtual ~Base(){} }; struct Derived : public Base {}; -int main(int ac, char ** av) +int main(int ac, char * av) { boost::shared_ptr ptr(new Base()); boost::shared_dynamic_cast(ptr); @@ -114,7 +114,11 @@ if(XDMF_BUILD_DSM) set(XdmfCoreSources ${XdmfCoreSources} XdmfHDF5ControllerDSM - XdmfHDF5WriterDSM) + XdmfHDF5WriterDSM + XdmfDSMCommMPI + XdmfDSMBuffer + XdmfDSMManager + XdmfDSMDriver) find_package(H5FDdsm REQUIRED NO_MODULE) if(H5FDdsm_FOUND) diff --git a/core/XdmfCore.i b/core/XdmfCore.i index 22635f32..f5a76c87 100644 --- a/core/XdmfCore.i +++ b/core/XdmfCore.i @@ -16,6 +16,9 @@ swig -v -c++ -python -o XdmfCorePython.cpp XdmfCore.i #include #include #include + #include + #include + #include #include #include #include @@ -968,6 +971,9 @@ swig -v -c++ -python -o XdmfCorePython.cpp XdmfCore.i #ifdef XDMF_BUILD_DSM %include XdmfHDF5ControllerDSM.hpp %include XdmfHDF5WriterDSM.hpp + %include XdmfDSMManager.hpp + %include XdmfDSMBuffer.hpp + %include XdmfDSMCommMPI.hpp #endif %include XdmfArray.hpp diff --git a/core/XdmfDSMBuffer.cpp b/core/XdmfDSMBuffer.cpp new file mode 100644 index 00000000..be4a9e50 --- /dev/null +++ b/core/XdmfDSMBuffer.cpp @@ -0,0 +1,1112 @@ +/*****************************************************************************/ +/* XDMF */ +/* eXtensible Data Model and Format */ +/* */ +/* Id : XdmfDSMBuffer.hpp */ +/* */ +/* Author: */ +/* Andrew Burns */ +/* andrew.j.burns2@us.army.mil */ +/* US Army Research Laboratory */ +/* Aberdeen Proving Ground, MD */ +/* */ +/* Copyright @ 2013 US Army Research Laboratory */ +/* All Rights Reserved */ +/* See Copyright.txt for details */ +/* */ +/* This software is distributed WITHOUT ANY WARRANTY; without */ +/* even the implied warranty of MERCHANTABILITY or FITNESS */ +/* FOR A PARTICULAR PURPOSE. See the above copyright notice */ +/* for more information. */ +/* */ +/*****************************************************************************/ + +/*========================================================================= + This code is derived from an earlier work and is distributed + with permission from, and thanks to ... +=========================================================================*/ + +/*============================================================================ + + Project : H5FDdsm + Module : H5FDdsmBufferService.cxx H5FDdsmBuffer.cxx + + Authors: + John Biddiscombe Jerome Soumagne + biddisco@cscs.ch soumagne@cscs.ch + + Copyright (C) CSCS - Swiss National Supercomputing Centre. + You may use modify and and distribute this code freely providing + 1) This copyright notice appears on all copies of source code + 2) An acknowledgment appears with any substantial usage of the code + 3) If this code is contributed to any other open source project, it + must not be reformatted such that the indentation, bracketing or + overall style is modified significantly. + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + + This work has received funding from the European Community's Seventh + Framework Programme (FP7/2007-2013) under grant agreement 225967 âxtMuSEâOC + +============================================================================*/ + +#include +#include +#include +#include +#include +#include +#include + +XdmfDSMBuffer::XdmfDSMBuffer() +{ + this->CommChannel = XDMF_DSM_INTER_COMM; + this->IsServer = true; + this->StartAddress = this->EndAddress = 0; + this->StartServerId = this->EndServerId = -1; + this->Length = 0; + this->TotalLength = 0; + this->BlockLength = 0; + this->Comm = NULL; + this->DataPointer = NULL; + this->IsConnected = false; +} + +XdmfDSMBuffer::~XdmfDSMBuffer() +{ + if (this->DataPointer) + { + free(this->DataPointer); + } + this->DataPointer = NULL; +} + +class XdmfDSMBuffer::CommandMsg +{ + public: + int Opcode; + int Source; + int Target; + int Address; + int Length; +}; + +class XdmfDSMBuffer::InfoMsg +{ + public: + int type; + unsigned int length; + unsigned int total_length; + unsigned int block_length; + int start_server_id; + int end_server_id; +}; + +void +XdmfDSMBuffer::ConfigureUniform(XdmfDSMCommMPI *aComm, long aLength, + int startId, int endId, long aBlockLength, + bool random) +{ + if (startId < 0) + { + startId = 0; + } + if (endId < 0) + { + endId = aComm->GetIntraSize() - 1; + } + this->SetDsmType(XDMF_DSM_TYPE_UNIFORM_RANGE); + if ((startId == 0) && (endId == aComm->GetIntraSize() - 1)) + { + this->SetDsmType(XDMF_DSM_TYPE_UNIFORM); + } + if (aBlockLength) + { + if (!random) + { + this->SetDsmType(XDMF_DSM_TYPE_BLOCK_CYCLIC); + } + else + { + this->SetDsmType(XDMF_DSM_TYPE_BLOCK_RANDOM); + } + this->SetBlockLength(aBlockLength); + } + this->StartServerId = startId; + this->EndServerId = endId; + this->SetComm(aComm); + if ((aComm->GetId() >= startId) && (aComm->GetId() <= endId) && this->IsServer) + { + try + { + if (aBlockLength) + { + // For optimization we make the DSM length fit to a multiple of block size + this->SetLength(((long)(aLength / aBlockLength)) * aBlockLength); + } + else + { + this->SetLength(aLength); + } + } + catch (XdmfError e) + { + throw e; + } + this->StartAddress = (aComm->GetId() - startId) * aLength; + this->EndAddress = this->StartAddress + aLength - 1; + } + else + { + if (aBlockLength) + { + this->Length = ((long)(aLength / aBlockLength)) * aBlockLength; + } + else + { + this->Length = aLength; + } + } + this->TotalLength = this->GetLength() * (endId - startId + 1); +} + +bool +XdmfDSMBuffer::GetIsConnected() +{ + return IsConnected; +} + +void +XdmfDSMBuffer::SetIsConnected(bool newStatus) +{ + IsConnected = newStatus; +} + +char * +XdmfDSMBuffer::GetDataPointer() +{ + return this->DataPointer; +} + +int +XdmfDSMBuffer::GetDsmType() +{ + return this->DsmType; +} + +void +XdmfDSMBuffer::SetDsmType(int newDsmType) +{ + this->DsmType = newDsmType; +} + +bool +XdmfDSMBuffer::GetIsServer() +{ + return this->IsServer; +} + +void +XdmfDSMBuffer::SetIsServer(bool newIsServer) +{ + this->IsServer = newIsServer; +} + +int +XdmfDSMBuffer::GetEndAddress() +{ + return this->EndAddress; +} + +int +XdmfDSMBuffer::GetStartAddress() +{ + return this->StartAddress; +} + +int +XdmfDSMBuffer::GetStartServerId() +{ + return this->StartServerId; +} + +int +XdmfDSMBuffer::GetEndServerId() +{ + return this->EndServerId; +} + +long +XdmfDSMBuffer::GetLength() +{ + return this->Length; +} + +long +XdmfDSMBuffer::GetTotalLength() +{ + return this->TotalLength; +} + +long +XdmfDSMBuffer::GetBlockLength() +{ + return this->BlockLength; +} + +void +XdmfDSMBuffer::SetBlockLength(long newBlock) +{ + this->BlockLength = newBlock; +} + +XdmfDSMCommMPI * +XdmfDSMBuffer::GetComm() +{ + return this->Comm; +} + +void +XdmfDSMBuffer::SetComm(XdmfDSMCommMPI * newComm) +{ + this->Comm = newComm; +} + +void +XdmfDSMBuffer::SetLength(long aLength) +{ + this->Length = aLength; + if (this->DataPointer) + { + // try to reallocate + // this should not be called in most cases + this->DataPointer = static_cast(realloc(this->DataPointer, this->Length*sizeof(char))); + } + else + { +#ifdef _WIN32 + this->DataPointer = calloc(this->Length, sizeof(char)); +#else + posix_memalign((void **)(&this->DataPointer), getpagesize(), this->Length); + memset(this->DataPointer, 0, this->Length); +#endif + } + + if (this->DataPointer == NULL) + { + std::stringstream message; + message << "Allocation Failed, unable to allocate " << this->Length; + XdmfError::message(XdmfError::FATAL, message.str()); + } +} + +void +XdmfDSMBuffer::SendCommandHeader(int opcode, int dest, int address, int aLength, int comm) +{ + int status; + CommandMsg cmd; + memset(&cmd, 0, sizeof(CommandMsg)); + cmd.Opcode = opcode; + cmd.Source = this->Comm->GetId(); + cmd.Target = dest; + cmd.Address = address; + cmd.Length = aLength; + + if (comm == XDMF_DSM_INTRA_COMM) + { + status = MPI_Send(&cmd, sizeof(CommandMsg), MPI_UNSIGNED_CHAR, dest, XDMF_DSM_COMMAND_TAG, static_cast(this->Comm)->GetIntraComm()); + } + else if (comm == XDMF_DSM_INTER_COMM) + { + int interSource = 0; + MPI_Comm_rank(static_cast(this->Comm)->GetInterComm(), &interSource); + cmd.Source = interSource; + status = MPI_Send(&cmd, sizeof(CommandMsg), MPI_UNSIGNED_CHAR, dest, XDMF_DSM_COMMAND_TAG, static_cast(this->Comm)->GetInterComm()); + } + else + {//in this case the comm should be a pointer to an MPI_Comm object + status = MPI_Send(&cmd, sizeof(CommandMsg), MPI_UNSIGNED_CHAR, dest, XDMF_DSM_COMMAND_TAG, comm); + } + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Error: Failed to send command header"); + } + catch (XdmfError e) + { + throw e; + } + } +} + +void +XdmfDSMBuffer::ReceiveCommandHeader(int *opcode, int *source, int *address, int *aLength, int comm, int remoteSource) +{ + CommandMsg cmd; + memset(&cmd, 0, sizeof(CommandMsg)); + int status = MPI_ERR_OTHER; + MPI_Status signalStatus; + + if (remoteSource < 0) + { + remoteSource = MPI_ANY_SOURCE; + } + + if (comm == XDMF_DSM_INTRA_COMM) + { + status = MPI_Recv(&cmd, sizeof(CommandMsg), MPI_UNSIGNED_CHAR, remoteSource, XDMF_DSM_COMMAND_TAG, static_cast(this->Comm)->GetIntraComm(), &signalStatus); + } + else if (comm == XDMF_DSM_INTER_COMM) + { + status = MPI_Recv(&cmd, sizeof(CommandMsg), MPI_UNSIGNED_CHAR, remoteSource, XDMF_DSM_COMMAND_TAG, static_cast(this->Comm)->GetInterComm(), &signalStatus); + } + else + {//in this case the integer is probably a pointer to an MPI_Comm object + status = MPI_Recv(&cmd, sizeof(CommandMsg), MPI_UNSIGNED_CHAR, remoteSource, XDMF_DSM_COMMAND_TAG, comm, &signalStatus); + } + + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Error: Failed to receive command header"); + } + catch (XdmfError e) + { + throw e; + } + } + else + { + *opcode = cmd.Opcode; + *source = cmd.Source; + *address = cmd.Address; + *aLength = cmd.Length; + } +} + + +void +XdmfDSMBuffer::SendData(int dest, char * data, int aLength, int tag, int aAddress, int comm) +{ + int status; + if (comm == XDMF_DSM_INTRA_COMM) + { + status = MPI_Send(data, aLength, MPI_UNSIGNED_CHAR, dest, tag, static_cast(this->Comm)->GetIntraComm()); + } + else if (comm == XDMF_DSM_INTER_COMM) + { + status = MPI_Send(data, aLength, MPI_UNSIGNED_CHAR, dest, tag, static_cast(this->Comm)->GetInterComm()); + } + else + { + status = MPI_Send(data, aLength, MPI_UNSIGNED_CHAR, dest, tag, comm); + } + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Error: Failed to send data"); + } + catch (XdmfError e) + { + throw e; + } + } +} + +void +XdmfDSMBuffer::ReceiveData(int source, char * data, int aLength, int tag, int aAddress, int comm) +{ + int status; + MPI_Status signalStatus; + if (comm == XDMF_DSM_INTRA_COMM) + { + status = MPI_Recv(data, aLength, MPI_UNSIGNED_CHAR, source, tag, static_cast(this->Comm)->GetIntraComm(), &signalStatus); + } + else if (comm == XDMF_DSM_INTER_COMM) + { + status = MPI_Recv(data, aLength, MPI_UNSIGNED_CHAR, source, tag, static_cast(this->Comm)->GetInterComm(), &signalStatus); + } + else + { + status = MPI_Recv(data, aLength, MPI_UNSIGNED_CHAR, source, tag, comm, &signalStatus); + } + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Error: Failed to receive data"); + } + catch (XdmfError e) + { + throw e; + } + } +} + +void +XdmfDSMBuffer::SendAcknowledgment(int dest, int data, int tag, int comm) +{ + int status; + + if (comm == XDMF_DSM_INTRA_COMM) + { + status = MPI_Send(&data, sizeof(int), MPI_UNSIGNED_CHAR, dest, tag, static_cast(this->Comm)->GetIntraComm()); + } + else if (comm == XDMF_DSM_INTER_COMM) + { + status = MPI_Send(&data, sizeof(int), MPI_UNSIGNED_CHAR, dest, tag, static_cast(this->Comm)->GetInterComm()); + } + else + { + status = MPI_Send(&data, sizeof(int), MPI_UNSIGNED_CHAR, dest, tag, comm); + } + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Error: Failed to receive data"); + } + catch (XdmfError e) + { + throw e; + } + } +} + +void +XdmfDSMBuffer::ReceiveAcknowledgment(int source, int &data, int tag, int comm) +{ + int status; + MPI_Status signalStatus; + if (comm == XDMF_DSM_INTRA_COMM) + { + status = MPI_Recv(&data, sizeof(int), MPI_UNSIGNED_CHAR, source, tag, static_cast(this->Comm)->GetIntraComm(), &signalStatus); + } + else if (comm == XDMF_DSM_INTER_COMM) + { + status = MPI_Recv(&data, sizeof(int), MPI_UNSIGNED_CHAR, source, tag, static_cast(this->Comm)->GetInterComm(), &signalStatus); + } + else + { + status = MPI_Recv(&data, sizeof(int), MPI_UNSIGNED_CHAR, source, tag, comm, &signalStatus); + } + + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Error: Failed to receive data"); + } + catch (XdmfError e) + { + throw e; + } + } +} + +void +XdmfDSMBuffer::SendInfo() +{ + InfoMsg dsmInfo; + int status; + + memset(&dsmInfo, 0, sizeof(InfoMsg)); + dsmInfo.type = this->GetDsmType(); + dsmInfo.length = this->GetLength(); + dsmInfo.total_length = this->GetTotalLength(); + dsmInfo.block_length = this->GetBlockLength(); + dsmInfo.start_server_id = this->GetStartServerId(); + dsmInfo.end_server_id = this->GetEndServerId(); + if (this->Comm->GetId() == 0) + { + status = MPI_Send(&dsmInfo, sizeof(InfoMsg), MPI_UNSIGNED_CHAR, 0, XDMF_DSM_EXCHANGE_TAG, static_cast(this->Comm)->GetInterComm()); + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Error: Failed to send info"); + } + catch (XdmfError e) + { + throw e; + } + } + } + status = MPI_Barrier(this->Comm->GetIntraComm()); + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Error: Failed to send info"); + } + catch (XdmfError e) + { + throw e; + } + } +} + +void +XdmfDSMBuffer::ReceiveInfo() +{ + InfoMsg dsmInfo; + int status; + MPI_Status signalStatus; + + memset(&dsmInfo, 0, sizeof(InfoMsg)); + if (this->Comm->GetId() == 0) + { + status = MPI_Recv(&dsmInfo, sizeof(InfoMsg), MPI_UNSIGNED_CHAR, XDMF_DSM_ANY_SOURCE, XDMF_DSM_EXCHANGE_TAG, static_cast(this->Comm)->GetInterComm(), &signalStatus); + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Error: Failed to receive info"); + } + catch (XdmfError e) + { + throw e; + } + } + } + status = MPI_Bcast(&dsmInfo, sizeof(InfoMsg), MPI_UNSIGNED_CHAR, 0, static_cast(this->Comm)->GetIntraComm()); + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Error: Failed to broadcast info"); + } + catch (XdmfError e) + { + throw e; + } + } + this->SetDsmType(dsmInfo.type); + // We are a client so don't allocate anything but only set a virtual remote length + this->SetLength(dsmInfo.length); + this->TotalLength = dsmInfo.total_length; + this->SetBlockLength(dsmInfo.block_length); + this->StartServerId = dsmInfo.start_server_id; + this->EndServerId = dsmInfo.end_server_id; +} + +void +XdmfDSMBuffer::BroadcastComm(int *comm, int root) +{ + int status; + + status = MPI_Bcast(comm, sizeof(int), MPI_UNSIGNED_CHAR, root, this->Comm->GetIntraComm()); + if (status != MPI_SUCCESS) + { + try + { + XdmfError(XdmfError::FATAL, "Broadcast of Comm failed"); + } + catch (XdmfError e) + { + throw e; + } + } +} + +//redefined from H5FDBufferService + +void +XdmfDSMBuffer::BufferServiceLoop(int *returnOpcode) +{ + int op, status = XDMF_DSM_SUCCESS; + while (status == XDMF_DSM_SUCCESS) { + try { + status = this->BufferService(&op); + } + catch (XdmfError e) { + throw e; + } + if (returnOpcode) *returnOpcode = op; + if (op == XDMF_DSM_OPCODE_DONE) { + break; + } + } +} + + + +int +XdmfDSMBuffer::BufferService(int *returnOpcode) +{ + int opcode, who, status = XDMF_DSM_FAIL; + int aLength; + int address; + char *datap; + static int syncId = -1; + + if (this->CommChannel == XDMF_DSM_ANY_COMM) { + if (this->Comm->GetId() == 0) { + try { + this->ProbeCommandHeader(&this->CommChannel); + } + catch (XdmfError e) { + throw e; + } + } + try { + this->BroadcastComm(&this->CommChannel, 0); + } + catch (XdmfError e) { + throw e; + } + } + + + try { + this->ReceiveCommandHeader(&opcode, &who, &address, &aLength, this->CommChannel, syncId); + } + catch (XdmfError e) { + throw e; + } + + + + // connection is an ID for client or server, + // we can use the communicator ID interchangably, but if the architecture is altered - be careful + int communicatorId = this->CommChannel; + + switch(opcode) { + + // H5FD_DSM_OPCODE_PUT + case XDMF_DSM_OPCODE_PUT: + if (((unsigned int) aLength + address) > this->Length) { + try { + std::stringstream message; + message << "Length " << aLength << " too long for Address " << address << "\n" << + "Server Start = " << this->StartAddress << " End = " << this->EndAddress; + XdmfError::message(XdmfError::FATAL, message.str()); + } + catch (XdmfError e) { + throw e; + } + } + if ((datap = this->DataPointer) == NULL) { + try { + XdmfError::message(XdmfError::FATAL, "Null Data Pointer when trying to put data"); + } + catch (XdmfError e) { + throw e; + } + } + datap += address; + try { + this->ReceiveData(who, datap, aLength, XDMF_DSM_PUT_DATA_TAG, 0, this->CommChannel); + } + catch (XdmfError e) { + throw e; + } + break; + + // H5FD_DSM_OPCODE_GET + case XDMF_DSM_OPCODE_GET: + if (((unsigned int) aLength + address) > this->Length) { + try { + std::stringstream message; + message << "Length " << aLength << " too long for Address " << address << "\n" << + "Server Start = " << this->StartAddress << " End = " << this->EndAddress; + XdmfError::message(XdmfError::FATAL, message.str()); + } + catch (XdmfError e) { + throw e; + } + } + if ((datap = this->DataPointer) == NULL) { + try { + XdmfError::message(XdmfError::FATAL, "Null Data Pointer when trying to put data"); + } + catch (XdmfError e) { + throw e; + } + } + datap += address; + try { + this->SendData(who, datap, aLength, XDMF_DSM_GET_DATA_TAG, 0, this->CommChannel); + } + catch (XdmfError e) { + throw e; + } + break; + + // H5FD_DSM_LOCK_ACQUIRE + // Comes from client or server depending on communicator +// case XDMF_DSM_LOCK_ACQUIRE: + //is->SendAcknowledgment(who, -1, XDMF_DSM_CLIENT_ACK_TAG, this->CommChannel); +/* // wait for all processes to sync before doing anything + if (this->Comm->ChannelSynced(who, &syncId, communicatorId)) { + // only rank 0 actually handles the lock, the other ranks just mimic it later when they get an acknowledgement + if (this->Comm->GetId() == 0) { + if (this->BufferServiceInternals->BufferLock.Lock(communicatorId)) { + // notify all other server nodes - to update their local locks to match rank 0 + H5FDdsmInt32 numberOfRanks = this->Comm->GetIntraSize(); + for (H5FDdsmInt32 who = 1; who < numberOfRanks; ++who) { + this->SendAcknowledgment(who, communicatorId, H5FD_DSM_SERVER_ACK_TAG, H5FD_DSM_INTRA_COMM); + } + // notify the ranks that made the request + numberOfRanks = (communicatorId==H5FD_DSM_SERVER_ID) ? this->Comm->GetIntraSize() : this->Comm->GetInterSize(); + for (H5FDdsmInt32 who = 0; who < numberOfRanks; ++who) { + this->SendAcknowledgment(who, communicatorId, H5FD_DSM_CLIENT_ACK_TAG, this->CommChannel); + } + } + // we were not given the lock, so go back to listening for anyone + else { + this->CommChannel = H5FD_DSM_ANY_COMM; + // notify all other server nodes that lock request failed and to change communicator + H5FDdsmInt32 numberOfRanks = this->Comm->GetIntraSize(); + for (H5FDdsmInt32 who = 1; who < numberOfRanks; ++who) { + this->SendAcknowledgment(who, -1, H5FD_DSM_SERVER_ACK_TAG, H5FD_DSM_INTRA_COMM); + } + } + } + else { + // all server nodes need to update their local locks to match rank 0 + this->ReceiveAcknowledgment(0, communicatorId, H5FD_DSM_SERVER_ACK_TAG, H5FD_DSM_INTRA_COMM); + // the lock request failed, so we don't give the lock to the requestor + if (communicatorId == -1) { + this->CommChannel = H5FD_DSM_ANY_COMM; + } else { + this->BufferServiceInternals->BufferLock.Lock(communicatorId); + } + } + }*/ +// break; + + // H5FD_DSM_LOCK_RELEASE + // Comes from client or server depending on communicator +// case XDMF_DSM_LOCK_RELEASE: +/* // wait for all processes to sync before doing anything + if (this->Comm->ChannelSynced(who, &syncId, communicatorId)) { + // only rank 0 actually handles the lock, the other ranks just mimic it later when they get an acknowledgement + H5FDdsmInt32 newLockOwner = -1; + if (this->Comm->GetId() == 0) { + // When we release the lock, it may be passed straight to the next owner, + // if this happens, we must inform the other server nodes who the owner is + newLockOwner = this->BufferServiceInternals->BufferLock.Unlock(communicatorId); + H5FDdsmInt32 numberOfRanks = this->Comm->GetIntraSize(); + for (H5FDdsmInt32 who = 1; who < numberOfRanks; ++who) { + this->SendAcknowledgment(who, newLockOwner, H5FD_DSM_SERVER_ACK_TAG, H5FD_DSM_INTRA_COMM); + } + } else { + // all server nodes need to update their local locks to match rank 0 + this->ReceiveAcknowledgment(0, newLockOwner, H5FD_DSM_SERVER_ACK_TAG, H5FD_DSM_INTRA_COMM); + this->BufferServiceInternals->BufferLock.Unlock(communicatorId); + } + + // + // the lock has been released : if the client unlocked, wake up waiting server app thread + // note that a lock count decrease returns the same lock owner, so we don't trigger on that event + // + if (newLockOwner != communicatorId && communicatorId == H5FD_DSM_CLIENT_ID) { + // the address flag holds our unlock status (only treat it when received from client) + this->SignalUnlock(address, H5FD_DSM_FALSE); + } + // + // if it has been taken by another communicator/connection, do what's needed + // + if (newLockOwner == -1) { + this->CommChannel = H5FD_DSM_ANY_COMM; + H5FDdsmDebug("Lock released, Switched to " << H5FDdsmCommToString(this->CommChannel)); + } + else if (newLockOwner != communicatorId) { + this->CommChannel = newLockOwner; + H5FDdsmDebug("Lock retaken, Switched to " << H5FDdsmCommToString(this->CommChannel)); + if (this->Comm->GetId() != 0) { + newLockOwner = this->BufferServiceInternals->BufferLock.Lock(newLockOwner); + } + if (this->Comm->GetId() == 0) { + // notify the ranks that made the original lock request + H5FDdsmInt32 numberOfRanks = (newLockOwner == H5FD_DSM_SERVER_ID) ? this->Comm->GetIntraSize() : this->Comm->GetInterSize(); + for (H5FDdsmInt32 who = 0; who < numberOfRanks; ++who) { + this->SendAcknowledgment(who, newLockOwner, H5FD_DSM_CLIENT_ACK_TAG, this->CommChannel); + } + } + } + }*/ +// break; + + // H5FD_DSM_OPCODE_DONE + // Always received from server + case XDMF_DSM_OPCODE_DONE: + break; + + // DEFAULT + default : + try { + std::stringstream message; + message << "Error: Unknown Opcode " << opcode; + XdmfError::message(XdmfError::FATAL, message.str()); + } + catch (XdmfError e) { + throw e; + } + } + + if (returnOpcode) *returnOpcode = opcode; + return(XDMF_DSM_SUCCESS); +} + + +void +XdmfDSMBuffer::SendDone() +{ + try { + if (static_cast(this->Comm)->GetInterComm() == MPI_COMM_NULL)//this only stops the first core controlled by the server + { + for (int i = this->StartServerId; i < this->EndServerId; ++i) { + if (i != this->Comm->GetId()){ + this->SendCommandHeader(XDMF_DSM_OPCODE_DONE, i, 0, 0, XDMF_DSM_INTRA_COMM); + } + } + } + else + { + for (int i = this->StartServerId; i < this->EndServerId; ++i) { + if (i != this->Comm->GetId()){ + this->SendCommandHeader(XDMF_DSM_OPCODE_DONE, i, 0, 0, XDMF_DSM_INTER_COMM); + } + } + } + } + catch (XdmfError e) { + throw e; + } +} + +void +XdmfDSMBuffer::ProbeCommandHeader(int *comm)//used for finding a comm that has a waiting command, then sets the comm +{ + int status = XDMF_DSM_FAIL; + MPI_Status signalStatus; + + int flag; + MPI_Comm probeComm = static_cast(this->Comm)->GetIntraComm(); + + // Spin until a message is found on one of the communicators + while (status != XDMF_DSM_SUCCESS) { + status = MPI_Iprobe(XDMF_DSM_ANY_SOURCE, XDMF_DSM_ANY_TAG, probeComm, &flag, &signalStatus); + if (status != MPI_SUCCESS) + { + try { + XdmfError::message(XdmfError::FATAL, "Error: Failed to probe for command header"); + } + catch (XdmfError e) { + throw e; + } + } + if (flag) { + status = XDMF_DSM_SUCCESS; + } + else { + if (static_cast(this->Comm)->GetInterComm() != MPI_COMM_NULL) { + if (probeComm == static_cast(this->Comm)->GetIntraComm()) { + probeComm = static_cast(this->Comm)->GetInterComm(); + } + else { + probeComm = static_cast(this->Comm)->GetIntraComm(); + } + } + } + } + if (probeComm == static_cast(this->Comm)->GetInterComm()) { + *comm = XDMF_DSM_INTER_COMM; + } + else + { + *comm = XDMF_DSM_INTRA_COMM; + } + + probeComm = MPI_COMM_NULL; +} + + +int +XdmfDSMBuffer::AddressToId(int Address){ + int ServerId = XDMF_DSM_FAIL; + + switch(this->DsmType) { + case XDMF_DSM_TYPE_UNIFORM : + case XDMF_DSM_TYPE_UNIFORM_RANGE : + // All Servers have same length + // This finds out which server the address provided starts on + ServerId = this->StartServerId + (Address / this->Length); + if(ServerId > this->EndServerId ){ + try { + std::stringstream message; + message << "ServerId " << ServerId << " for Address " << Address << " is larger than EndServerId " << this->EndServerId; + XdmfError::message(XdmfError::FATAL, message.str()); + } + catch (XdmfError e) { + throw e; + } + } + break; + default : + // Not Implemented + try { + std::stringstream message; + message << "DsmType " << this->DsmType << " not yet implemented"; + XdmfError::message(XdmfError::FATAL, message.str()); + } + catch (XdmfError e) { + throw e; + } + break; + } + return(ServerId); +} + +void +XdmfDSMBuffer::GetAddressRangeForId(int Id, int *Start, int *End){ + switch(this->DsmType) { + case XDMF_DSM_TYPE_UNIFORM : + case XDMF_DSM_TYPE_UNIFORM_RANGE : + // All Servers have same length + // Start index is equal to the id inside the servers times the length of the block per server + // It is the starting index of the server's data block relative to the entire block + *Start = (Id - this->StartServerId) * this->Length; + // End index is simply the start index + the length of the server's data block. + // The range produced is the start of the server's data block to its end. + *End = *Start + Length - 1; + break; + default : + // Not Implemented + try { + std::stringstream message; + message << "DsmType " << this->DsmType << " not yet implemented"; + XdmfError::message(XdmfError::FATAL, message.str()); + } + catch (XdmfError e) { + throw e; + } + break; + } +} + +void +XdmfDSMBuffer::Get(long Address, long aLength, void *Data) +{ + int who, MyId = this->Comm->GetId(); + int astart, aend, len; + char *datap = (char *)Data; + + // While there is length left + while(aLength) + { + // Figure out what server core the address is located on + who = this->AddressToId(Address); + if(who == XDMF_DSM_FAIL){ + try { + XdmfError::message(XdmfError::FATAL, "Address Error"); + } + catch (XdmfError e) { + throw e; + } + } + // Get the start and end of the block listed + this->GetAddressRangeForId(who, &astart, &aend); + // Determine the amount of data to be written to that core + // Basically, it's how much data will fit from the starting point of the address to the end + len = std::min(aLength, aend - Address + 1); + // If the data is on the core running this code, then the put is simple + if(who == MyId){ + char *dp; + + dp = this->DataPointer; + dp += Address - this->StartAddress; + memcpy(datap, dp, len); + + }else{ + // Otherwise send it to the appropriate core to deal with + int status; + int dataComm = XDMF_DSM_INTRA_COMM; + if (this->Comm->GetInterComm() != MPI_COMM_NULL) { + dataComm = XDMF_DSM_INTER_COMM; + } + try { + this->SendCommandHeader(XDMF_DSM_OPCODE_GET, who, Address - astart, len, dataComm); + } + catch (XdmfError e) { + throw e; + } + try { + this->ReceiveData(who, datap, len, XDMF_DSM_GET_DATA_TAG, Address - astart, dataComm); + } + catch (XdmfError e) { + throw e; + } + } + // Shift all the numbers by the length of the data written + // Until aLength = 0 + aLength -= len; + Address += len; + datap += len; + } +} + +void +XdmfDSMBuffer::Put(long Address, long aLength, const void *Data){ + int who, MyId = this->Comm->GetId(); + int astart, aend, len; + char *datap = (char *)Data; + + // While there is length left + while(aLength){ + // Figure out what server core the address is located on + who = this->AddressToId(Address); + if(who == XDMF_DSM_FAIL){ + try { + XdmfError::message(XdmfError::FATAL, "Address Error"); + } + catch (XdmfError e) { + throw e; + } + } + // Get the start and end of the block listed + this->GetAddressRangeForId(who, &astart, &aend); + // Determine the amount of data to be written to that core + // Basically, it's how much data will fit from the starting point of the address to the end + len = std::min(aLength, aend - Address + 1); + // If the data is on the core running this code, then the put is simple + if(who == MyId){ + char *dp; + + dp = this->DataPointer; + dp += Address - this->StartAddress; + memcpy(dp, datap, len); + + }else{ + // Otherwise send it to the appropriate core to deal with + int status; + int dataComm = XDMF_DSM_INTRA_COMM; + if (this->Comm->GetInterComm() != MPI_COMM_NULL) { + dataComm = XDMF_DSM_INTER_COMM; + } + try { + this->SendCommandHeader(XDMF_DSM_OPCODE_PUT, who, Address - astart, len, dataComm); + } + catch (XdmfError e) { + throw e; + } + try { + this->SendData(who, datap, len, XDMF_DSM_PUT_DATA_TAG, Address - astart, dataComm); + } + catch (XdmfError e) { + throw e; + } + } + // Shift all the numbers by the length of the data written + // Until aLength = 0 + aLength -= len; + Address += len; + datap += len; + } +} diff --git a/core/XdmfDSMBuffer.hpp b/core/XdmfDSMBuffer.hpp new file mode 100644 index 00000000..bce8bc71 --- /dev/null +++ b/core/XdmfDSMBuffer.hpp @@ -0,0 +1,1405 @@ +/*****************************************************************************/ +/* XDMF */ +/* eXtensible Data Model and Format */ +/* */ +/* Id : XdmfDSMBuffer.hpp */ +/* */ +/* Author: */ +/* Andrew Burns */ +/* andrew.j.burns2@us.army.mil */ +/* US Army Research Laboratory */ +/* Aberdeen Proving Ground, MD */ +/* */ +/* Copyright @ 2013 US Army Research Laboratory */ +/* All Rights Reserved */ +/* See Copyright.txt for details */ +/* */ +/* This software is distributed WITHOUT ANY WARRANTY; without */ +/* even the implied warranty of MERCHANTABILITY or FITNESS */ +/* FOR A PARTICULAR PURPOSE. See the above copyright notice */ +/* for more information. */ +/* */ +/*****************************************************************************/ + +/*========================================================================= + This code is derived from an earlier work and is distributed + with permission from, and thanks to ... +=========================================================================*/ + +/*========================================================================= + + Project : H5FDdsm + Module : H5FDdsmBufferService.h, H5FDdsmBuffer.h + + Authors: + John Biddiscombe Jerome Soumagne + biddisco@cscs.ch soumagne@cscs.ch + + Copyright (C) CSCS - Swiss National Supercomputing Centre. + You may use modify and and distribute this code freely providing + 1) This copyright notice appears on all copies of source code + 2) An acknowledgment appears with any substantial usage of the code + 3) If this code is contributed to any other open source project, it + must not be reformatted such that the indentation, bracketing or + overall style is modified significantly. + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + + This work has received funding from the European Community's Seventh + Framework Programme (FP7/2007-2013) under grant agreement 225967 âxtMuSEâOB +=========================================================================*/ + +#ifndef XDMFDSMBUFFER_HPP_ +#define XDMFDSMBUFFER_HPP_ + +// Forward Declarations + +// Includes +#include +#include +#include + +#define XDMF_DSM_DEFAULT_TAG 0x80 +#define XDMF_DSM_COMMAND_TAG 0x81 +#define XDMF_DSM_SERVER_ACK_TAG 0x82 +#define XDMF_DSM_CLIENT_ACK_TAG 0x83 +#define XDMF_DSM_PUT_DATA_TAG 0x84 +#define XDMF_DSM_GET_DATA_TAG 0x85 +#define XDMF_DSM_EXCHANGE_TAG 0x86 + +#define XDMF_DSM_ANY_TAG -1 +#define XDMF_DSM_ANY_SOURCE -2 + +#define XDMF_DSM_TYPE_UNIFORM 0 +#define XDMF_DSM_TYPE_UNIFORM_RANGE 1 +#define XDMF_DSM_TYPE_MIXED 2 +#define XDMF_DSM_TYPE_BLOCK_CYCLIC 3 +#define XDMF_DSM_TYPE_BLOCK_RANDOM 4 + +#define XDMF_DSM_DEFAULT_LENGTH 10000 +#define XDMF_DSM_DEFAULT_BLOCK_LENGTH 1024 +#define XDMF_DSM_ALIGNMENT 4096 + +#define XDMF_DSM_OPCODE_PUT 0x01 +#define XDMF_DSM_OPCODE_GET 0x02 + +#define XDMF_DSM_LOCK_ACQUIRE 0x03 +#define XDMF_DSM_LOCK_RELEASE 0x05 + +#define XDMF_DSM_OPCODE_DONE 0xFF + +#define XDMF_DSM_SUCCESS 1 +#define XDMF_DSM_FAIL -1 + + +/** + * @brief Controls the data buffer for DSM. + * + * XdmfDSMBuffer takes the place of the H5FDdsmBuffer defined in H5FD. + * It is primarily for allowing the XdmfDSM to interact with HDF5 dsm without threads. + */ +class XDMFCORE_EXPORT XdmfDSMBuffer { + +public: + + XdmfDSMBuffer(); + ~XdmfDSMBuffer(); + + /** + * Find the Id of the core that the provided address resides on. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline AddressToId + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline AddressToId + * + * @param Address The address to be found + * @return The id of the core that the address resides on + */ + int AddressToId(int Address); + + /** + * Broadcasts the provided comm from the specified core to all other cores. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline workerComm + * @until cfree + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline broadcastComm + * @until BroadcastComm + * @skip Section + * @skipline } + * + * Python: + * Unusable in python unless an object of a cpointer type is passed via wrapped code + * + * + * + * @param comm The communicator to be transmitted + * @param root The core that the broadcast is originating from + */ + void BroadcastComm(int *comm, int root); + + /** + * One iteration of the service loop. + * It is not advised to use this function manually. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline workerComm + * @until cfree + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skip RecieveInfo + * @skipline if + * @until BufferService + * @skipline } + * @until SendCommandHeader + * @skipline } + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skip ServerIds + * @skip SendInfo + * @skipline if + * @skipline serviceOut + * @skipline if + * @skipline SendCommandHeader + * + * @param returnOpcode A variable that will hold the code at the end of the loop + * @return If the iteration executed without problem returns XDMF_DSM_SUCCESS + */ + int BufferService(int *returnOpcode = 0); + + /** + * Starts up the service loop. + * The loop then executes until the op code "Done" is sent to this core. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline workerComm + * @until cfree + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skip RecieveInfo + * @skipline if + * @until BufferServiceLoop + * @skipline } + * @until SendCommandHeader + * @skipline } + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skip ServerIds + * @skip SendInfo + * @skipline if + * @skipline BufferServiceLoop + * @skipline if + * @skipline SendCommandHeader + * + * @param returnOpcode A variable that will hold the code at the end of the loop + */ + void BufferServiceLoop(int *returnOpcode = 0); + + /** + * Configures the Buffer to match the configuration details provided. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline ConfigureUniform + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline ConfigureUniform + * + * @param Comm The communicator that will be handling the communications for the DSM + * @param Length The length of the data buffer on server cores + * @param StartId The id that the server cores will start on, if set to -1 then it will default to 0 + * @param EndId The id that the server cores will end on, if set to -1 then it will be the last core in the communicator + * @param aBlockLength The block size of the data buffer, 0 is no blocking + * @param random Whether the assignment is random or cyclic. Default is cyclic + */ + void ConfigureUniform(XdmfDSMCommMPI *Comm, long Length, + int StartId = -1, int EndId = -1, + long aBlockLength = 0, bool random = false); + + /** + * Gets data from the server cores. + * It is not advised to use this function manually. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline core0StartAddress + * @until GetAddressRange + * @skipline dsmData + * @skipline if + * @until { + * @skipline Get + * @until message + * Skipline } + * @skip Section + * @skipline } + * + * + * Python: + * Unusable in python unless an object of a cpointer type is passed via wrapped code + * Use the XdmfHDF5WriterDSM for this functionality + * + * + * + * @param Address The starting address of the data retrieved + * @param aLength The length of the data to be retrieved + * @param Data A pointer in which the data is to be stored after retieval + */ + void Get(long Address, long aLength, void *Data); + + /** + * Gets the starting address and ending address for the core of the provided Id. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline core0StartAddress + * @until GetAddressRange + * @skip Section + * @skipline } + * + * Python: + * Unusable in python unless an object of a cpointer type is passed via wrapped code + * + * + * + * @param Id The core for which the start and end address are to be found + * @param Start A pointer in which the start address is to be placed + * @param End A pointer in which the end address is to be placed + */ + void GetAddressRangeForId(int Id, int *Start, int *End); + + /** + * Gets the size of the blocks for the data buffer. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline GetBlockLength + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline GetBlockLength + * + * @return The size of the blocks in the DSM buffer + */ + long GetBlockLength(); + + /** + * Gets the Comm being used to facilitate the communications for the DSM + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline XdmfDSMCommMPI + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline exampleDSMComm + * + * @return The Comm controlling the communications for the DSM + */ + XdmfDSMCommMPI * GetComm(); + + /** + * Gets the data pointer that the buffer controls. Should be NULL on non-server cores. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline GetDataPointer + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline GetDataPointer + * + * @return The buffer's internal data pointer. + */ + char * GetDataPointer(); + + /** + * Gets the DSM type currently being used. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline GetDsmType + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline GetDsmType + * + * @return The integer representation of the DSM type + */ + int GetDsmType(); + + /** + * Gets the address at the end of DSM buffer for this buffer. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline GetEndAddress + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline GetEndAddress + * + * @return The end address of the DSM buffer + */ + int GetEndAddress(); + + /** + * Gets the id of the last of the server cores that handle the DSM buffer. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline GetEndServerId + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline GetEndServerId + * + * @return The id of the last core that serves as a DSM server + */ + int GetEndServerId(); + + /** + * Gets if the Buffer is connected to an intercomm + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline GetIsConnected + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline GetIsConnected + * + * @return Whether the buffer is connected + */ + bool GetIsConnected(); + + /** + * Gets if the buffer is a DSM server. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline GetIsServer + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline GetIsServer + * + * @return If the Buffer is a DSM server + */ + bool GetIsServer(); + + /** + * The length of the buffer per core. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline GetLength + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline GetLength + * + * @return The length of the data buffer per core + */ + long GetLength(); + + /** + * Gets the address at the beginning of the DSM buffer for this buffer. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline GetStartAddress + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline GetStartAddress + * + * @return The beginning address of the DSM buffer + */ + int GetStartAddress(); + + /** + * Gets the id of the first of the server cores that handle the DSM buffer. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline GetStartServerId + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline GetStartServerId + * + * @return The id of the first core that serves as a DSM server + */ + int GetStartServerId(); + + /** + * The total length of the data buffer when combining the buffers in all cores. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline GetTotalLength + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline GetTotalLength + * + * @return The total length of the data buffer + */ + long GetTotalLength(); + + /** + * Probes inter and intra comms until a command is found. + * Then sets the comm that the command was found on to the provided variable + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skip ConfigureUniform + * @skipline id + * @until ReceiveCommandHeader + * @skipline } + * @skip Section + * @skipline } + * + * Python: + * Unusable in python unless an object of a cpointer type is passed via wrapped code + * + * + * + * @param comm a pointer to the variable that the integer code for the comm is placed in + */ + void ProbeCommandHeader(int *comm); + + /** + * Puts data to the server cores. + * It is not advised to use this function manually. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline core0StartAddress + * @until GetAddressRange + * @skipline dsmData + * @until Put + * @skipline } + * @until message + * Skipline } + * @skip Section + * @skipline } + * + * Python: + * Unusable in python unless an object of a cpointer type is passed via wrapped code + * Use the XdmfHDF5WriterDSM for this functionality + * + * + * + * @param Address The starting address that the data will be placed at + * @param aLength The length of the data to be sent + * @param Data A pointer to the data to be sent + */ + void Put(long Address, long aLength, const void *Data); + + /** + * Recieves an integer as an acknowledgement from the specified core. + * It is not advised to use this function manually. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skip ReceiveData + * @skipline id + * @until ReceiveAcknowledgment + * @skipline } + * @skip Section + * @skipline } + * + * Python: + * Unusable in python unless an object of a cpointer type is passed via wrapped code + * + * + * + * @param source The core to recieve from + * @param data The integer that the recieved data will be stored in + * @param tag The tag associated with the acknowledgement + * @param comm The comm over which the acknowldedgement will occur + */ + void ReceiveAcknowledgment(int source, int &data, int tag, int comm); + + /** + * Recieves command data from either a specified or unspecified source. + * If remoteSource is not provided any source currently sending a command is used. + * It is not advised to use this function manually. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skip ConfigureUniform + * @skipline id + * @until ReceiveCommandHeader + * @skipline } + * @skip Section + * @skipline } + * + * Python: + * Unusable in python unless an object of a cpointer type is passed via wrapped code + * + * + * + * @param opcode A pointer to the location where the code associated with the operation will be stored + * @param source A pointer to the location where the index of the source core will be stored (will be the same as remoteSource if provided) + * @param address A pointer to the location where the address specified by the command will be stored + * @param aLength A pointer to the location where the length of the data specified by the command will be stored + * @param comm The communicator over which the transmission will occur + * @param remoteSource If provided, the core being recieved from + */ + void ReceiveCommandHeader(int *opcode, int *source, int *address, int *aLength, int comm, int remoteSource = -1); + + /** + * Recieves data from a specific core and stores it in a pointer. + * It is not advised to use this function manually. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skip ReceiveCommandHeader + * @skipline id + * @until ReceiveData + * @skipline } + * @skip Section + * @skipline } + * + * Python: + * Unusable in python unless an object of a cpointer type is passed via wrapped code + * + * + * + * @param source The core data will be recieved from + * @param data The pointer where the recieved data will be stored + * @param aLength The length of the data transmitted + * @param aAddress The location where the data will be written on the data buffer + * @param comm The comunicator over which the data transfer will occur + */ + void ReceiveData(int source, char * data, int aLength, int tag, int aAddress, int comm); + + /** + * With the Comm with ID 0 recieve information + * about a server from another core on the intercomm. + * Used to recieve server data from different managers. + * It is not advised to use this function manually. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline workerComm + * @until cfree + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline readComm + * @until cfree + * @skipline if + * @until workerComm + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline ServerIds + * @until (workerComm) + * + */ + void ReceiveInfo(); + + /** + * Sends an integer as an acknowledgement to the specified core. + * It is not advised to use this function manually. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skip ReceiveData + * @skipline id + * @until ReceiveAcknowledgment + * @skipline } + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skip SendData + * @skipline if + * @skipline SendAcknowledgment + * + * @param dest The core to send to + * @param data The integer to send + * @param tag The tag associated with the acknowledgement + * @param comm The comm over which the acknowldedgement will occur + */ + void SendAcknowledgment(int dest, int data, int tag, int comm); + + /** + * Sends command data to a specified core. + * It is not advised to use this function manually. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skip ConfigureUniform + * @skipline id + * @until ReceiveCommandHeader + * @skipline } + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skip ConfigureUniform + * @skipline if + * @skipline SendCommandHeader + * + * @param opcode The code for the command to be sent + * @param dest The core that the command is to be sent to + * @param address The address to be referenced by the command + * @param aLength The length of the data to be used by the command + * @param comm The communicator over which the transmission will occur + */ + void SendCommandHeader(int opcode, int dest, int address, int aLength, int comm); + + /** + * Sends data from a pointer to a specified core. + * It is not advised to use this function manually. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skip ReceiveCommandHeader + * @skipline id + * @until ReceiveData + * @skipline } + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skip SendCommandHeader + * @skipline if + * @skipline SendData + * + * @param dest The core that the data will be sent to + * @param data A pointer to the location of the data being sent + * @param aLength The length of the data being sent + * @param aAddress The address on the recieveing core's buffer that the data is to be placed in + * @param comm The communicator over which the data transfer will take place + */ + void SendData(int dest, char * data, int aLength, int tag, int aAddress, int comm); + + /** + * Ends the service loop server cores associated with this buffer + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline closeFile + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline SendDone + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline Section + * @skipline if + * @skipline GetServerBuffer + * @skipline SendDone + * + */ + void SendDone(); + + /** + * From the Comm with ID 0 send out information + * about the server to another core on the intercomm. + * Used to send server data to different managers. + * It is not advised to use this function manually. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline workerComm + * @until cfree + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline readComm + * @until cfree + * @skipline if + * @until workerComm + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline ServerIds + * @until (workerComm) + * + */ + void SendInfo(); + + /** + * Sets the DSM type to the provided type. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline SetDsmType + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline SetDsmType + * + * @param newDsmType The Dsm type that the buffer will be changed to + */ + void SetDsmType(int newDsmType); + + /** + * Sets the size of the blocks used in the data buffer. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline SetBlockLength + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline SetBlockLength + * + * @param newBlock The new block size to be used + */ + void SetBlockLength(long newBlock); + + /** + * Sets the Comm to be used to facilitate the communications for the DSM + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline XdmfDSMCommMPI + * @skipline SetComm + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline exampleDSMComm + * @skipline SetComm + * + * @param newComm The communicator that is to be used by the DSM + */ + void SetComm(XdmfDSMCommMPI * newComm); + + /** + * Sets the Buffer's connection status. Used if the XdmfDSMCommMPI is connected or disconnected manually. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline SetIsConnected + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline GetIsConnected + * @skipline SetIsConnected + * + * @param newStatus The new connection status + */ + void SetIsConnected(bool newStatus); + + /** + * Sets whether the buffer is a DSM server. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline SetIsServer + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline GetIsServer + * @skipline SetIsServer + * + * @param newIsServer Whether the buffer is to be a DSM server or not + */ + void SetIsServer(bool newIsServer); + +protected: + + +private: + + void SetLength(long aLength); + + class CommandMsg; + class InfoMsg; + + bool IsServer; + + int EndAddress; + int StartAddress; + + int StartServerId; + int EndServerId; + + unsigned int Length; + unsigned int TotalLength; + unsigned int BlockLength; + + XdmfDSMCommMPI *Comm; + + char *DataPointer; + + int DsmType; + + int CommChannel; + bool IsConnected; +}; + +#endif /* XDMFDSMBUFFER_HPP_ */ + diff --git a/core/XdmfDSMCommMPI.cpp b/core/XdmfDSMCommMPI.cpp new file mode 100644 index 00000000..64a16322 --- /dev/null +++ b/core/XdmfDSMCommMPI.cpp @@ -0,0 +1,405 @@ +/*****************************************************************************/ +/* XDMF */ +/* eXtensible Data Model and Format */ +/* */ +/* Id : XdmfDSMCommMPI.cpp */ +/* */ +/* Author: */ +/* Andrew Burns */ +/* andrew.j.burns2@us.army.mil */ +/* US Army Research Laboratory */ +/* Aberdeen Proving Ground, MD */ +/* */ +/* Copyright @ 2013 US Army Research Laboratory */ +/* All Rights Reserved */ +/* See Copyright.txt for details */ +/* */ +/* This software is distributed WITHOUT ANY WARRANTY; without */ +/* even the implied warranty of MERCHANTABILITY or FITNESS */ +/* FOR A PARTICULAR PURPOSE. See the above copyright notice */ +/* for more information. */ +/* */ +/*****************************************************************************/ + +/*========================================================================= + This code is derived from an earlier work and is distributed + with permission from, and thanks to ... +=========================================================================*/ + +/*============================================================================ + + Project : H5FDdsm + Module : H5FDdsmCommMpi.cxx + + Authors: + John Biddiscombe Jerome Soumagne + biddisco@cscs.ch soumagne@cscs.ch + + Copyright (C) CSCS - Swiss National Supercomputing Centre. + You may use modify and and distribute this code freely providing + 1) This copyright notice appears on all copies of source code + 2) An acknowledgment appears with any substantial usage of the code + 3) If this code is contributed to any other open source project, it + must not be reformatted such that the indentation, bracketing or + overall style is modified significantly. + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + + This work has received funding from the European Community's Seventh + Framework Programme (FP7/2007-2013) under grant agreement 225967 âxtMuSEâOC + +============================================================================*/ + +#include +#include +#include +#include + +XdmfDSMCommMPI::XdmfDSMCommMPI() +{ + IntraComm = MPI_COMM_NULL; + Id = -1; + IntraSize = -1; + InterComm = MPI_COMM_NULL; + InterSize = -1; + SetDsmMasterHostName(""); + InterCommType = XDMF_DSM_COMM_MPI; +} + +XdmfDSMCommMPI::~XdmfDSMCommMPI() +{ + if (InterComm != MPI_COMM_NULL) + { + int status = MPI_Comm_free(&InterComm); + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Failed to free intercomm Comm"); + } + catch (XdmfError e) + { + throw e; + } + } + } + if (IntraComm != MPI_COMM_NULL) + { + int status = MPI_Comm_free(&IntraComm); + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Failed to free intercomm Comm"); + } + catch (XdmfError e) + { + throw e; + } + } + } +} + +void +XdmfDSMCommMPI::SetDsmMasterHostName(const char *hostName) +{ + strcpy(DsmMasterHostName, hostName); +} + +char * +XdmfDSMCommMPI::GetDsmMasterHostName() +{ + return DsmMasterHostName; +} + +int +XdmfDSMCommMPI::GetId() +{ + return this->Id; +} + +int +XdmfDSMCommMPI::GetIntraSize() +{ + return this->IntraSize; +} + +int +XdmfDSMCommMPI::GetInterSize() +{ + return this->InterSize; +} + +int +XdmfDSMCommMPI::GetInterCommType() +{ + return this->InterCommType; +} + +void +XdmfDSMCommMPI::Init() +{ + int size, rank; + if (MPI_Comm_size(this->IntraComm, &size) != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Failed to initialize size"); + } + catch (XdmfError e) + { + throw e; + } + } + if (MPI_Comm_rank(this->IntraComm, &rank) != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Failed to initialize rank"); + } + catch (XdmfError e) + { + throw e; + } + } + + this->Id = rank; + this->IntraSize = size; +} + +//possibly use setenv/unsetenv/getenv to pass dsm port +//http://linux.die.net/man/3/setenv + +void +XdmfDSMCommMPI::OpenPort() +{ + if (Id == 0) + { + int status = MPI_Open_port(MPI_INFO_NULL, DsmMasterHostName); + if (status != MPI_SUCCESS) + { + try + { + std::string message = "Failed to open port "; + message = message + DsmMasterHostName; + XdmfError::message(XdmfError::FATAL, message); + } + catch (XdmfError e) + { + throw e; + } + } + } +} + +void +XdmfDSMCommMPI::ClosePort() +{ + if (Id == 0) + { + int status; + status = MPI_Open_port(MPI_INFO_NULL, DsmMasterHostName); + if (status != MPI_SUCCESS) + { + try + { + std::string message = "Failed to close port "; + message = message + DsmMasterHostName; + XdmfError::message(XdmfError::FATAL, message); + } + catch (XdmfError e) + { + throw e; + } + } + } +} + +void +XdmfDSMCommMPI::Accept() +{ + int status = MPI_Comm_accept(DsmMasterHostName, MPI_INFO_NULL, 0, IntraComm, &InterComm); + if (status != MPI_SUCCESS) + { + try + { + std::string message = "Failed to accept port "; + message = message + DsmMasterHostName; + XdmfError::message(XdmfError::FATAL, message); + } + catch (XdmfError e) + { + throw e; + } + } + else + { + MPI_Comm_remote_size(InterComm, &InterSize); + } +} + +int +XdmfDSMCommMPI::Connect() +{ + if (InterComm != MPI_COMM_NULL) + { + // If the intercomm already exists, no need to connect + // If you want to reset the intercomm, set it to MPI_COMM_NULL before calling this + // using either SetInterComm or Disconnect + return MPI_SUCCESS; + } + else + { + MPI_Errhandler_set(IntraComm, MPI_ERRORS_RETURN); + int status = MPI_Comm_connect(DsmMasterHostName, MPI_INFO_NULL, 0, IntraComm, &InterComm); + MPI_Errhandler_set(IntraComm, MPI_ERRORS_ARE_FATAL); + if (status != MPI_SUCCESS) + { + try + { + std::string message = "Failed to connect to port "; + message = message + DsmMasterHostName; + XdmfError::message(XdmfError::FATAL, message); + } + catch (XdmfError e) + { + throw e; + } + } + else + { + status = MPI_Comm_remote_size(InterComm, &InterSize); + return MPI_SUCCESS; + } + } + return MPI_SUCCESS; +} + +void +XdmfDSMCommMPI::Disconnect() +{ + if (InterComm != MPI_COMM_NULL) + { + int status = MPI_Comm_free(&InterComm); + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Failed to disconnect Comm"); + } + catch (XdmfError e) + { + throw e; + } + } + } + InterComm = MPI_COMM_NULL; +} + +void +XdmfDSMCommMPI::DupComm(MPI_Comm comm) +{ + if (IntraComm != comm) + { + int status; + if (IntraComm != MPI_COMM_NULL) + { + status = MPI_Comm_free(&IntraComm); + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Failed to disconnect Comm"); + } + catch (XdmfError e) + { + throw e; + } + } + } + if (comm != MPI_COMM_NULL) + { + status = MPI_Comm_dup(comm, &IntraComm); + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Failed to duplicate Comm"); + } + catch (XdmfError e) + { + throw e; + } + } + else + { + status = MPI_Comm_size(IntraComm, &IntraSize); + status = MPI_Comm_rank(IntraComm, &Id); + } + } + } +} + +void +XdmfDSMCommMPI::DupInterComm(MPI_Comm comm) +{ + if (InterComm != comm) + { + int status; + if (InterComm != MPI_COMM_NULL) + { + status = MPI_Comm_free(&InterComm); + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Failed to disconnect Comm"); + } + catch (XdmfError e) + { + throw e; + } + } + } + if (comm != MPI_COMM_NULL) + { + status = MPI_Comm_dup(comm, &InterComm); + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Failed to duplicate Comm"); + } + catch (XdmfError e) + { + throw e; + } + } + else + { + status = MPI_Comm_size(InterComm, &InterSize); + if (status != MPI_SUCCESS) + { + MPI_Comm_remote_size(InterComm, &InterSize); + } + } + } + else + { + InterSize = -1; + } + } +} + +MPI_Comm +XdmfDSMCommMPI::GetInterComm() +{ + return InterComm; +} + +MPI_Comm +XdmfDSMCommMPI::GetIntraComm() +{ + return IntraComm; +} diff --git a/core/XdmfDSMCommMPI.hpp b/core/XdmfDSMCommMPI.hpp new file mode 100644 index 00000000..784190d8 --- /dev/null +++ b/core/XdmfDSMCommMPI.hpp @@ -0,0 +1,669 @@ +/*****************************************************************************/ +/* XDMF */ +/* eXtensible Data Model and Format */ +/* */ +/* Id : XdmfDSMCommMPI.hpp */ +/* */ +/* Author: */ +/* Andrew Burns */ +/* andrew.j.burns2@us.army.mil */ +/* US Army Research Laboratory */ +/* Aberdeen Proving Ground, MD */ +/* */ +/* Copyright @ 2013 US Army Research Laboratory */ +/* All Rights Reserved */ +/* See Copyright.txt for details */ +/* */ +/* This software is distributed WITHOUT ANY WARRANTY; without */ +/* even the implied warranty of MERCHANTABILITY or FITNESS */ +/* FOR A PARTICULAR PURPOSE. See the above copyright notice */ +/* for more information. */ +/* */ +/*****************************************************************************/ + +#ifndef XDMFDSMCOMMMPI_HPP_ +#define XDMFDSMCOMMMPI_HPP_ + +// Forward Declarations + +#define XDMF_DSM_COMM_MPI 0x11 + +#define XDMF_DSM_INTRA_COMM 0x00 +#define XDMF_DSM_INTER_COMM 0x01 +#define XDMF_DSM_ANY_COMM 0x02 + +#define XDMF_DSM_SERVER_ID XDMF_DSM_INTRA_COMM +#define XDMF_DSM_CLIENT_ID XDMF_DSM_INTER_COMM +#define XDMF_DSM_NUM_CONNECTION_IDS 0x02 + +// Includes +#include + +#include + +/** + * @brief Holds communicators for interacting with H5FD dsm. + * + * XdmfDSMCommMPI takes the place of the H5FDdsmComm defined in H5FD. + * It provides more access to the the intra and inter communicators. + * It is primarily for allowing the XdmfDSM to interact with HDF5 dsm without threads. + */ +class XDMFCORE_EXPORT XdmfDSMCommMPI { + +public: + + XdmfDSMCommMPI(); + ~XdmfDSMCommMPI(); + + /** + * Accepts connections to the port currently named by DsmMasterHostName. Called on server side, accepts from core 0. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline XdmfDSMCommMPI + * @skipline connectingGroup + * @until false + * @skipline } + * @skipline (!connectingGroup + * @until } + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline exampleDSMComm + * @skipline connectingGroup + * @until ClosePort + * + */ + void Accept(); + + /** + * Closes the port currently named by DsmMasterHostName. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline XdmfDSMCommMPI + * @skipline connectingGroup + * @until False + * @skipline } + * @skipline (!connectingGroup + * @until } + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline exampleDSMComm + * @skipline connectingGroup + * @until ClosePort + * + */ + void ClosePort(); + + /** + * If core ID is 0 then attempts to connect to the port currently named by DsmMasterHostName + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline XdmfDSMCommMPI + * @skipline connectingGroup + * @until false + * @skipline } + * @skipline (connectingGroup + * @until } + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline exampleDSMComm + * @skipline connectingGroup + * @until False + * @skipline (connectingGroup + * @until Disconnect + * + * @return Whether the connection was successful or not + */ + int Connect(); + + /** + * Disconnects the intercomm if not in static intercomm mode. Then sets the intercomm to MPI_COMM_NULL. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline XdmfDSMCommMPI + * @skipline connectingGroup + * @until false + * @skipline } + * @skipline (connectingGroup + * @until } + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline exampleDSMComm + * @skipline connectingGroup + * @until False + * @skipline (connectingGroup + * @until Disconnect + * + */ + void Disconnect(); + + /** + * Sets the IntraComm to the provided comm by duplicating it. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline XdmfDSMCommMPI + * @skipline DupComm + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline exampleDSMComm + * @skipline GetIntraComm + * @skipline DupComm + * + * @param comm The comm to be used as the IntraComm + */ + void DupComm(MPI_Comm comm); + + /** + * Sets the intercomm to the communicator provided by duplicating it. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline XdmfDSMCommMPI + * @skipline DupInterComm + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline Section + * @skipline exampleDSMComm + * @skipline GetInterComm + * @skipline DupInterComm + * + * @param comm The comm to be used as the intercomm + */ + void DupInterComm(MPI_Comm comm); + + /** + * Gets the port name that will be connected to when Connect/Accept is called. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline XdmfDSMCommMPI + * @skipline connectingGroup + * @until false + * @skipline } + * @skipline (!connectingGroup + * @until } + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline exampleDSMComm + * @skipline connectingGroup + * @until ClosePort + * + * @return a pointer to the character string that specifies the port + */ + char * GetDsmMasterHostName(); + + /** + * Gets the Id with regards to the IntraComm. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline XdmfDSMCommMPI + * @skipline GetId + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline exampleDSMComm + * @skipline GetId + * + * @return The Id of the core with regards to the IntraComm + */ + int GetId(); + + /** + * Gets the communicator that is currently functioning as the intercomm. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline XdmfDSMCommMPI + * @skipline GetInterComm + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline Section + * @skipline exampleDSMComm + * @skipline GetInterComm + * + * @return The communicator currently serving as the intercomm. + */ + MPI_Comm GetInterComm(); + + /** + * Gets the type of the InterComm. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline XdmfDSMCommMPI + * @skipline GetInterCommType + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline exampleDSMComm + * @skipline GetInterCommType + * + * @return An integer representation of the InterComm's type + */ + int GetInterCommType(); + + /** + * Gets the number of cores contained in the InterComm. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline XdmfDSMCommMPI + * @skipline GetInterSize + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline exampleDSMComm + * @skipline GetInterSize + * + * @return The number of cores in the InterComm + */ + int GetInterSize(); + + /** + * Gets the communicator that is currently functioning as the intracomm. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline XdmfDSMCommMPI + * @skipline GetIntraComm + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline exampleDSMComm + * @skipline GetIntraComm + * + * @return The communicator currently serving as the intracomm. + */ + MPI_Comm GetIntraComm(); + + /** + * Gets the number of cores contained in the IntraComm. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline XdmfDSMCommMPI + * @skipline GetIntraSize + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline exampleDSMComm + * @skipline GetIntraSize + * + * @return The number of cores in the IntraComm + */ + int GetIntraSize(); + + /** + * Initializes the Intracomm rank and size to the associated variables in the internal structure + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline XdmfDSMCommMPI + * @skipline Init + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline exampleDSMComm + * @skipline Init + * + */ + void Init(); + + /** + * Opens a port and stores the port name in DsmMasterHostName. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline XdmfDSMCommMPI + * @skipline connectingGroup + * @until false + * @skipline } + * @skipline (!connectingGroup + * @until } + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline exampleDSMComm + * @skipline connectingGroup + * @until ClosePort + * + */ + void OpenPort(); + + /** + * Sets the port name that will be connected to when Connect/Accept is called. + * Data is copied, so the provided string is not modified. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline XdmfDSMCommMPI + * @skipline connectingGroup + * @until false + * @skipline } + * @skipline (connectingGroup + * @until } + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline exampleDSMComm + * @skipline connectingGroup + * @until False + * @skipline (connectingGroup + * @until Disconnect + * + * @param hostName a pointer to the character string that specifies the port + */ + void SetDsmMasterHostName(const char *hostName); + +protected: + + +private: + MPI_Comm IntraComm; + int Id; + int IntraSize; + MPI_Comm InterComm; + int InterSize; + int InterCommType; + char DsmMasterHostName[MPI_MAX_PORT_NAME]; +}; + +#endif /* XDMFDSMCOMMMPI_HPP_ */ + diff --git a/core/XdmfDSMDriver.cpp b/core/XdmfDSMDriver.cpp new file mode 100644 index 00000000..a5e43915 --- /dev/null +++ b/core/XdmfDSMDriver.cpp @@ -0,0 +1,1345 @@ +/*****************************************************************************/ +/* XDMF */ +/* eXtensible Data Model and Format */ +/* */ +/* Id : XdmfDSMDriver.cpp */ +/* */ +/* Author: */ +/* Andrew Burns */ +/* andrew.j.burns2@us.army.mil */ +/* US Army Research Laboratory */ +/* Aberdeen Proving Ground, MD */ +/* */ +/* Copyright @ 2013 US Army Research Laboratory */ +/* All Rights Reserved */ +/* See Copyright.txt for details */ +/* */ +/* This software is distributed WITHOUT ANY WARRANTY; without */ +/* even the implied warranty of MERCHANTABILITY or FITNESS */ +/* FOR A PARTICULAR PURPOSE. See the above copyright notice */ +/* for more information. */ +/* */ +/*****************************************************************************/ + +/*========================================================================= + This code is derived from an earlier work and is distributed + with permission from, and thanks to ... +=========================================================================*/ + +/*============================================================================ + + Project : H5FDdsm + Module : H5FDdsmDriver.cxx H5FDdsm.c + + Authors: + John Biddiscombe Jerome Soumagne + biddisco@cscs.ch soumagne@cscs.ch + + Copyright (C) CSCS - Swiss National Supercomputing Centre. + You may use modify and and distribute this code freely providing + 1) This copyright notice appears on all copies of source code + 2) An acknowledgment appears with any substantial usage of the code + 3) If this code is contributed to any other open source project, it + must not be reformatted such that the indentation, bracketing or + overall style is modified significantly. + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + + This work has received funding from the European Community's Seventh + Framework Programme (FP7/2007-2013) under grant agreement 225967 âxtMuSEâOC + +============================================================================*/ + +#include +#include +#include +#include +#include +#include + +#include + +typedef struct XDMF_dsm_t +{ + H5FD_t pub; /* public stuff, must be first */ + char *name; /* for equivalence testing */ + MPI_Comm intra_comm; /* intra-communicator */ + int intra_rank; /* this process's rank in intra_comm */ + int intra_size; /* total number of processes in intra_comm */ + void *local_buf_ptr; /* underlying local DSM buffer */ + size_t local_buf_len; /* local DSM buffer length */ + haddr_t eoa; /* end of address marker */ + haddr_t eof; /* end of file marker */ + haddr_t start; /* current DSM start address */ + haddr_t end; /* current DSM end address */ + hbool_t read_only; /* file access is read-only */ + hbool_t dirty; /* dirty marker */ +} XDMF_dsm_t; + +typedef struct XDMF_dsm_fapl_t +{ + MPI_Comm intra_comm; /* intra-communicator */ + void *local_buf_ptr; /* local buffer pointer */ + size_t local_buf_len; /* local buffer length */ +} XDMF_dsm_fapl_t; + +typedef struct +{ + long start; + long end; +} XdmfDSMEntry; + +/* The driver identification number, initialized at runtime */ +static hid_t XDMF_DSM_g = 0; + +//from driver +XdmfDSMManager *dsmManager = NULL; + +#define MAXADDR ((haddr_t)((~(size_t)0)-1)) +#define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || (A) > (haddr_t)MAXADDR) +#define SIZE_OVERFLOW(Z) ((Z) > (hsize_t)MAXADDR) +#define REGION_OVERFLOW(A,Z) (ADDR_OVERFLOW(A) || SIZE_OVERFLOW(Z) || \ + HADDR_UNDEF==(A)+(Z) || \ + (size_t)((A)+(Z))<(size_t)(A)) + +extern "C" { +/* Private Prototypes */ +static void *XDMF_dsm_fapl_get(H5FD_t *_file); +static void *XDMF_dsm_fapl_copy(const void *_old_fa); +static herr_t XDMF_dsm_fapl_free(void *_fa); +static H5FD_t *XDMF_dsm_open(const char *name, unsigned flags, hid_t fapl_id, + haddr_t maxaddr); +static herr_t XDMF_dsm_close(H5FD_t *_file); +static herr_t XDMF_dsm_query(const H5FD_t *_f1, unsigned long *flags); +static haddr_t XDMF_dsm_get_eoa(const H5FD_t *_file, H5FD_mem_t type); +static herr_t XDMF_dsm_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); +static haddr_t XDMF_dsm_get_eof(const H5FD_t *_file); +static herr_t XDMF_dsm_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, + haddr_t addr, size_t size, void *buf); +static herr_t XDMF_dsm_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, + haddr_t addr, size_t size, const void *buf); +static herr_t XDMF_dsm_flush(H5FD_t *_file, hid_t dxpl_id, unsigned closing); +static int XDMF_dsm_mpi_rank(const H5FD_t *_file); +static int XDMF_dsm_mpi_size(const H5FD_t *_file); +static MPI_Comm XDMF_dsm_communicator(const H5FD_t *_file); +} + +/* The DSM file driver information */ +static const H5FD_class_mpi_t XDMF_dsm_g = { + { + "dsm", /* name */ + MAXADDR, /* maxaddr */ + H5F_CLOSE_SEMI, /* fc_degree */ +#if H5_VERSION_GE(1,9,0) + XDMF_dsm_term, /* terminate */ +#endif + NULL, /* sb_size */ + NULL, /* sb_encode */ + NULL, /* sb_decode */ + sizeof(XDMF_dsm_fapl_t), /* fapl_size */ + XDMF_dsm_fapl_get, /* fapl_get */ + XDMF_dsm_fapl_copy, /* fapl_copy */ + XDMF_dsm_fapl_free, /* fapl_free */ + 0, /* dxpl_size */ + NULL, /* dxpl_copy */ + NULL, /* dxpl_free */ + XDMF_dsm_open, /* open */ + XDMF_dsm_close, /* close */ + NULL, /* cmp */ + XDMF_dsm_query, /* query */ + NULL, /* get_type_map */ + NULL, /* alloc */ + NULL, /* free */ +#ifdef H5_HAVE_VFD_EXTENSIONS + XDMF_dsm_term, /* terminate */ +#endif + XDMF_dsm_get_eoa, /* get_eoa */ + XDMF_dsm_set_eoa, /* set_eoa */ + XDMF_dsm_get_eof, /* get_eof */ + NULL, /* get_handle */ + XDMF_dsm_read, /* read */ + XDMF_dsm_write, /* write */ + XDMF_dsm_flush, /* flush */ + NULL, /* truncate */ + NULL, /* lock */ + NULL, /* unlock */ + H5FD_FLMAP_SINGLE /* fl_map */ + }, + XDMF_dsm_mpi_rank, /* get_rank */ + XDMF_dsm_mpi_size, /* get_size */ + XDMF_dsm_communicator /* get_comm */ +}; + +extern "C" { + +#define H5_INTERFACE_INIT_FUNC XDMF_dsm_init_interface + +#include "H5FDprivate.h" /* File drivers */ +#include "H5private.h" /* Generic Functions */ +#include "H5ACprivate.h" /* Metadata cache */ +#include "H5Dprivate.h" /* Dataset functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* File access */ +#include "H5FDmpi.h" /* MPI-based file drivers */ +#include "H5Iprivate.h" /* IDs */ +#include "H5Pprivate.h" /* Property lists */ + +static herr_t +XDMF_dsm_init_interface(void) +{ +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_NOAPI_NOINIT_NOERR +#else + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(XDMF_dsm_init_interface) +#endif + + FUNC_LEAVE_NOAPI(XDMF_dsm_init()) +} + +hid_t +XDMF_dsm_init(void) +{ + hid_t ret_value; /* Return value */ + +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_NOAPI(FAIL) +#else + FUNC_ENTER_NOAPI(XDMF_dsm_init, FAIL) +#endif + + if (H5I_VFL != H5Iget_type(XDMF_DSM_g)) {// registering the driver + XDMF_DSM_g = H5FD_register(&XDMF_dsm_g, sizeof(H5FD_class_mpi_t), FALSE); + } + + /* Set return value */ + ret_value = XDMF_DSM_g;// return value is the new id of the driver + +done: + if (err_occurred) { + /* Nothing */ + } + + FUNC_LEAVE_NOAPI(ret_value) +} + + +#if H5_VERSION_GE(1,9,0) +herr_t +#else +void +#endif +XDMF_dsm_term(void) +{ +#if H5_VERSION_GE(1,9,0) + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) +#else +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_NOAPI_NOINIT_NOERR +#else + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(XDMF_dsm_term) +#endif +#endif + + if (SUCCEED != xdmf_dsm_free()) { +#if H5_VERSION_GE(1,9,0) + HGOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "dsm_free failed"); +#endif + } + + /* Reset VFL ID */ + XDMF_DSM_g = 0; + +#if H5_VERSION_GE(1,9,0) +done: + FUNC_LEAVE_NOAPI(ret_value) +#else + FUNC_LEAVE_NOAPI_VOID +#endif +} + +herr_t +XDMF_dsm_set_options(unsigned long flags) +{ + herr_t ret_value = SUCCEED; + +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_NOAPI(FAIL) +#else + FUNC_ENTER_NOAPI(XDMF_dsm_set_options, FAIL) +#endif + + if (SUCCEED != xdmf_dsm_set_options(flags)) + HGOTO_ERROR(H5E_VFL, H5E_CANTMODIFY, FAIL, "cannot set options") + +done: + if (err_occurred) { + /* Nothing */ + } + + FUNC_LEAVE_NOAPI(ret_value) +} + +herr_t XDMF_dsm_lock(void) +{ + herr_t ret_value = SUCCEED; + +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_NOAPI(FAIL) +#else + FUNC_ENTER_NOAPI(XDMF_dsm_lock, FAIL) +#endif + + if (SUCCEED != xdmf_dsm_lock()) + HGOTO_ERROR(H5E_VFL, H5E_CANTMODIFY, FAIL, "cannot lock") + +done: + if (err_occurred) { + /* Nothing */ + } + + FUNC_LEAVE_NOAPI(ret_value) +} + +herr_t XDMF_dsm_unlock(unsigned long flag) +{ + herr_t ret_value = SUCCEED; + +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_NOAPI(FAIL) +#else + FUNC_ENTER_NOAPI(XDMF_dsm_unlock, FAIL) +#endif + + if (SUCCEED != xdmf_dsm_unlock(flag)) + HGOTO_ERROR(H5E_VFL, H5E_CANTMODIFY, FAIL, "cannot lock") + +done: + if (err_occurred) { + /* Nothing */ + } + + FUNC_LEAVE_NOAPI(ret_value) +} + +herr_t +XDMF_dsm_set_manager(void *manager) +{ + herr_t ret_value = SUCCEED; + +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_NOAPI(FAIL) +#else + FUNC_ENTER_NOAPI(XDMF_dsm_set_manager, FAIL) +#endif + + xdmf_dsm_set_manager(manager); + +done: + if (err_occurred) { + /* Nothing */ + } + + FUNC_LEAVE_NOAPI(ret_value) +} + +herr_t +XDMFH5Pset_fapl_dsm(hid_t fapl_id, MPI_Comm intra_comm, void *local_buf_ptr, + size_t local_buf_len) +{ + XDMF_dsm_fapl_t fa; + herr_t ret_value = SUCCEED; + H5P_genplist_t *plist; /* Property list pointer */ + +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_API(FAIL) +#else + FUNC_ENTER_API(XDMFH5Pset_fapl_dsm, FAIL) +#endif + /* Check arguments */ + if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + if (MPI_COMM_NULL == intra_comm) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a valid communicator") + + if (!xdmf_dsm_get_manager()) { + // throw error here instead of calling alloc + /*if (SUCCEED != dsm_alloc(intra_comm, local_buf_ptr, local_buf_len)) + HGOTO_ERROR(H5E_PLIST, H5E_CANTALLOC, FAIL, "cannot allocate DSM buffer")*/ + } + + if (SUCCEED != xdmf_dsm_get_properties(&fa.intra_comm, &fa.local_buf_ptr, &fa.local_buf_len)) + HGOTO_ERROR(H5E_PLIST, H5E_CANTALLOC, FAIL, "cannot get DSM properties") + + if (!xdmf_dsm_is_server() && !xdmf_dsm_is_connected()) { + + //it should already be connected when this is called + //potentially the connect will connect via port + xdmf_dsm_connect(); + } + + /* duplication is done during driver setting */ + ret_value = H5P_set_driver(plist, XDMF_DSM, &fa); + +done: + FUNC_LEAVE_API(ret_value) +} + +herr_t +XDMFH5Pget_fapl_dsm(hid_t fapl_id, MPI_Comm *intra_comm /* out */, + void **local_buf_ptr_ptr /* out */, size_t *local_buf_len_ptr /* out */) +{ + XDMF_dsm_fapl_t *fa; + MPI_Comm intra_comm_tmp = MPI_COMM_NULL; + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; + int mpi_code; + +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_API(FAIL) +#else + FUNC_ENTER_API(XDMFH5Pget_fapl_dsm, FAIL) +#endif + + if (NULL == (plist = (H5P_genplist_t*) H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + if (XDMF_DSM != H5P_get_driver(plist)) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver") + if (NULL == (fa = (XDMF_dsm_fapl_t*) H5P_get_driver_info(plist))) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "bad VFL driver info") + + if (intra_comm) { + if (MPI_SUCCESS != (mpi_code = MPI_Comm_dup(fa->intra_comm, &intra_comm_tmp))) + HMPI_GOTO_ERROR(FAIL, "MPI_Comm_dup failed", mpi_code) + *intra_comm = intra_comm_tmp; + } + + if (local_buf_ptr_ptr) *local_buf_ptr_ptr = fa->local_buf_ptr; + + if (local_buf_len_ptr) *local_buf_len_ptr = fa->local_buf_len; + +done: + if (FAIL == ret_value) { + /* need to free anything created here */ + if (intra_comm_tmp != MPI_COMM_NULL) + MPI_Comm_free(&intra_comm_tmp); + } + + FUNC_LEAVE_API(ret_value) +} + +static void * +XDMF_dsm_fapl_get(H5FD_t *_file) +{ + XDMF_dsm_t *file = (XDMF_dsm_t*) _file; + XDMF_dsm_fapl_t *fa = NULL; + void *ret_value = NULL; + int mpi_code; + +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_NOAPI_NOINIT +#else + FUNC_ENTER_NOAPI_NOINIT(XDMF_dsm_fapl_get) +#endif + + HDassert(file); + HDassert(XDMF_DSM == file->pub.driver_id); + + if (NULL == (fa = (XDMF_dsm_fapl_t *) calloc((size_t)1, sizeof(XDMF_dsm_fapl_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + + /* Duplicate communicator. */ + fa->intra_comm = MPI_COMM_NULL; + if (MPI_SUCCESS != (mpi_code = MPI_Comm_dup(file->intra_comm, &fa->intra_comm))) + HMPI_GOTO_ERROR(NULL, "MPI_Comm_dup failed", mpi_code) + + fa->local_buf_ptr = file->local_buf_ptr; + fa->local_buf_len = file->local_buf_len; + + /* Set return value */ + ret_value = fa; + +done: + if ((NULL == ret_value) && err_occurred) { + /* need to free anything created here */ + if (fa && (MPI_COMM_NULL != fa->intra_comm)) + MPI_Comm_free(&fa->intra_comm); + } + + FUNC_LEAVE_NOAPI(ret_value) +} + +static void * +XDMF_dsm_fapl_copy(const void *_old_fa) +{ + void *ret_value = NULL; + const XDMF_dsm_fapl_t *old_fa = (const XDMF_dsm_fapl_t*)_old_fa; + XDMF_dsm_fapl_t *new_fa = NULL; + int mpi_code; + +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_NOAPI_NOINIT +#else + FUNC_ENTER_NOAPI_NOINIT(XDMF_dsm_fapl_copy) +#endif + + if(NULL == (new_fa = (XDMF_dsm_fapl_t *) calloc((size_t)1, sizeof(XDMF_dsm_fapl_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + + /* Copy the general information */ + HDmemcpy(new_fa, old_fa, sizeof(XDMF_dsm_fapl_t)); + + /* Duplicate communicator. */ + new_fa->intra_comm = MPI_COMM_NULL; + + if (MPI_SUCCESS != (mpi_code = MPI_Comm_dup(old_fa->intra_comm, &new_fa->intra_comm))) + HMPI_GOTO_ERROR(NULL, "MPI_Comm_dup failed", mpi_code) + + ret_value = new_fa; + +done: + if ((NULL == ret_value) && err_occurred) { + /* cleanup */ + if (new_fa && (MPI_COMM_NULL != new_fa->intra_comm)) + MPI_Comm_free(&new_fa->intra_comm); + if (new_fa) free(new_fa); + } + + FUNC_LEAVE_NOAPI(ret_value) +} + +static herr_t +XDMF_dsm_fapl_free(void *_fa) +{ + herr_t ret_value = SUCCEED; + XDMF_dsm_fapl_t *fa = (XDMF_dsm_fapl_t*)_fa; + +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_NOAPI_NOINIT_NOERR +#else + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(XDMF_dsm_fapl_free) +#endif + + assert(fa); + + /* Free the internal communicator */ + assert(MPI_COMM_NULL != fa->intra_comm); + MPI_Comm_free(&fa->intra_comm); + + free(fa); + + FUNC_LEAVE_NOAPI(ret_value) +} + +static H5FD_t * +XDMF_dsm_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) +{ + XDMF_dsm_t *file = NULL; + int mpi_rank; /* MPI rank of this process */ + int mpi_size; /* Total number of MPI processes */ + int mpi_code; /* mpi return code */ + const XDMF_dsm_fapl_t *fa = NULL; + MPI_Comm intra_comm_dup = MPI_COMM_NULL; + H5P_genplist_t *plist; /* Property list pointer */ + H5FD_t *ret_value = NULL; + herr_t dsm_code = SUCCEED; + +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_NOAPI_NOINIT +#else + FUNC_ENTER_NOAPI_NOINIT(XDMF_dsm_open) +#endif + + /* Check arguments */ + if(!name || !*name) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file name") + if (0 == maxaddr || HADDR_UNDEF == maxaddr) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "bogus maxaddr") + if (ADDR_OVERFLOW(maxaddr)) + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, NULL, "maxaddr overflow") + + + if (H5P_DEFAULT != fapl_id) { + + if (NULL == (plist = (H5P_genplist_t*) H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list") + fa = (const XDMF_dsm_fapl_t *) H5P_get_driver_info(plist); + assert(fa); + } + + /* Duplicate communicator. */ + if (MPI_SUCCESS != (mpi_code = MPI_Comm_dup(fa->intra_comm, &intra_comm_dup))) + HMPI_GOTO_ERROR(NULL, "MPI_Comm_dup failed", mpi_code) + + /* Get the MPI rank of this process and the total number of processes */ + if (MPI_SUCCESS != (mpi_code = MPI_Comm_rank (fa->intra_comm, &mpi_rank))) + HMPI_GOTO_ERROR(NULL, "MPI_Comm_rank failed", mpi_code) + if (MPI_SUCCESS != (mpi_code = MPI_Comm_size (fa->intra_comm, &mpi_size))) + HMPI_GOTO_ERROR(NULL, "MPI_Comm_size failed", mpi_code) + + /* Build the return value and initialize it */ + if (NULL == (file = (XDMF_dsm_t *) calloc((size_t)1, sizeof(XDMF_dsm_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + + file->intra_comm = intra_comm_dup; + + file->intra_rank = mpi_rank; + file->intra_size = mpi_size; + + if (name && *name) { + file->name = HDstrdup(name); + } + + /* Get address information from DSM */ + if (!xdmf_dsm_get_manager()) + HGOTO_ERROR(H5E_VFL, H5E_NOTFOUND, NULL, "DSM buffer not found") + + file->local_buf_ptr = fa->local_buf_ptr; + file->local_buf_len = fa->local_buf_len; + + /* locking needs to be changed + if (SUCCEED != dsm_lock()) + HGOTO_ERROR(H5E_VFL, H5E_CANTLOCK, NULL, "cannot lock DSM") + */ + + // find the start and end of the entry on core 0 + if ((file->intra_rank == 0) && (SUCCEED != xdmf_dsm_get_entry(&file->start, &file->end))) + dsm_code = FAIL; + + /* Wait for the DSM entry to be updated */ + if (MPI_SUCCESS != (mpi_code = MPI_Bcast(&dsm_code, sizeof(herr_t), + MPI_UNSIGNED_CHAR, 0, file->intra_comm))) + HMPI_GOTO_ERROR(NULL, "MPI_Bcast failed", mpi_code) + if (SUCCEED != dsm_code) + HGOTO_ERROR(H5E_VFL, H5E_CANTRESTORE, NULL, "cannot restore DSM entries") + + if (MPI_SUCCESS != (mpi_code = MPI_Bcast(&file->start, sizeof(haddr_t), + MPI_UNSIGNED_CHAR, 0, file->intra_comm))) + HMPI_GOTO_ERROR(NULL, "MPI_Bcast failed", mpi_code) + if (MPI_SUCCESS != (mpi_code = MPI_Bcast(&file->end, sizeof(haddr_t), + MPI_UNSIGNED_CHAR, 0, file->intra_comm))) + HMPI_GOTO_ERROR(NULL, "MPI_Bcast failed", mpi_code) + + if (H5F_ACC_RDWR & flags) { + file->read_only = FALSE; + } else { + file->read_only = TRUE; + } + + if (H5F_ACC_CREAT & flags) { + /* Reset start and end markers */ + file->start = 0; + file->end = 0; + file->eof = 0; + } else { + file->eof = file->end - file->start; + } + + /* Don't let any proc return until all have created the file. */ + if ((H5F_ACC_CREAT & flags)) { + if (MPI_SUCCESS != (mpi_code = MPI_Barrier(intra_comm_dup))) + HMPI_GOTO_ERROR(NULL, "MPI_Barrier failed in open", mpi_code) + } + + /* Set return value */ + ret_value = (H5FD_t *) file; + +// HGOTO_ERROR(H5E_VFL, H5E_CANTLOCK, NULL, "FAKE ERROR") + +done: + if((ret_value == NULL) && err_occurred) { + if (file && file->name) HDfree(file->name); + if ((MPI_COMM_NULL != intra_comm_dup)) MPI_Comm_free(&intra_comm_dup); + if (file) free(file); + } /* end if */ + + + FUNC_LEAVE_NOAPI(ret_value) +} + +static herr_t +XDMF_dsm_close(H5FD_t *_file) +{ + XDMF_dsm_t *file = (XDMF_dsm_t*) _file; + herr_t ret_value = SUCCEED; /* Return value */ + int mpi_code; + herr_t dsm_code = SUCCEED; + unsigned long unlock_flag; + +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_NOAPI_NOINIT +#else + FUNC_ENTER_NOAPI_NOINIT(XDMF_dsm_close) +#endif + + assert(file); + assert(XDMF_DSM == file->pub.driver_id); + + if (!file->read_only) { + file->end = MAX((file->start + file->eof), file->end); + + if ((file->intra_rank == 0) && (SUCCEED != xdmf_dsm_update_entry(file->start, file->end))) + dsm_code = FAIL; + /* Wait for the DSM entry to be updated */ + if (MPI_SUCCESS != (mpi_code = MPI_Bcast(&dsm_code, sizeof(herr_t), + MPI_UNSIGNED_CHAR, 0, file->intra_comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_code) + if (SUCCEED != dsm_code) + HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "cannot update DSM entries") + + /* + * Be sure that everyone's here before releasing resources (done with + * collective op). Gather all the dirty flags because some processes may + * not have written yet. + */ + if (MPI_SUCCESS != (mpi_code = MPI_Allreduce(MPI_IN_PLACE, &file->dirty, + sizeof(hbool_t), MPI_UNSIGNED_CHAR, MPI_MAX, file->intra_comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_code) + } + + /* if ReleaseLockOnClose was set, unlocks using whatever notification + * unlock_flag was set. + */ + unlock_flag = (file->dirty) ? XDMF_DSM_NOTIFY_DATA : XDMF_DSM_NOTIFY_NONE; + if (SUCCEED != xdmf_dsm_unlock(unlock_flag)) HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCK, FAIL, "cannot unlock DSM") + + /* Release resources */ + if (file->name) HDfree(file->name); + if (MPI_COMM_NULL != file->intra_comm) MPI_Comm_free(&file->intra_comm); + HDmemset(file, 0, sizeof(XDMF_dsm_t)); + free(file); + +done: + if (err_occurred) { + /* Nothing */ + } + + FUNC_LEAVE_NOAPI(ret_value) +} + +static herr_t +XDMF_dsm_query(const H5FD_t UNUSED *_file, unsigned long *flags /* out */) +{ +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_NOAPI_NOINIT_NOERR +#else + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(XDMF_dsm_query) +#endif + + /* Set the VFL feature flags that this driver supports */ + if (flags) { + *flags = 0; + *flags |= H5FD_FEAT_AGGREGATE_METADATA; /* OK to aggregate metadata allocations */ + *flags |= H5FD_FEAT_AGGREGATE_SMALLDATA; /* OK to aggregate "small" raw data allocations */ + *flags |= H5FD_FEAT_HAS_MPI; /* This driver uses MPI */ + *flags |= H5FD_FEAT_ALLOCATE_EARLY; /* Allocate space early instead of late */ + } /* end if */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} + +static haddr_t +XDMF_dsm_get_eoa(const H5FD_t *_file, H5FD_mem_t UNUSED type) +{ + const XDMF_dsm_t *file = (const XDMF_dsm_t*) _file; + +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_NOAPI_NOINIT_NOERR +#else + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(XDMF_dsm_get_eoa) +#endif + + assert(file); + assert(XDMF_DSM == file->pub.driver_id); + + FUNC_LEAVE_NOAPI(file->eoa) +} + +static herr_t +XDMF_dsm_set_eoa(H5FD_t *_file, H5FD_mem_t UNUSED type, haddr_t addr) +{ + XDMF_dsm_t *file = (XDMF_dsm_t*) _file; + herr_t ret_value = SUCCEED; /* Return value */ + int mpi_code; + herr_t dsm_code = SUCCEED; + +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_NOAPI_NOINIT +#else + FUNC_ENTER_NOAPI_NOINIT(XDMF_dsm_set_eoa) +#endif + + assert(file); + assert(XDMF_DSM == file->pub.driver_id); + + if (ADDR_OVERFLOW(addr)) + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "address overflow") + + file->eoa = addr; + + file->end = MAX((file->start + file->eoa), file->end); + file->eof = file->end - file->start; + if (!file->read_only) { + if ((file->intra_rank == 0) && (SUCCEED != xdmf_dsm_update_entry(file->start, file->end))) + dsm_code = FAIL; + /* Wait for the DSM entry to be updated */ + if (MPI_SUCCESS != (mpi_code = MPI_Bcast(&dsm_code, sizeof(herr_t), + MPI_UNSIGNED_CHAR, 0, file->intra_comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_code) + if (SUCCEED != dsm_code) + HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "cannot update DSM entries") + } + +done: + if (err_occurred) { + /* Nothing */ + } + + FUNC_LEAVE_NOAPI(ret_value) +} + +static haddr_t +XDMF_dsm_get_eof(const H5FD_t *_file) +{ + const XDMF_dsm_t *file = (const XDMF_dsm_t*) _file; + +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_NOAPI_NOINIT_NOERR +#else + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(XDMF_dsm_get_eof) +#endif + + assert(file); + assert(XDMF_DSM == file->pub.driver_id); + + FUNC_LEAVE_NOAPI(MAX(file->eof, file->eoa)) +} + +static herr_t +XDMF_dsm_read(H5FD_t *_file, H5FD_mem_t UNUSED type, hid_t UNUSED dxpl_id, + haddr_t addr, size_t size, void *buf /* out */) +{ + XDMF_dsm_t *file = (XDMF_dsm_t*) _file; + herr_t ret_value = SUCCEED; /* Return value */ + +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_NOAPI_NOINIT +#else + FUNC_ENTER_NOAPI_NOINIT(XDMF_dsm_read) +#endif + + assert(file); + assert(XDMF_DSM == file->pub.driver_id); + assert(buf); + + /* Check for overflow conditions */ + if (HADDR_UNDEF == addr) + HGOTO_ERROR(H5E_IO, H5E_OVERFLOW, FAIL, "file address overflowed") + if (REGION_OVERFLOW(addr, size)) + HGOTO_ERROR(H5E_IO, H5E_OVERFLOW, FAIL, "file address overflowed") + if (addr + size > file->eoa) + HGOTO_ERROR(H5E_IO, H5E_OVERFLOW, FAIL, "file address overflowed") + + /* Read the part which is before the EOF marker */ + if (addr < file->eof) { + size_t nbytes; + hsize_t temp_nbytes; + + temp_nbytes = file->eof - addr; + H5_CHECK_OVERFLOW(temp_nbytes,hsize_t,size_t); + nbytes = MIN(size,(size_t)temp_nbytes); + + /* Read from DSM to BUF */ + if (SUCCEED != xdmf_dsm_read(file->start + addr, nbytes, buf)) { + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "cannot read from DSM") + } else { + size -= nbytes; + addr += nbytes; + buf = (char*) buf + nbytes; + } + } + /* Read zeros for the part which is after the EOF markers */ + if (size > 0) HDmemset(buf, 0, size); + +done: + if (err_occurred) { + /* Nothing */ + } + + FUNC_LEAVE_NOAPI(ret_value) +} + +static herr_t +XDMF_dsm_write(H5FD_t *_file, H5FD_mem_t UNUSED type, hid_t UNUSED dxpl_id, + haddr_t addr, size_t size, const void *buf) +{ + XDMF_dsm_t *file = (XDMF_dsm_t*) _file; + herr_t ret_value = SUCCEED; /* Return value */ + +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_NOAPI_NOINIT +#else + FUNC_ENTER_NOAPI_NOINIT(XDMF_dsm_write) +#endif + + assert(file); + assert(XDMF_DSM == file->pub.driver_id); + assert(buf); + + if (file->read_only) + HGOTO_ERROR(H5E_IO, H5E_RESOURCE, FAIL, "cannot write to DSM open in read-only") + + /* Check for overflow conditions */ + if (REGION_OVERFLOW(addr, size)) + HGOTO_ERROR(H5E_IO, H5E_OVERFLOW, FAIL, "file address overflowed") + if (addr + size > file->eoa) + HGOTO_ERROR(H5E_IO, H5E_OVERFLOW, FAIL, "file address overflowed") + + /* For now, do not allow dynamic reallocation of the DSM */ + if (addr + size > file->eof) + HGOTO_ERROR(H5E_IO, H5E_NOSPACE, FAIL, "not enough space in DSM") + + /* Write from BUF to DSM */ + if (SUCCEED != xdmf_dsm_write(file->start + addr, size, buf)) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot write to DSM") + + /* Set dirty flag so that we know someone has written something */ + file->dirty = TRUE; + +done: + if (err_occurred) { + /* Nothing */ + } + + FUNC_LEAVE_NOAPI(ret_value) +} + +static herr_t +XDMF_dsm_flush(H5FD_t *_file, hid_t UNUSED dxpl_id, unsigned UNUSED closing) +{ + /* H5FD_dsm_t *file = (H5FD_dsm_t*) _file; */ + herr_t ret_value = SUCCEED; /* Return value */ + +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_NOAPI_NOINIT_NOERR +#else + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(XDMF_dsm_flush) +#endif + + /* Write to backing store */ + /* + if (file->dirty) { + haddr_t size = file->eof; + + unsigned char *ptr = file->mem; + + if (0!=HDlseek(file->fd, (off_t)0, SEEK_SET)) + HGOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "error seeking in backing store") + + while (size) { + ssize_t n; + + H5_CHECK_OVERFLOW(size,hsize_t,size_t); + n = HDwrite(file->fd, ptr, (size_t)size); + if (n<0 && EINTR==errno) + continue; + if (n<0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "error writing backing store") + ptr += (size_t)n; + size -= (size_t)n; + } + + file->dirty = FALSE; + } + */ + + FUNC_LEAVE_NOAPI(ret_value) +} + +static int +XDMF_dsm_mpi_rank(const H5FD_t *_file) +{ + const XDMF_dsm_t *file = (const XDMF_dsm_t*) _file; + +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_NOAPI_NOINIT_NOERR +#else + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(XDMF_dsm_mpi_rank) +#endif + + assert(file); + assert(XDMF_DSM == file->pub.driver_id); + + FUNC_LEAVE_NOAPI(file->intra_rank) +} + +static int +XDMF_dsm_mpi_size(const H5FD_t *_file) +{ + const XDMF_dsm_t *file = (const XDMF_dsm_t*) _file; + +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_NOAPI_NOINIT_NOERR +#else + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(XDMF_dsm_mpi_size) +#endif + + assert(file); + assert(XDMF_DSM == file->pub.driver_id); + + FUNC_LEAVE_NOAPI(file->intra_size) +} + +static MPI_Comm +XDMF_dsm_communicator(const H5FD_t *_file) +{ + const XDMF_dsm_t *file = (const XDMF_dsm_t*) _file; + +#if H5_VERSION_GE(1,8,9) + FUNC_ENTER_NOAPI_NOINIT_NOERR +#else + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(XDMF_dsm_communicator) +#endif + + assert(file); + assert(XDMF_DSM == file->pub.driver_id); + + FUNC_LEAVE_NOAPI(file->intra_comm) +} + +} + +//from Driver + +void* +xdmf_dsm_get_manager() +{ + void *ret_value = NULL; + if (dsmManager) ret_value = static_cast (dsmManager); + return(ret_value); +} + +herr_t +xdmf_dsm_get_properties(MPI_Comm *intra_comm, void **buf_ptr_ptr, size_t *buf_len_ptr) +{ + if (!dsmManager) { + try { + XdmfError::message(XdmfError::FATAL, "No DSM manager found"); + } + catch (XdmfError e) { + return FAIL; + } + } + + if (intra_comm) *intra_comm = dsmManager->GetDsmBuffer()->GetComm()->GetIntraComm(); + if (dsmManager->GetIsServer()) { + if (buf_ptr_ptr) *buf_ptr_ptr = + dsmManager->GetDsmBuffer()->GetDataPointer(); + if (buf_len_ptr) *buf_len_ptr = + dsmManager->GetDsmBuffer()->GetLength(); + } else { + if (buf_ptr_ptr) *buf_ptr_ptr = NULL; + if (buf_len_ptr) *buf_len_ptr = 0; + } + + return(SUCCEED); +} + +void +xdmf_dsm_set_manager(void *manager) +{ + dsmManager = static_cast (manager); +} + +/* generates a Manager if one doesn't already exist. probably a bad idea in most cases +herr_t +dsm_alloc(MPI_Comm intra_comm, void *buf_ptr, size_t buf_len) +{ + // Check arguments + if (dsmManager) DSM_DRIVER_ERROR("DSM manager already allocated") + if (intra_comm == MPI_COMM_NULL) DSM_DRIVER_ERROR("invalid intra comm argument") + if (buf_ptr && !buf_len) DSM_DRIVER_ERROR("invalid buffer length argument") + + dsmManager = new H5FDdsmManager(); + dsmManager->ReadConfigFile(); + dsmManager->SetIsAutoAllocated(H5FD_DSM_TRUE); + dsmManager->SetMpiComm(intra_comm); + if (buf_ptr) { + // TODO initialize DSM from given buffer + } + if (dsmManager->Create() != H5FD_DSM_SUCCESS) + DSM_DRIVER_ERROR("Cannot create DSM manager") + + return(SUCCEED); +}*/ + +herr_t +xdmf_dsm_free() +{ + if (dsmManager) { + /* probably not required, since the autoallocation is not on + if (dsmManager->GetIsAutoAllocated()) { + if (dsmManager->GetIsConnected()) dsmManager->Disconnect(); + + delete dsmManager; + dsmManager = NULL; + } + */ + } + + return(SUCCEED); +} + +hbool_t +xdmf_dsm_is_server() +{ + hbool_t ret_value = TRUE; + + if (!dsmManager) { + XdmfError::message(XdmfError::FATAL, "No DSM manager found"); + } + + ret_value = dsmManager->GetIsServer(); + + return(ret_value); +} + +herr_t +xdmf_dsm_set_options(unsigned long flags) +{ + XdmfDSMBuffer *dsmBuffer = NULL; + if (dsmManager != NULL) { + dsmBuffer = dsmManager->GetDsmBuffer(); + } + else { + try { + XdmfError::message(XdmfError::FATAL, "No DSM manager found"); + } + catch (XdmfError e) { + return FAIL; + } + } + + //currently no options to set + + return(SUCCEED); +} + +hbool_t +xdmf_dsm_is_connected() +{ + hbool_t ret_value = TRUE; + + if (!dsmManager) { + XdmfError::message(XdmfError::FATAL, "No DSM manager found"); + } + + ret_value = dsmManager->GetIsConnected(); + + return(ret_value); +} + +herr_t +xdmf_dsm_connect() +{ + + if (!dsmManager) { + try { + XdmfError::message(XdmfError::FATAL, "No DSM manager found"); + } + catch (XdmfError e) { + return FAIL; + } + } + + // Initialize the connection if it has not been done already + if (dsmManager->GetIsConnected()) { + try { + XdmfError::message(XdmfError::FATAL, "Already Connected"); + } + catch (XdmfError e) { + return FAIL; + } + } + + try { + dsmManager->Connect(); + } + catch (XdmfError e) { + return FAIL; + } + + return(SUCCEED); +} + +herr_t +xdmf_dsm_update_entry(haddr_t start, haddr_t end) +{ + haddr_t addr; + XdmfDSMEntry entry; + XdmfDSMBuffer *dsmBuffer = NULL; + + if (dsmManager != NULL) { + dsmBuffer = dsmManager->GetDsmBuffer(); + } + else { + try { + XdmfError::message(XdmfError::FATAL, "No DSM manager found"); + } + catch (XdmfError e) { + return FAIL; + } + } + + entry.start = start; + entry.end = end; + addr = (int) (dsmBuffer->GetTotalLength() - sizeof(XdmfDSMEntry) - 1); + + // Do not send anything if the end of the file is 0 + if (entry.end > 0) { + try { + dsmBuffer->Put(addr, sizeof(entry), &entry); + } + catch (XdmfError e) { + return FAIL; + } + } + + return SUCCEED; +} + +herr_t +xdmf_dsm_get_entry(haddr_t *start_ptr, haddr_t *end_ptr) +{ + haddr_t addr; + XdmfDSMEntry entry; + XdmfDSMBuffer *dsmBuffer = NULL; + + if (dsmManager != NULL) { + dsmBuffer = dsmManager->GetDsmBuffer(); + } + else { + try { + XdmfError::message(XdmfError::FATAL, "No DSM manager found"); + } + catch (XdmfError e) { + return FAIL; + } + } + + addr = (int) (dsmBuffer->GetTotalLength() - sizeof(XdmfDSMEntry) - 1); + + try { + dsmBuffer->Get(addr, sizeof(entry), &entry); + } + catch (XdmfError e) { + return FAIL; + } + + *start_ptr = entry.start; + *end_ptr = entry.end; + + return SUCCEED; +} + +herr_t +xdmf_dsm_lock() +{ + XdmfDSMBuffer *dsmBuffer = NULL; + if (dsmManager != NULL) { + dsmBuffer = dsmManager->GetDsmBuffer(); + } + else { + try { + XdmfError::message(XdmfError::FATAL, "No DSM manager found"); + } + catch (XdmfError e) { + return FAIL; + } + } + +/* behavior will be different here + H5FDdsmBoolean parallel = (dsmManager->GetIsDriverSerial() == H5FD_DSM_TRUE) ? FALSE : TRUE; + if (dsmBufferService->RequestLockAcquire(parallel) != H5FD_DSM_SUCCESS) + DSM_DRIVER_ERROR("Cannot request lock acquisition") +*/ + return(SUCCEED); +} + +herr_t +xdmf_dsm_unlock(unsigned long flag) +{ + XdmfDSMBuffer *dsmBuffer = NULL; + if (dsmManager != NULL) { + dsmBuffer = dsmManager->GetDsmBuffer(); + } + else { + try { + XdmfError::message(XdmfError::FATAL, "No DSM manager found"); + } + catch (XdmfError e) { + return FAIL; + } + } + +/*behavior will be different here + bool parallel = (dsmManager->GetIsDriverSerial() == H5FD_DSM_TRUE) ? FALSE : TRUE; + if (dsmBufferService->RequestLockRelease(flag, parallel) != H5FD_DSM_SUCCESS) + DSM_DRIVER_ERROR("Cannot request lock release") +*/ + return(SUCCEED); +} + +herr_t +xdmf_dsm_read(haddr_t addr, size_t len, void *buf_ptr) +{ + XdmfDSMBuffer *dsmBuffer = NULL; + if (dsmManager != NULL) { + dsmBuffer = dsmManager->GetDsmBuffer(); + } + else { + try { + XdmfError::message(XdmfError::FATAL, "No DSM manager found"); + } + catch (XdmfError e) { + return FAIL; + } + } + + try { + dsmBuffer->Get(addr, len, buf_ptr); + } + catch (XdmfError e) { + return FAIL; + } + + return(SUCCEED); +} + +herr_t +xdmf_dsm_write(haddr_t addr, size_t len, const void *buf_ptr) +{ + XdmfDSMBuffer *dsmBuffer = NULL; + if (dsmManager != NULL) { + dsmBuffer = dsmManager->GetDsmBuffer(); + } + else { + try { + XdmfError::message(XdmfError::FATAL, "No DSM manager found"); + } + catch (XdmfError e) { + return FAIL; + } + } + + try { + dsmBuffer->Put(addr, len, buf_ptr); + } + catch (XdmfError e) { + return FAIL; + } + + return(SUCCEED); +} diff --git a/core/XdmfDSMDriver.hpp b/core/XdmfDSMDriver.hpp new file mode 100644 index 00000000..b8817809 --- /dev/null +++ b/core/XdmfDSMDriver.hpp @@ -0,0 +1,127 @@ +/*****************************************************************************/ +/* XDMF */ +/* eXtensible Data Model and Format */ +/* */ +/* Id : XdmfDSMDriver.hpp */ +/* */ +/* Author: */ +/* Andrew Burns */ +/* andrew.j.burns2@us.army.mil */ +/* US Army Research Laboratory */ +/* Aberdeen Proving Ground, MD */ +/* */ +/* Copyright @ 2013 US Army Research Laboratory */ +/* All Rights Reserved */ +/* See Copyright.txt for details */ +/* */ +/* This software is distributed WITHOUT ANY WARRANTY; without */ +/* even the implied warranty of MERCHANTABILITY or FITNESS */ +/* FOR A PARTICULAR PURPOSE. See the above copyright notice */ +/* for more information. */ +/* */ +/*****************************************************************************/ + +/*========================================================================= + This code is derived from an earlier work and is distributed + with permission from, and thanks to ... +=========================================================================*/ + +/*========================================================================= + + Project : H5FDdsm + Module : H5FDdsmDriver.h H5FDdsm.h + + Authors: + John Biddiscombe Jerome Soumagne + biddisco@cscs.ch soumagne@cscs.ch + + Copyright (C) CSCS - Swiss National Supercomputing Centre. + You may use modify and and distribute this code freely providing + 1) This copyright notice appears on all copies of source code + 2) An acknowledgment appears with any substantial usage of the code + 3) If this code is contributed to any other open source project, it + must not be reformatted such that the indentation, bracketing or + overall style is modified significantly. + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + + This work has received funding from the European Community's Seventh + Framework Programme (FP7/2007-2013) under grant agreement 225967 âxtMuSEâOB +=========================================================================*/ + +#ifndef XDMFDSMDRIVER_HPP_ +#define XDMFDSMDRIVER_HPP_ + +// Forward Declarations + +// Includes +#include +#include +#include +#include +#include + +#include "H5Ipublic.h" +#include "H5FDpublic.h" + +#ifndef H5_HAVE_PARALLEL + #pragma Error : The H5FDdsm virtual File Driver for HDF5 can only be compiled against an HDF5 library with parallel IO support +#endif + +#define XDMF_DSM_NOTIFY_NONE 0x0000 +#define XDMF_DSM_NOTIFY_DATA 0x0001 /* this is the default */ +#define XDMF_DSM_NOTIFY_INFORMATION 0x0002 +/* Internal notifications */ +#define XDMF_DSM_NOTIFY_WAIT 0x0003 +#define XDMF_DSM_NOTIFY_CONNECTED 0x0004 +#define XDMF_DSM_NOTIFY_DISCONNECTED 0x0005 +/* User notifications */ +#define XDMF_DSM_NOTIFY_USER 0x0010 + +#define XDMF_DSM (XDMF_dsm_init()) + +extern "C" { + XDMFCORE_EXPORT hid_t XDMF_dsm_init(void); +#if H5_VERSION_GE(1,9,0) + XDMFCORE_EXPORT herr_t XDMF_dsm_term(void); +#else + XDMFCORE_EXPORT void XDMF_dsm_term(void); +#endif + XDMFCORE_EXPORT herr_t XDMF_dsm_lock(void);// lock and unlock are going to need different behavior + XDMFCORE_EXPORT herr_t XDMF_dsm_unlock(unsigned long flag); + XDMFCORE_EXPORT herr_t XDMF_dsm_set_options(unsigned long flags);// currently no options to set + XDMFCORE_EXPORT herr_t XDMF_dsm_set_manager(void *manager); + XDMFCORE_EXPORT herr_t XDMFH5Pset_fapl_dsm(hid_t fapl_id, MPI_Comm intra_comm, + void *local_buf_ptr, size_t local_buf_len); + XDMFCORE_EXPORT herr_t XDMFH5Pget_fapl_dsm(hid_t fapl_id, MPI_Comm *intra_comm /* out */, + void **local_buf_ptr_ptr /* out */, size_t *local_buf_len_ptr /* out */); + + + + + XDMFCORE_EXPORT void *xdmf_dsm_get_manager(); + XDMFCORE_EXPORT herr_t xdmf_dsm_get_properties(MPI_Comm *intra_comm, + void **buf_ptr_ptr, size_t *buf_len_ptr); + XDMFCORE_EXPORT void xdmf_dsm_set_manager(void *manager); + + XDMFCORE_EXPORT herr_t xdmf_dsm_alloc(MPI_Comm intra_comm, void *buf_ptr, size_t buf_len);// Probably a bad idea to create managers automatically for the non-threaded version + XDMFCORE_EXPORT herr_t xdmf_dsm_free(); + + XDMFCORE_EXPORT hbool_t xdmf_dsm_is_server(); + XDMFCORE_EXPORT herr_t xdmf_dsm_set_options(unsigned long flags);// currently no options to set + + XDMFCORE_EXPORT hbool_t xdmf_dsm_is_connected(); + XDMFCORE_EXPORT herr_t xdmf_dsm_connect(); + + XDMFCORE_EXPORT herr_t xdmf_dsm_update_entry(haddr_t start, haddr_t end); + XDMFCORE_EXPORT herr_t xdmf_dsm_get_entry(haddr_t *start_ptr, haddr_t *end_ptr); + + XDMFCORE_EXPORT herr_t xdmf_dsm_lock();// Lock and unlock will need different behavior + XDMFCORE_EXPORT herr_t xdmf_dsm_unlock(unsigned long flag); + + XDMFCORE_EXPORT herr_t xdmf_dsm_read(haddr_t addr, size_t len, void *buf_ptr); + XDMFCORE_EXPORT herr_t xdmf_dsm_write(haddr_t addr, size_t len, const void *buf_ptr); +} + +#endif /* XDMFDSMDRIVER_HPP_ */ diff --git a/core/XdmfDSMManager.cpp b/core/XdmfDSMManager.cpp new file mode 100644 index 00000000..9f026c70 --- /dev/null +++ b/core/XdmfDSMManager.cpp @@ -0,0 +1,316 @@ +/*****************************************************************************/ +/* XDMF */ +/* eXtensible Data Model and Format */ +/* */ +/* Id : XdmfDSMManager.cpp */ +/* */ +/* Author: */ +/* Andrew Burns */ +/* andrew.j.burns2@us.army.mil */ +/* US Army Research Laboratory */ +/* Aberdeen Proving Ground, MD */ +/* */ +/* Copyright @ 2013 US Army Research Laboratory */ +/* All Rights Reserved */ +/* See Copyright.txt for details */ +/* */ +/* This software is distributed WITHOUT ANY WARRANTY; without */ +/* even the implied warranty of MERCHANTABILITY or FITNESS */ +/* FOR A PARTICULAR PURPOSE. See the above copyright notice */ +/* for more information. */ +/* */ +/*****************************************************************************/ + +/*========================================================================= + This code is derived from an earlier work and is distributed + with permission from, and thanks to ... +=========================================================================*/ + +/*============================================================================ + + Project : H5FDdsm + Module : H5FDdsmManger.cxx + + Authors: + John Biddiscombe Jerome Soumagne + biddisco@cscs.ch soumagne@cscs.ch + + Copyright (C) CSCS - Swiss National Supercomputing Centre. + You may use modify and and distribute this code freely providing + 1) This copyright notice appears on all copies of source code + 2) An acknowledgment appears with any substantial usage of the code + 3) If this code is contributed to any other open source project, it + must not be reformatted such that the indentation, bracketing or + overall style is modified significantly. + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + + This work has received funding from the European Community's Seventh + Framework Programme (FP7/2007-2013) under grant agreement 225967 âxtMuSEâOC + +============================================================================*/ + +#include +#include +#include +#include +#include + +XdmfDSMManager::XdmfDSMManager() +{ + this->MpiComm = MPI_COMM_NULL; + this->UpdatePiece = 0; + this->UpdateNumPieces = 0; + this->LocalBufferSizeMBytes = 128; + + this->DsmBuffer = NULL; + this->DsmComm = NULL; + this->IsServer = true; + this->DsmType = XDMF_DSM_TYPE_UNIFORM; + this->BlockLength = XDMF_DSM_DEFAULT_BLOCK_LENGTH; + this->InterCommType = XDMF_DSM_COMM_MPI; +} + +XdmfDSMManager::~XdmfDSMManager() +{ + this->Destroy(); +} + +XdmfDSMBuffer * +XdmfDSMManager::GetDsmBuffer() +{ + return this->DsmBuffer; +} + +void +XdmfDSMManager::SetDsmBuffer(XdmfDSMBuffer * newBuffer) +{ + this->DsmBuffer = newBuffer; +} + +int +XdmfDSMManager::GetUpdatePiece() +{ + return this->UpdatePiece; +} + +int +XdmfDSMManager::GetUpdateNumPieces() +{ + return this->UpdateNumPieces; +} + +MPI_Comm +XdmfDSMManager::GetMpiComm() +{ + return this->MpiComm; +} + +void +XdmfDSMManager::SetMpiComm(MPI_Comm comm) +{ + if (comm != this->MpiComm) { + this->MpiComm = comm; + if (this->MpiComm != MPI_COMM_NULL) { + MPI_Comm_size(this->MpiComm, &this->UpdateNumPieces); + MPI_Comm_rank(this->MpiComm, &this->UpdatePiece); + } + } +} + +void +XdmfDSMManager::SetLocalBufferSizeMBytes(unsigned int newSize) +{ + this->LocalBufferSizeMBytes = newSize; +} + +unsigned int +XdmfDSMManager::GetLocalBufferSizeMBytes() +{ + return this->LocalBufferSizeMBytes; +} + +void +XdmfDSMManager::SetIsServer(bool newStatus) +{ + this->IsServer = newStatus; +} + +bool +XdmfDSMManager::GetIsServer() +{ + return this->IsServer; +} + +void +XdmfDSMManager::SetDsmType(int newType) +{ + this->DsmType = newType; +} + +int +XdmfDSMManager::GetDsmType() +{ + return this->DsmType; +} + +void +XdmfDSMManager::SetBlockLength(long newSize) +{ + this->BlockLength = newSize; +} + +long +XdmfDSMManager::GetBlockLength() +{ + return this->BlockLength; +} + +void +XdmfDSMManager::SetInterCommType(int newType) +{ + this->InterCommType = newType; +} + +int +XdmfDSMManager::GetInterCommType() +{ + return this->InterCommType; +} + +bool +XdmfDSMManager::GetIsConnected() +{ + if (this->DsmBuffer) + { + return this->DsmBuffer->GetIsConnected(); + } + else + { + return false; + } +} + +void +XdmfDSMManager::Destroy() +{ + // Watch out that all processes have empty message queues + // Should be already done during the disconnection + if (this->DsmBuffer) { + delete this->DsmBuffer; + this->DsmBuffer = NULL; + //Will be replaced by an Xdmf version + //H5FD_dsm_set_manager(NULL); + } + if (this->DsmComm) { + delete this->DsmComm; + this->DsmComm = NULL; + } +} + +void +XdmfDSMManager::Create(int startId, int endId) +{ + if (!this->DsmBuffer) { + + MPI_Comm_size(this->MpiComm, &this->UpdateNumPieces); + MPI_Comm_rank(this->MpiComm, &this->UpdatePiece); + // + // Create DSM communicator + // + switch (this->GetInterCommType()) { + case XDMF_DSM_COMM_MPI: + this->DsmComm = new XdmfDSMCommMPI(); + break; + default: + try { + XdmfError::message(XdmfError::FATAL, "DSM communication type not supported"); + } + catch (XdmfError e) { + throw e; + } + } + this->DsmComm->DupComm(this->MpiComm); + this->DsmComm->Init(); + // + // Create the DSM buffer + // + this->DsmBuffer = new XdmfDSMBuffer(); + // + this->DsmBuffer->SetIsServer(this->IsServer); + // Uniform Dsm : every node has a buffer the same size. (Addresses are sequential) + long length = (long) (this->GetLocalBufferSizeMBytes())*1024LU*1024LU; + switch (this->DsmType) { + case XDMF_DSM_TYPE_UNIFORM: + case XDMF_DSM_TYPE_UNIFORM_RANGE: + this->DsmBuffer->ConfigureUniform(this->DsmComm, length, startId, endId); + break; + case XDMF_DSM_TYPE_BLOCK_CYCLIC: + this->DsmBuffer->ConfigureUniform(this->DsmComm, length, startId, endId, this->BlockLength, false); + break; + case XDMF_DSM_TYPE_BLOCK_RANDOM: + this->DsmBuffer->ConfigureUniform(this->DsmComm, length, startId, endId, this->BlockLength, true); + break; + default: + try { + XdmfError(XdmfError::FATAL, "DSM configuration type not supported"); + } + catch (XdmfError e) { + throw e; + } + } + } +} + + +void +XdmfDSMManager::Connect(bool persist) +{ + int status; + + if (!(dynamic_cast (this->DsmBuffer)->GetIsConnected())) { + + do { + try { + status = this->DsmBuffer->GetComm()->Connect(); + } + catch (XdmfError e) { + throw e; + } + if (status == MPI_SUCCESS) { + dynamic_cast (this->DsmBuffer)->SetIsConnected(true); + try { + this->DsmBuffer->ReceiveInfo(); + } + catch (XdmfError e) { + throw e; + } + } + else { +#ifdef _WIN32 + Sleep(1000);//since windows has a different sleep command +#else + sleep(1); +#endif + } + } while (persist && (status != MPI_SUCCESS)); + } +} + +void +XdmfDSMManager::Disconnect() +{ + //disconnecting is done manually + + try + { + this->DsmBuffer->GetComm()->Disconnect(); + } + catch (XdmfError e) + { + throw e; + } + + dynamic_cast (this->DsmBuffer)->SetIsConnected(false); +} diff --git a/core/XdmfDSMManager.hpp b/core/XdmfDSMManager.hpp new file mode 100644 index 00000000..cde30879 --- /dev/null +++ b/core/XdmfDSMManager.hpp @@ -0,0 +1,812 @@ +/*****************************************************************************/ +/* XDMF */ +/* eXtensible Data Model and Format */ +/* */ +/* Id : XdmfDSMManager.hpp */ +/* */ +/* Author: */ +/* Andrew Burns */ +/* andrew.j.burns2@us.army.mil */ +/* US Army Research Laboratory */ +/* Aberdeen Proving Ground, MD */ +/* */ +/* Copyright @ 2013 US Army Research Laboratory */ +/* All Rights Reserved */ +/* See Copyright.txt for details */ +/* */ +/* This software is distributed WITHOUT ANY WARRANTY; without */ +/* even the implied warranty of MERCHANTABILITY or FITNESS */ +/* FOR A PARTICULAR PURPOSE. See the above copyright notice */ +/* for more information. */ +/* */ +/*****************************************************************************/ + +/*========================================================================= + This code is derived from an earlier work and is distributed + with permission from, and thanks to ... +=========================================================================*/ + +/*========================================================================= + + Project : H5FDdsm + Module : H5FDdsmManager.h + + Authors: + John Biddiscombe Jerome Soumagne + biddisco@cscs.ch soumagne@cscs.ch + + Copyright (C) CSCS - Swiss National Supercomputing Centre. + You may use modify and and distribute this code freely providing + 1) This copyright notice appears on all copies of source code + 2) An acknowledgment appears with any substantial usage of the code + 3) If this code is contributed to any other open source project, it + must not be reformatted such that the indentation, bracketing or + overall style is modified significantly. + + This software is distributed WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + + This work has received funding from the European Community's Seventh + Framework Programme (FP7/2007-2013) under grant agreement 225967 âxtMuSEâOB +=========================================================================*/ + +#ifndef XDMFDSMMANAGER_HPP_ +#define XDMFDSMMANAGER_HPP_ + +// Forward Declarations + +// Includes +#include +#include +#include +#include + + +/** + * @brief Holds communicators for interacting with H5FD dsm. + * + * XdmfDSMManager takes the place of the H5FDdsmManager defined in H5FD. + * It is primarily for allowing the XdmfDSM to interact with HDF5 dsm without threads. + */ +class XDMFCORE_EXPORT XdmfDSMManager { + +public: + + XdmfDSMManager(); + ~XdmfDSMManager(); + + /** + * Attempts to connect the buffer to the port that is currently set. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline XdmfDSMCommMPI + * @skipline connectingGroup + * @until false + * @skipline } + * @skip (connectingGroup + * @until } + * @skipline (connectingGroup + * @until } + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline exampleDSMComm + * @skipline connectingGroup + * @until False + * @skip Disconnect + * @skipline (connectingGroup + * @until Disconnect + * + * @param persist Whether to try to connect repeatedly + */ + void Connect(bool persist = false); + + /** + * Creates an internal buffer based on the information provided to the Manager already. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleManager + * @skipline exampleWriter + * @skipline Create + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleManager + * @skipline Create + * + * @param startId The index of the first server node + * @param endId The index of the last server node + */ + void Create(int startId = -1, int endId = -1); + + /** + * Disposes of the current DSM data buffer. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleManager + * @skipline exampleWriter + * @skipline Destroy + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleManager + * @skipline Destroy + * + */ + void Destroy(); + + /** + * Disconnects the buffer from the port it was connected to. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline XdmfDSMCommMPI + * @skipline connectingGroup + * @until false + * @skipline } + * @skip (connectingGroup + * @until } + * @skipline (connectingGroup + * @until } + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline exampleDSMComm + * @skipline connectingGroup + * @until False + * @skip Disconnect + * @skipline (connectingGroup + * @until Disconnect + * + */ + void Disconnect(); + + /** + * gets the block length for the DSM data buffer. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleManager + * @skipline exampleWriter + * @skipline GetBlockLength + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleManager + * @skipline exampleBlockLength + * + * @return The size of the blocks currently being used + */ + long GetBlockLength(); + + /** + * Gets the manager's internal buffer. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleManager + * @skipline exampleWriter + * @skipline exampleBuffer + * @skipline GetDSMBuffer + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleManager + * @skip exampleBuffer + * @skipline GetDSMBuffer + * + * @return The buffer that the manager is using as its internal buffer + */ + XdmfDSMBuffer * GetDsmBuffer(); + + /** + * Gets the current type of DSM that the Manager is using. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleManager + * @skipline exampleWriter + * @skipline GetDsmType + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleManager + * @skipline exampleType + * + * @return They type of DSM being used + */ + int GetDsmType(); + + /** + * Gets the type of intercomm that the manager is currently using. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleManager + * @skipline exampleWriter + * @skipline GetInterCommType + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleManager + * @skipline exampleCommType + * + * @return They type of intercomm currently being used + */ + int GetInterCommType(); + + /** + * Gets if the Buffer is connected via intercomm. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleManager + * @skipline exampleWriter + * @skipline GetIsConnected + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleManager + * @skipline GetIsConnected + * + * @return Whether the Buffer is connected + */ + bool GetIsConnected(); + + /** + * Gets whether the Manager is managing a server or not. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleManager + * @skipline exampleWriter + * @skipline GetIsServer + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleManager + * @skipline exampleIsServer + * + * @return Whether the Manager is a server or not + */ + bool GetIsServer(); + + /** + * Gets the MpiComm that the manager is currently using. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleManager + * @skipline exampleWriter + * @skipline exampleManagerComm + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleManager + * @skipline exampleManagerComm + * + * @return The MpiComm that the manager is currently using + */ + MPI_Comm GetMpiComm(); + + /** + * Gets the maximum size of the local buffer on server cores. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleManager + * @skipline exampleWriter + * @skipline MBytes + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleManager + * @skipline exampleBufferSize + * + * @return the maximum size of the data buffer on server cores + */ + unsigned int GetLocalBufferSizeMBytes(); + + /** + * Gets the id of the core with regards to the MpiComm + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleManager + * @skipline exampleWriter + * @skipline intraId + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleManager + * @skipline intraId + * + * @return The Id of the core calling this function + */ + int GetUpdatePiece(); + + /** + * Gets the total number of cores on the MpiComm + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleManager + * @skipline exampleWriter + * @skipline intraSize + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleManager + * @skipline intraSize + * + * @return The total number of cores over the MpiComm + */ + int GetUpdateNumPieces(); + + /** + * Sets the block length for the DSM data buffer. + * Memory will be alloted in a multiple of the block size. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleManager + * @skipline exampleWriter + * @skipline GetBlockLength + * @skipline Set + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleManager + * @skipline exampleBlockLength + * @skipline SetBlockLength + * + * @param newSize The size to the blocks to be used + */ + void SetBlockLength(long newSize); + + /** + * Sets the manager's internal buffer to the buffer provided. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleManager + * @skipline exampleWriter + * @skipline exampleBuffer + * @skipline getServerBuffer + * @skipline SetDSMBuffer + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleManager + * @skipline exampleBuffer + * @skipline SetDSMBuffer + * + * @param newBuffer The buffer that the Manager is to use as its internal buffer + */ + void SetDsmBuffer(XdmfDSMBuffer * newBuffer); + + /** + * Sets the DSM type that the Manager will set up when create is called + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleManager + * @skipline exampleWriter + * @skipline SetDsmType + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleManager + * @skipline SetDsmType + * + * @param newType The Type of DSM that the manager should generate + */ + void SetDsmType(int newType); + + /** + * Sets whether this Manager is managing a server or not. + * If false it will attempt to use the intercomm for data manipulation. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleManager + * @skipline exampleWriter + * @skipline GetIsServer + * @skipline Set + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleManager + * @skipline exampleIsServer + * @skipline SetIsServer + * + * @param newStatus Whether the Manager is to be a server or not + */ + void SetIsServer(bool newStatus); + + /** + * Sets the type of intercomm that the Manager will use. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleManager + * @skipline exampleWriter + * @skipline SetInterCommType + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleManager + * @skipline SetInterCommType + * + * @param newType The type of intercomm to be generated for now on + */ + void SetInterCommType(int newType); + + /** + * Sets the maximum size of the local buffer when generating data buffers for server cores. + * When using blocked mode it generates a buffer that is a multiple of the block size + * that is less than or equal to this number. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleManager + * @skipline exampleWriter + * @skipline MBytes + * @skipline Set + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleManager + * @skipline exampleBufferSize + * @skipline SetLocalBufferSize + * + * @param newSize The new maximum size of the data buffer on the server cores + */ + void SetLocalBufferSizeMBytes(unsigned int newSize); + + /** + * Sets the MpiComm to the provided communicator and updates UpdatePiece and UpdateNumPieces + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleManager + * @skipline exampleWriter + * @skipline exampleManagerComm + * @skipline SetMpiComm + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleManager + * @skipline exampleManagerComm + * @skipline SetMpiComm + * + * @param comm The comm to be set as the MpiComm for this manager + */ + void SetMpiComm(MPI_Comm comm); + +protected: + + +private: + + int UpdatePiece; + int UpdateNumPieces; + unsigned int LocalBufferSizeMBytes; + + MPI_Comm MpiComm; + + XdmfDSMBuffer *DsmBuffer; + XdmfDSMCommMPI *DsmComm; + + bool IsServer; + int DsmType; + long BlockLength; + int InterCommType; +}; + +#endif /* XDMFDSMMANAGER_HPP_ */ diff --git a/core/XdmfHDF5Controller.cpp b/core/XdmfHDF5Controller.cpp index 239c934a..9f91b146 100644 --- a/core/XdmfHDF5Controller.cpp +++ b/core/XdmfHDF5Controller.cpp @@ -100,6 +100,7 @@ XdmfHDF5Controller::read(XdmfArray * const array, const int fapl) std::vector stride(mStride.begin(), mStride.end()); std::vector count(mDimensions.begin(), mDimensions.end()); + status = H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, &start[0], @@ -164,6 +165,7 @@ XdmfHDF5Controller::read(XdmfArray * const array, const int fapl) } } + array->initialize(mType, mDimensions); if(numVals != array->getSize()) { diff --git a/core/XdmfHDF5ControllerDSM.cpp b/core/XdmfHDF5ControllerDSM.cpp index 00df283f..33696cf2 100644 --- a/core/XdmfHDF5ControllerDSM.cpp +++ b/core/XdmfHDF5ControllerDSM.cpp @@ -25,6 +25,11 @@ #include #include #include "XdmfHDF5ControllerDSM.hpp" +#include "XdmfDSMManager.hpp" +#include "XdmfDSMBuffer.hpp" +#include "XdmfDSMCommMPI.hpp" +#include "XdmfDSMDriver.hpp" +#include "XdmfError.hpp" shared_ptr XdmfHDF5ControllerDSM::New(const std::string & hdf5FilePath, @@ -72,6 +77,60 @@ XdmfHDF5ControllerDSM::New(const std::string & hdf5FilePath, return p; } +//server/ nonthreaded versions +shared_ptr +XdmfHDF5ControllerDSM::New(const std::string & hdf5FilePath, + const std::string & dataSetPath, + const shared_ptr type, + const std::vector & start, + const std::vector & stride, + const std::vector & dimensions, + const std::vector & datspaceDimensions, + XdmfDSMBuffer * const dsmBuffer, + int startCoreIndex, + int endCoreIndex) +{ + shared_ptr + p(new XdmfHDF5ControllerDSM(hdf5FilePath, + dataSetPath, + type, + start, + stride, + dimensions, + datspaceDimensions, + dsmBuffer, + startCoreIndex, + endCoreIndex)); + return p; +} + +shared_ptr +XdmfHDF5ControllerDSM::New(const std::string & hdf5FilePath, + const std::string & dataSetPath, + const shared_ptr type, + const std::vector & start, + const std::vector & stride, + const std::vector & dimensions, + const std::vector & datspaceDimensions, + MPI_Comm comm, + unsigned int bufferSize, + int startCoreIndex, + int endCoreIndex) +{ + shared_ptr + p(new XdmfHDF5ControllerDSM(hdf5FilePath, + dataSetPath, + type, + start, + stride, + dimensions, + datspaceDimensions, + comm, + bufferSize, + startCoreIndex, + endCoreIndex)); + return p; +} XdmfHDF5ControllerDSM::XdmfHDF5ControllerDSM(const std::string & hdf5FilePath, const std::string & dataSetPath, @@ -88,7 +147,18 @@ XdmfHDF5ControllerDSM::XdmfHDF5ControllerDSM(const std::string & hdf5FilePath, stride, dimensions, dataspaceDimensions), - mDSMBuffer(dsmBuffer) + mDSMManager(NULL), + mDSMBuffer(dsmBuffer), + mDSMServerBuffer(NULL), + mDSMServerManager(NULL), + mGroupComm(MPI_COMM_NULL), + mServerComm(MPI_COMM_NULL), + mWorkerComm(MPI_COMM_NULL), + mStartCoreIndex(-1), + mEndCoreIndex(-1), + mRank(-1), + mGroupSize(-1), + mServerMode(false) { } @@ -107,7 +177,18 @@ XdmfHDF5ControllerDSM::XdmfHDF5ControllerDSM(const std::string & hdf5FilePath, start, stride, dimensions, - dataspaceDimensions) + dataspaceDimensions), + mDSMServerBuffer(NULL), + mDSMServerManager(NULL), + mGroupComm(MPI_COMM_NULL), + mServerComm(MPI_COMM_NULL), + mWorkerComm(MPI_COMM_NULL), + mStartCoreIndex(-1), + mEndCoreIndex(-1), + mRank(-1), + mGroupSize(-1), + mServerMode(false) + { H5FDdsmManager * newManager = new H5FDdsmManager(); newManager->SetMpiComm(comm); @@ -125,6 +206,160 @@ XdmfHDF5ControllerDSM::XdmfHDF5ControllerDSM(const std::string & hdf5FilePath, mDSMBuffer = newBuffer; } +XdmfHDF5ControllerDSM::XdmfHDF5ControllerDSM(const std::string & hdf5FilePath, + const std::string & dataSetPath, + const shared_ptr type, + const std::vector & start, + const std::vector & stride, + const std::vector & dimensions, + const std::vector & dataspaceDimensions, + XdmfDSMBuffer * const dsmBuffer, + int startCoreIndex, + int endCoreIndex) : + XdmfHDF5Controller(hdf5FilePath, + dataSetPath, + type, + start, + stride, + dimensions, + dataspaceDimensions), + mDSMManager(NULL), + mDSMBuffer(NULL), + mDSMServerBuffer(dsmBuffer), + mDSMServerManager(NULL), + mStartCoreIndex(startCoreIndex), + mEndCoreIndex(endCoreIndex), + mServerMode(true) +{ + mGroupComm = mDSMServerBuffer->GetComm()->GetInterComm(); + MPI_Comm_rank(mGroupComm, &mRank); + MPI_Comm_size(mGroupComm, &mGroupSize); + if (mRank >=mStartCoreIndex && mRank <=mEndCoreIndex) { + mServerComm = mDSMServerBuffer->GetComm()->GetIntraComm(); + mWorkerComm = MPI_COMM_NULL; + } + else { + mServerComm = MPI_COMM_NULL; + mWorkerComm = mDSMServerBuffer->GetComm()->GetIntraComm(); + } +} + +XdmfHDF5ControllerDSM::XdmfHDF5ControllerDSM(const std::string & hdf5FilePath, + const std::string & dataSetPath, + const shared_ptr type, + const std::vector & start, + const std::vector & stride, + const std::vector & dimensions, + const std::vector & dataspaceDimensions, + MPI_Comm comm, + unsigned int bufferSize, + int startCoreIndex, + int endCoreIndex) : + XdmfHDF5Controller(hdf5FilePath, + dataSetPath, + type, + start, + stride, + dimensions, + dataspaceDimensions), + mDSMBuffer(NULL), + mDSMManager(NULL), + mServerMode(true) + +{ + //negative values will be changed to maximum range + if (startCoreIndex < 0) + { + startCoreIndex = 0; + } + if (endCoreIndex < 0) + { + endCoreIndex = mGroupSize - 1; + } + + //ensure start index is less than end index + if (startCoreIndex > endCoreIndex) + { + int tempholder = startCoreIndex; + startCoreIndex = endCoreIndex; + endCoreIndex = tempholder; + } + + mGroupComm = comm; + mStartCoreIndex = startCoreIndex; + mEndCoreIndex = endCoreIndex; + + MPI_Comm_size(comm, &mGroupSize); + MPI_Comm_rank(comm, &mRank); + + MPI_Group workers, dsmgroup, serversplit, servergroup; + + int * ServerIds = (int *)calloc((3), sizeof(int)); + unsigned int index = 0; + for(int i=mStartCoreIndex ; i <= mEndCoreIndex ; ++i) + { + ServerIds[index++] = i; + } + + MPI_Comm_group(comm, &serversplit); + MPI_Group_incl(serversplit, index, ServerIds, &servergroup); + MPI_Comm_create(comm, servergroup, &mServerComm); + MPI_Comm_group(comm, &dsmgroup); + MPI_Group_excl(dsmgroup, index, ServerIds, &workers); + MPI_Comm_create(comm, workers, &mWorkerComm); + cfree(ServerIds); + + //create the manager + + mDSMServerManager = new XdmfDSMManager(); + + mDSMServerManager->SetLocalBufferSizeMBytes(bufferSize); + mDSMServerManager->SetInterCommType(H5FD_DSM_COMM_MPI); + + if (mRank >=mStartCoreIndex && mRank <=mEndCoreIndex) + { + mDSMServerManager->SetMpiComm(mServerComm); + mDSMServerManager->Create(); + } + else + { + mDSMServerManager->SetMpiComm(mWorkerComm); + mDSMServerManager->SetIsServer(false); + mDSMServerManager->Create(mStartCoreIndex, mEndCoreIndex); + } + + XDMF_dsm_set_manager(mDSMServerManager); + + mDSMServerBuffer = mDSMServerManager->GetDsmBuffer(); + + mDSMServerBuffer->GetComm()->DupInterComm(mGroupComm); + mDSMServerBuffer->SetIsConnected(true); + + MPI_Barrier(comm); + + //loop needs to be started before anything can be done to the file, since the service is what sets up the file + + if (mRank < mStartCoreIndex || mRank > mEndCoreIndex) + { + //turn off the server designation + mDSMServerBuffer->SetIsServer(H5FD_DSM_FALSE);//if this is set to false then the buffer will attempt to connect to the intercomm for DSM stuff + mDSMServerManager->SetIsServer(H5FD_DSM_FALSE); + } + else + { + //on cores where memory is set up, start the service loop + //this should iterate infinitely until a value to end the loop is passed + H5FDdsmInt32 returnOpCode; + try + { + mDSMServerBuffer->BufferServiceLoop(&returnOpCode); + } + catch (XdmfError e) + { + throw e; + } + } +} XdmfHDF5ControllerDSM::~XdmfHDF5ControllerDSM() { @@ -145,6 +380,47 @@ H5FDdsmBuffer * XdmfHDF5ControllerDSM::getBuffer() return mDSMBuffer; } +XdmfDSMManager * XdmfHDF5ControllerDSM::getServerManager() +{ + return mDSMServerManager; +} + +XdmfDSMBuffer * XdmfHDF5ControllerDSM::getServerBuffer() +{ + return mDSMServerBuffer; +} + +bool XdmfHDF5ControllerDSM::getServerMode() +{ + return mServerMode; +} + +MPI_Comm XdmfHDF5ControllerDSM::getServerComm() +{ + MPI_Comm returnComm = MPI_COMM_NULL; + int status = MPI_Comm_dup(mServerComm, &returnComm); + return returnComm; +} + +MPI_Comm XdmfHDF5ControllerDSM::getWorkerComm() +{ + MPI_Comm returnComm = MPI_COMM_NULL; + int status = MPI_Comm_dup(mWorkerComm, &returnComm); + return returnComm; +} + +void XdmfHDF5ControllerDSM::setManager(XdmfDSMManager * newManager) +{ + XdmfDSMBuffer * newBuffer = newManager->GetDsmBuffer(); + mDSMServerManager = newManager; + mDSMServerBuffer = newBuffer; +} + +void XdmfHDF5ControllerDSM::setBuffer(XdmfDSMBuffer * newBuffer) +{ + mDSMServerBuffer = newBuffer; +} + void XdmfHDF5ControllerDSM::setManager(H5FDdsmManager * newManager) { H5FDdsmBuffer * newBuffer = newManager->GetDsmBuffer(); @@ -157,12 +433,127 @@ void XdmfHDF5ControllerDSM::setBuffer(H5FDdsmBuffer * newBuffer) mDSMBuffer = newBuffer; } +void XdmfHDF5ControllerDSM::setServerMode(bool newMode) +{ + mServerMode = newMode; +} + +void XdmfHDF5ControllerDSM::setServerComm(MPI_Comm comm) +{ + int status; + if (mServerComm != MPI_COMM_NULL) + { + status = MPI_Comm_free(&mServerComm); + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Failed to disconnect Comm"); + } + catch (XdmfError e) + { + throw e; + } + } + } + if (comm != MPI_COMM_NULL) + { + status = MPI_Comm_dup(comm, &mServerComm); + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Failed to duplicate Comm"); + } + catch (XdmfError e) + { + throw e; + } + } + } + mDSMServerBuffer->GetComm()->DupComm(comm); +} + +void XdmfHDF5ControllerDSM::setWorkerComm(MPI_Comm comm) +{ + int status; + if (mWorkerComm != MPI_COMM_NULL) + { + status = MPI_Comm_free(&mWorkerComm); + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Failed to disconnect Comm"); + } + catch (XdmfError e) + { + throw e; + } + } + } + if (comm != MPI_COMM_NULL) + { + status = MPI_Comm_dup(comm, &mWorkerComm); + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Failed to duplicate Comm"); + } + catch (XdmfError e) + { + throw e; + } + } + } + mDSMServerBuffer->GetComm()->DupComm(comm); +} + +void XdmfHDF5ControllerDSM::stopDSM() +{ + //send manually + for (int i = mStartCoreIndex; i <= mEndCoreIndex; ++i) + { + try + { + mDSMServerBuffer->SendCommandHeader(H5FD_DSM_OPCODE_DONE, i, 0, 0, H5FD_DSM_INTER_COMM); + } + catch (XdmfError e) + { + throw e; + } + //originally this was set to the intra comm + //that doesn't work in this instance because it won't reach the server cores + } +} + +void XdmfHDF5ControllerDSM::restartDSM() +{ + if (mRank >= mStartCoreIndex && mRank <= mEndCoreIndex) + { + H5FDdsmInt32 returnOpCode; + try + { + mDSMServerBuffer->BufferServiceLoop(&returnOpCode); + } + catch (XdmfError e) + { + throw e; + } + } +} + void XdmfHDF5ControllerDSM::deleteManager() { if (mDSMManager != NULL) { delete mDSMManager; } + if (mDSMServerManager != NULL) + { + delete mDSMServerManager; + } } void XdmfHDF5ControllerDSM::read(XdmfArray * const array) @@ -171,7 +562,14 @@ void XdmfHDF5ControllerDSM::read(XdmfArray * const array) hid_t fapl = H5Pcreate(H5P_FILE_ACCESS); // Use DSM driver - H5Pset_fapl_dsm(fapl, MPI_COMM_WORLD, mDSMBuffer, 0); + if (mServerMode) { + if (mWorkerComm != MPI_COMM_NULL) { + XDMFH5Pset_fapl_dsm(fapl, mWorkerComm, mDSMServerBuffer, 0); + } + } + else { + H5Pset_fapl_dsm(fapl, MPI_COMM_WORLD, mDSMBuffer, 0); + } // Read from DSM Buffer XdmfHDF5Controller::read(array, fapl); diff --git a/core/XdmfHDF5ControllerDSM.hpp b/core/XdmfHDF5ControllerDSM.hpp index 10940640..0a2bcd5f 100644 --- a/core/XdmfHDF5ControllerDSM.hpp +++ b/core/XdmfHDF5ControllerDSM.hpp @@ -31,6 +31,8 @@ class H5FDdsmManager; // Includes #include "XdmfCore.hpp" #include "XdmfHDF5Controller.hpp" +#include "XdmfDSMManager.hpp" +#include "XdmfDSMBuffer.hpp" /** * @brief Couples an XdmfArray with HDF5 data stored in a DSM buffer. @@ -179,10 +181,106 @@ public: MPI_Comm comm, unsigned int bufferSize); - std::string getName() const; + /** + * Create a new controller for an DSM data set. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline writeController + * @until size-1 + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline writeController + * @until size-1 + * + * @param hdf5FilePath The path to the hdf5 file that the controller will be accessing + * @param dataSetPath The location within the file of the data the controller with be accessing + * @param type The data type of the data Ex: XdmfArrayType::Int32() + * @param start A vector of the start indexes for all dimensions of the data + * @param stride A vector of the distance between reads for all dimensions of the data + * @param dimensions A vector of the number of values read from all dimensions of the data + * @param dataspaceDimensions A vecotr containing the total size of the dimension in the data space + * @param dsmBuffer A pointer to the dsm buffer + * @param startCoreIndex The index at which the server cores for the buffer start + * @param endCoreIndex The index at which the server cores for the buffer end + */ + static shared_ptr + New(const std::string & hdf5FilePath, + const std::string & dataSetPath, + const shared_ptr type, + const std::vector & start, + const std::vector & stride, + const std::vector & dimensions, + const std::vector & datspaceDimensions, + XdmfDSMBuffer * const dsmBuffer, + int startCoreIndex, + int endCoreIndex); /** - * Returns the current dsmManager for the Controller. If there is no manager then it returns null + * Create a new controller for an DSM data set. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until workerComm + * @skipline exampleController + * @until size-1 + * + * @param hdf5FilePath The path to the hdf5 file that the controller will be accessing + * @param dataSetPath The location within the file of the data the controller with be accessing + * @param type The data type of the data Ex: XdmfArrayType::Int32() + * @param start A vector of the start indexes for all dimensions of the data + * @param start A vector of the start indexes for all dimensions of the data + * @param stride A vector of the distance between reads for all dimensions of the data + * @param dimensions A vector of the number of values read from all dimensions of the data + * @param dataspaceDimensions A vecotr containing the total size of the dimension in the data space + * @param comm The communicator that the DSM buffer will reference + * @param bufferSize The size of the buffer to be created on the core calling this function + * @param startCoreIndex The index at which the server cores for the buffer start + * @param endCoreIndex The index at which the server cores for the buffer end + */ + static shared_ptr + New(const std::string & hdf5FilePath, + const std::string & dataSetPath, + const shared_ptr type, + const std::vector & start, + const std::vector & stride, + const std::vector & dimensions, + const std::vector & datspaceDimensions, + MPI_Comm comm, + unsigned int bufferSize, + int startCoreIndex, + int endCoreIndex); + + /** + * Deletes the manager that the controller contains. + * Used during cleanup. * * Example of Use: * @@ -195,9 +293,8 @@ public: * @until 4*size * @skipline writeController * @until dsmSize - * @skipline XdmfHDF5WriterDSM::New * @skipline dsmManager - * @skipline writeController->getManager() + * @skipline writeController->deleteManager() * * Python * @@ -205,19 +302,17 @@ public: * @skipline comm * @until size*4 * @skipline coreSize - * @until setManager + * @until getBuffer * @skipline setMode * @until setHeavyDataController * @skipline accept * @skipline segfault * @skipline writeController - * - * @return the dsmManager of the controller */ - H5FDdsmManager * getManager(); + void deleteManager(); /** - * Returns the current dsmBuffer the Controller. If there is no manager then it returns null + * Returns the current dsmManager for the Controller. If there is no manager then it returns null * * Example of Use: * @@ -240,20 +335,19 @@ public: * @skipline comm * @until size*4 * @skipline coreSize - * @until setBuffer + * @until setManager * @skipline setMode * @until setHeavyDataController * @skipline accept * @skipline segfault * @skipline writeController * - * @return the dsmBuffer of the controller + * @return the dsmManager of the controller */ - H5FDdsmBuffer * getBuffer(); + H5FDdsmManager * getManager(); /** - * Sets the controller's dsmManager to the provided manager. - * Then the dsmBuffer controlled by the manager is set to the controller + * Returns the current dsmBuffer the Controller. If there is no manager then it returns null * * Example of Use: * @@ -267,7 +361,6 @@ public: * @skipline writeController * @until dsmSize * @skipline XdmfHDF5WriterDSM::New - * @until setManager * @skipline dsmManager * @skipline writeController->getManager() * @@ -277,23 +370,190 @@ public: * @skipline comm * @until size*4 * @skipline coreSize - * @until If - * @skip delete - * @skipline writeController - * @until setManager + * @until setBuffer * @skipline setMode * @until setHeavyDataController * @skipline accept * @skipline segfault * @skipline writeController * - * @param newManager the manager to be set + * @return the dsmBuffer of the controller */ - void setManager(H5FDdsmManager * newManager); + H5FDdsmBuffer * getBuffer(); /** - * Sets the controller's dsmBuffer to the provided buffer + * Gets the buffer for the non-threaded version of DSM + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until workerComm + * @skipline exampleController + * @until size-1 + * @skipline if + * @skip exampleBuffer + * @skipline exampleController + * + * @return The XdmfDSMBuffer that is controlling the data for the DSM + */ + XdmfDSMBuffer * getServerBuffer(); + + /** + * Gets the communicator that the servers use to communicate between themselves. + * Will be MPI_COMM_NULL on worker cores. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline getServerComm + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until workerComm + * @skipline exampleController + * @until size-1 + * @skipline if + * @skipline exampleController + * + * @return The comm that the servers are using. + */ + MPI_Comm getServerComm(); + + /** + * Gets the manager for the non-threaded version of DSM + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMManager + * @skipline exampleController + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until workerComm + * @skipline exampleController + * @until size-1 + * @skipline if + * @skip exampleManager + * @skipline exampleController + * + * @return The XdmfDSMManager that is controlling the DSM + */ + XdmfDSMManager * getServerManager(); + + /** + * Checks if the DSM is in server mode or not. + * True is server mode, false is threaded + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline writeController + * @until size-1 + * @skipline if + * @until { + * @skipline getServerMode + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline writeController + * @until size-1 + * @skipline exampleControllerServerMode + * + * @return If the DSM is in server mode or not + */ + bool getServerMode(); + + std::string getName() const; + + /** + * Gets the Communicator that the workers are using to communicate between themselves * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline getWorkerComm + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until workerComm + * @skipline exampleController + * @until size-1 + * @skipline if + * @skipline exampleController + * @skipline getworkerComm + * + * @return The comm that the workers are using. + */ + MPI_Comm getWorkerComm(); + + /** + * Sets the controller's dsmBuffer to the provided buffer + * * Example of Use: * * C++ @@ -326,13 +586,49 @@ public: * @skipline segfault * @skipline writeController * - * @param newBuffer the buffer to be set + * @param newBuffer the buffer to be set */ void setBuffer(H5FDdsmBuffer * newBuffer); /** - * Deletes the manager that the controller contains. - * Used during cleanup. + * Sets the controller's dsmBuffer to the provided buffer + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMBuffer + * @skipline exampleController + * @skipline setBuffer + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until workerComm + * @skipline exampleController + * @until size-1 + * @skipline if + * @skip exampleBuffer + * @skipline exampleController + * @until setBuffer + * + * @param newBuffer A pointer to the buffer to be set + */ + void setBuffer(XdmfDSMBuffer * newBuffer); + + /** + * Sets the controller's dsmManager to the provided manager. + * Then the dsmBuffer controlled by the manager is set to the controller * * Example of Use: * @@ -345,8 +641,10 @@ public: * @until 4*size * @skipline writeController * @until dsmSize + * @skipline XdmfHDF5WriterDSM::New + * @until setManager * @skipline dsmManager - * @skipline writeController->deleteManager() + * @skipline writeController->getManager() * * Python * @@ -354,17 +652,227 @@ public: * @skipline comm * @until size*4 * @skipline coreSize - * @until getBuffer + * @until If + * @skip delete + * @skipline writeController + * @until setManager * @skipline setMode * @until setHeavyDataController * @skipline accept * @skipline segfault * @skipline writeController + * + * @param newManager the manager to be set */ - void deleteManager(); + void setManager(H5FDdsmManager * newManager); + + /** + * Sets the controller's dsmManager to the provided manager. + * Then the dsmBuffer controlled by the manager is set to the Writer + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline XdmfDSMManager + * @skipline exampleController + * @skipline setManager + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until workerComm + * @skipline exampleController + * @until size-1 + * @skipline if + * @skip exampleManager + * @skipline exampleController + * @until setManager + * + * @param newManager A pointer the the manager to be set. + */ + void setManager(XdmfDSMManager * newManager); + + /** + * Sets the comm that the servers will use to communicate with the other server cores. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline getServerComm + * @skipline setServerComm + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until workerComm + * @skipline exampleController + * @until size-1 + * @skipline if + * @skipline exampleController + * @skipline setServerComm + * + * @param comm The communicator that the server will be using to communicate with the other server cores. + */ + void setServerComm(MPI_Comm comm); + + /** + * Used to switch between server and threaded mode. + * True is server mode, false is threaded mode. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline writeController + * @until size-1 + * @skipline if + * @until { + * @skipline setServerMode + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline writeController + * @until size-1 + * @skipline setServerMode + * + * @param newMode The mode that the writer is to be set to. + */ + void setServerMode(bool newMode); + + /** + * Sets the comm that the workers will use to communicate with other worker cores + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleController + * @until size-1 + * @skipline if + * @until { + * @skipline getWorkerComm + * @skipline setWorkerComm + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until workerComm + * @skipline exampleController + * @until size-1 + * @skipline if + * @skip exampleController + * @skipline getWorkerComm + * @skipline setWorkerComm + * + * @param comm The communicator that the worker will be using to communicate with the other worker cores. + */ + void setWorkerComm(MPI_Comm comm); + + /** + * Sends a stop command to all the server cores that the controller is connected to, ending the DSM. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline closeFile + * @skipline if + * @until { + * @skipline exampleController + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until workerComm + * @skipline exampleController + * @until size-1 + * @skip Section + * @skipline if + * @skipline exampleController + * + */ + void stopDSM(); void read(XdmfArray * const array); + /** + * Restarts the DSM when called on server cores. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline closeFile + * @skipline if + * @until { + * @skipline exampleController + * @skipline } + * @skipline exampleController + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until workerComm + * @skipline exampleController + * @until size-1 + * @skip Section + * @skipline if + * @skipline exampleController + * @skip restartDSM + * @skipline exampleController + * + */ + void restartDSM(); + protected: XdmfHDF5ControllerDSM(const std::string & hdf5FilePath, @@ -386,6 +894,29 @@ protected: MPI_Comm comm, unsigned int bufferSize); + XdmfHDF5ControllerDSM(const std::string & hdf5FilePath, + const std::string & dataSetPath, + const shared_ptr type, + const std::vector & start, + const std::vector & stride, + const std::vector & dimensions, + const std::vector & dataspaceDimensions, + MPI_Comm comm, + unsigned int bufferSize, + int startCoreIndex, + int endCoreIndex); + + XdmfHDF5ControllerDSM(const std::string & hdf5FilePath, + const std::string & dataSetPath, + const shared_ptr type, + const std::vector & start, + const std::vector & stride, + const std::vector & dimensions, + const std::vector & dataspaceDimensions, + XdmfDSMBuffer * const dsmBuffer, + int startCoreIndex, + int endCoreIndex); + private: XdmfHDF5ControllerDSM(const XdmfHDF5Controller &); // Not implemented. @@ -393,6 +924,17 @@ private: H5FDdsmBuffer * mDSMBuffer; H5FDdsmManager * mDSMManager; + + XdmfDSMBuffer * mDSMServerBuffer; + XdmfDSMManager * mDSMServerManager; + MPI_Comm mGroupComm; + MPI_Comm mServerComm; + MPI_Comm mWorkerComm; + int mStartCoreIndex; + int mEndCoreIndex; + int mRank; + int mGroupSize; + bool mServerMode; }; #endif /* XDMFHDF5CONTROLLER_HPP_ */ diff --git a/core/XdmfHDF5Writer.cpp b/core/XdmfHDF5Writer.cpp index 605f9c8f..1a030f2c 100644 --- a/core/XdmfHDF5Writer.cpp +++ b/core/XdmfHDF5Writer.cpp @@ -34,7 +34,6 @@ #include "XdmfHDF5Controller.hpp" #include "XdmfHDF5Writer.hpp" - namespace { const static unsigned int DEFAULT_CHUNK_SIZE = 0; @@ -1435,9 +1434,9 @@ XdmfHDF5Writer::write(XdmfArray & array, partialArray->pushBack(array.getValue(j)); } } - if (partialArray->getSize()==0) { - break; - } +// if (partialArray->getSize()==0) { +// break; +// } arrayOffsetsWritten.push_back(controllerIndexOffset); controllerIndexOffset = j;//set the offset to the point after the end of the current subset @@ -1458,7 +1457,6 @@ XdmfHDF5Writer::write(XdmfArray & array, } - std::list::iterator fileNameWalker = filesWritten.begin(); std::list >::iterator arrayWalker = arraysWritten.begin(); std::list >::iterator startWalker = startsWritten.begin(); diff --git a/core/XdmfHDF5WriterDSM.cpp b/core/XdmfHDF5WriterDSM.cpp index eca46615..3aaa5ee2 100644 --- a/core/XdmfHDF5WriterDSM.cpp +++ b/core/XdmfHDF5WriterDSM.cpp @@ -25,8 +25,15 @@ #include #include #include +#include +#include +#include +#include +#include +#include #include "XdmfHDF5ControllerDSM.hpp" #include "XdmfHDF5WriterDSM.hpp" +#include "XdmfError.hpp" shared_ptr XdmfHDF5WriterDSM::New(const std::string & filePath, @@ -37,6 +44,19 @@ XdmfHDF5WriterDSM::New(const std::string & filePath, return p; } +shared_ptr +XdmfHDF5WriterDSM::New(const std::string & filePath, + XdmfDSMBuffer * const dsmBuffer, + int startCoreIndex, + int endCoreIndex) +{ + shared_ptr p(new XdmfHDF5WriterDSM(filePath, + dsmBuffer, + startCoreIndex, + endCoreIndex)); + return p; +} + shared_ptr XdmfHDF5WriterDSM::New(const std::string & filePath, MPI_Comm comm, @@ -48,11 +68,37 @@ XdmfHDF5WriterDSM::New(const std::string & filePath, return p; } +shared_ptr +XdmfHDF5WriterDSM::New(const std::string & filePath, + MPI_Comm comm, + unsigned int bufferSize, + int startCoreIndex, + int endCoreIndex) +{ + shared_ptr p(new XdmfHDF5WriterDSM(filePath, + comm, + bufferSize, + startCoreIndex, + endCoreIndex)); + return p; +} + XdmfHDF5WriterDSM::XdmfHDF5WriterDSM(const std::string & filePath, H5FDdsmBuffer * const dsmBuffer) : XdmfHDF5Writer(filePath), + mDSMManager(NULL), mDSMBuffer(dsmBuffer), - mFAPL(-1) + mFAPL(-1), + mDSMServerManager(NULL), + mDSMServerBuffer(NULL), + mGroupComm(MPI_COMM_NULL), + mServerComm(MPI_COMM_NULL), + mWorkerComm(MPI_COMM_NULL), + mStartCoreIndex(-1), + mEndCoreIndex(-1), + mRank(-1), + mGroupSize(-1), + mServerMode(false) { } @@ -60,7 +106,17 @@ XdmfHDF5WriterDSM::XdmfHDF5WriterDSM(const std::string & filePath, MPI_Comm comm, unsigned int bufferSize) : XdmfHDF5Writer(filePath), - mFAPL(-1) + mFAPL(-1), + mDSMServerManager(NULL), + mDSMServerBuffer(NULL), + mGroupComm(MPI_COMM_NULL), + mServerComm(MPI_COMM_NULL), + mWorkerComm(MPI_COMM_NULL), + mStartCoreIndex(-1), + mEndCoreIndex(-1), + mRank(-1), + mGroupSize(-1), + mServerMode(false) { H5FDdsmManager * newManager = new H5FDdsmManager(); newManager->SetMpiComm(comm); @@ -77,8 +133,146 @@ XdmfHDF5WriterDSM::XdmfHDF5WriterDSM(const std::string & filePath, mDSMBuffer = newBuffer; } + +//the database/nonthreaded version + +XdmfHDF5WriterDSM::XdmfHDF5WriterDSM(const std::string & filePath, + XdmfDSMBuffer * const dsmBuffer, + int startCoreIndex, + int endCoreIndex) : + XdmfHDF5Writer(filePath), + mDSMManager(NULL), + mDSMBuffer(NULL), + mFAPL(-1), + mDSMServerManager(NULL), + mDSMServerBuffer(dsmBuffer), + mServerMode(true), + mStartCoreIndex(startCoreIndex), + mEndCoreIndex(endCoreIndex) +{ + mGroupComm = mDSMServerBuffer->GetComm()->GetInterComm(); + MPI_Comm_rank(mGroupComm, &mRank); + MPI_Comm_size(mGroupComm, &mGroupSize); + if (mRank >=mStartCoreIndex && mRank <=mEndCoreIndex) { + mServerComm = mDSMServerBuffer->GetComm()->GetIntraComm(); + mWorkerComm = MPI_COMM_NULL; + } + else { + mServerComm = MPI_COMM_NULL; + mWorkerComm = mDSMServerBuffer->GetComm()->GetIntraComm(); + } +} + +XdmfHDF5WriterDSM::XdmfHDF5WriterDSM(const std::string & filePath, + MPI_Comm comm, + unsigned int bufferSize, + int startCoreIndex, + int endCoreIndex) : + XdmfHDF5Writer(filePath), + mFAPL(-1), + mDSMManager(NULL), + mDSMBuffer(NULL), + mServerMode(true) +{ + //negative values will be changed to maximum range + if (startCoreIndex < 0) + { + startCoreIndex = 0; + } + if (endCoreIndex < 0) + { + endCoreIndex = mGroupSize - 1; + } + + //ensure start index is less than end index + if (startCoreIndex > endCoreIndex) + { + int tempholder = startCoreIndex; + startCoreIndex = endCoreIndex; + endCoreIndex = tempholder; + } + + mGroupComm = comm; + mStartCoreIndex = startCoreIndex; + mEndCoreIndex = endCoreIndex; + + MPI_Comm_size(comm, &mGroupSize); + MPI_Comm_rank(comm, &mRank); + + MPI_Group workers, dsmgroup, serversplit, servergroup; + + int * ServerIds = (int *)calloc((mEndCoreIndex - mStartCoreIndex + 1), sizeof(int)); + unsigned int index = 0; + for(int i=mStartCoreIndex ; i <= mEndCoreIndex ; ++i) + { + ServerIds[index++] = i; + } + + MPI_Comm_group(comm, &serversplit); + MPI_Group_incl(serversplit, index, ServerIds, &servergroup); + MPI_Comm_create(comm, servergroup, &mServerComm); + MPI_Comm_group(comm, &dsmgroup); + MPI_Group_excl(dsmgroup, index, ServerIds, &workers); + MPI_Comm_create(comm, workers, &mWorkerComm); + cfree(ServerIds); + + //create the manager + + mDSMServerManager = new XdmfDSMManager(); + + mDSMServerManager->SetLocalBufferSizeMBytes(bufferSize); + mDSMServerManager->SetInterCommType(H5FD_DSM_COMM_MPI); + + MPI_Barrier(mGroupComm); + + if (mRank >=mStartCoreIndex && mRank <=mEndCoreIndex) + { + mDSMServerManager->SetMpiComm(mServerComm); + mDSMServerManager->Create(); + } + else + { + mDSMServerManager->SetMpiComm(mWorkerComm); + mDSMServerManager->SetIsServer(false); + mDSMServerManager->Create(mStartCoreIndex, mEndCoreIndex); + } + + XDMF_dsm_set_manager(mDSMServerManager); + + mDSMServerBuffer = mDSMServerManager->GetDsmBuffer(); + + mDSMServerBuffer->GetComm()->DupInterComm(mGroupComm); + mDSMServerBuffer->SetIsConnected(true); + + MPI_Barrier(comm); + + //loop needs to be started before anything can be done to the file, since the service is what sets up the file + + if (mRank < mStartCoreIndex || mRank > mEndCoreIndex) + { + //turn off the server designation + mDSMServerBuffer->SetIsServer(H5FD_DSM_FALSE);//if this is set to false then the buffer will attempt to connect to the intercomm for DSM stuff + mDSMServerManager->SetIsServer(H5FD_DSM_FALSE); + } + else + { + //on cores where memory is set up, start the service loop + //this should iterate infinitely until a value to end the loop is passed + H5FDdsmInt32 returnOpCode; + try + { + mDSMServerBuffer->BufferServiceLoop(&returnOpCode); + } + catch (XdmfError e) + { + throw e; + } + } +} + XdmfHDF5WriterDSM::~XdmfHDF5WriterDSM() { + } shared_ptr @@ -90,14 +284,28 @@ XdmfHDF5WriterDSM::createHDF5Controller(const std::string & hdf5FilePath, const std::vector & dimensions, const std::vector & dataspaceDimensions) { - return XdmfHDF5ControllerDSM::New(hdf5FilePath, - dataSetPath, - type, - start, - stride, - dimensions, - dataspaceDimensions, - mDSMBuffer); + if (mDSMServerBuffer != NULL) { + return XdmfHDF5ControllerDSM::New(hdf5FilePath, + dataSetPath, + type, + start, + stride, + dimensions, + dataspaceDimensions, + mDSMServerBuffer, + mStartCoreIndex, + mEndCoreIndex); + } + else { + return XdmfHDF5ControllerDSM::New(hdf5FilePath, + dataSetPath, + type, + start, + stride, + dimensions, + dataspaceDimensions, + mDSMBuffer); + } } H5FDdsmManager * XdmfHDF5WriterDSM::getManager() @@ -110,6 +318,35 @@ H5FDdsmBuffer * XdmfHDF5WriterDSM::getBuffer() return mDSMBuffer; } +XdmfDSMManager * XdmfHDF5WriterDSM::getServerManager() +{ + return mDSMServerManager; +} + +XdmfDSMBuffer * XdmfHDF5WriterDSM::getServerBuffer() +{ + return mDSMServerBuffer; +} + +bool XdmfHDF5WriterDSM::getServerMode() +{ + return mServerMode; +} + +MPI_Comm XdmfHDF5WriterDSM::getWorkerComm() +{ + MPI_Comm returnComm; + int status = MPI_Comm_dup(mWorkerComm, &returnComm); + return returnComm; +} + +MPI_Comm XdmfHDF5WriterDSM::getServerComm() +{ + MPI_Comm returnComm; + int status = MPI_Comm_dup(mServerComm, &returnComm); + return returnComm; +} + void XdmfHDF5WriterDSM::setManager(H5FDdsmManager * newManager) { H5FDdsmBuffer * newBuffer = newManager->GetDsmBuffer(); @@ -122,12 +359,140 @@ void XdmfHDF5WriterDSM::setBuffer(H5FDdsmBuffer * newBuffer) mDSMBuffer = newBuffer; } +void XdmfHDF5WriterDSM::setManager(XdmfDSMManager * newManager) +{ + XdmfDSMBuffer * newBuffer = newManager->GetDsmBuffer(); + mDSMServerManager = newManager; + mDSMServerBuffer = newBuffer; +} + +void XdmfHDF5WriterDSM::setBuffer(XdmfDSMBuffer * newBuffer) +{ + mDSMServerBuffer = newBuffer; +} + +void XdmfHDF5WriterDSM::setServerMode(bool newMode) +{ + mServerMode = newMode; +} + +void XdmfHDF5WriterDSM::setWorkerComm(MPI_Comm comm) +{ + int status; + if (mWorkerComm != MPI_COMM_NULL) + { + status = MPI_Comm_free(&mWorkerComm); + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Failed to disconnect Comm"); + } + catch (XdmfError e) + { + throw e; + } + } + } + if (comm != MPI_COMM_NULL) + { + status = MPI_Comm_dup(comm, &mWorkerComm); + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Failed to duplicate Comm"); + } + catch (XdmfError e) + { + throw e; + } + } + } + mDSMServerBuffer->GetComm()->DupComm(comm); +} + +void XdmfHDF5WriterDSM::setServerComm(MPI_Comm comm) +{ + int status; + if (mServerComm != MPI_COMM_NULL) + { + status = MPI_Comm_free(&mServerComm); + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Failed to disconnect Comm"); + } + catch (XdmfError e) + { + throw e; + } + } + } + if (comm != MPI_COMM_NULL) + { + status = MPI_Comm_dup(comm, &mServerComm); + if (status != MPI_SUCCESS) + { + try + { + XdmfError::message(XdmfError::FATAL, "Failed to duplicate Comm"); + } + catch (XdmfError e) + { + throw e; + } + } + } + mDSMServerBuffer->GetComm()->DupComm(comm); +} + void XdmfHDF5WriterDSM::deleteManager() { if (mDSMManager != NULL) { delete mDSMManager; } + if (mDSMServerManager != NULL) + { + closeFile(); + delete mDSMServerManager; + } +} + +void XdmfHDF5WriterDSM::stopDSM() +{ + //send manually + for (int i = mStartCoreIndex; i <= mEndCoreIndex; ++i) + { + try + { + mDSMServerBuffer->SendCommandHeader(H5FD_DSM_OPCODE_DONE, i, 0, 0, H5FD_DSM_INTER_COMM); + } + catch (XdmfError e) + { + throw e; + } + //originally this was set to the intra comm + //that doesn't work in this instance because it won't reach the server cores + } +} + +void XdmfHDF5WriterDSM::restartDSM() +{ + if (mRank >= mStartCoreIndex && mRank <= mEndCoreIndex) + { + H5FDdsmInt32 returnOpCode; + try + { + mDSMServerBuffer->BufferServiceLoop(&returnOpCode); + } + catch (XdmfError e) + { + throw e; + } + } } void @@ -150,9 +515,14 @@ XdmfHDF5WriterDSM::openFile() // Set file access property list for DSM mFAPL = H5Pcreate(H5P_FILE_ACCESS); - // Use DSM driver - H5Pset_fapl_dsm(mFAPL, MPI_COMM_WORLD, mDSMBuffer, 0); - + if (mServerMode) { + if (mWorkerComm != MPI_COMM_NULL) { + XDMFH5Pset_fapl_dsm(mFAPL, mWorkerComm, mDSMServerBuffer, 0); + } + } + else { + H5Pset_fapl_dsm(mFAPL, MPI_COMM_WORLD, mDSMBuffer, 0); + } XdmfHDF5Writer::openFile(mFAPL); } @@ -160,14 +530,19 @@ void XdmfHDF5WriterDSM::visit(XdmfArray & array, const shared_ptr) { bool closeFAPL = false; - - if(mFAPL < 0) { + if(mFAPL < 0) { // Set file access property list for DSM mFAPL = H5Pcreate(H5P_FILE_ACCESS); - // Use DSM driver - H5Pset_fapl_dsm(mFAPL, MPI_COMM_WORLD, mDSMBuffer, 0); + if (mServerMode) { + if (mWorkerComm != MPI_COMM_NULL) { + XDMFH5Pset_fapl_dsm(mFAPL, mWorkerComm, mDSMServerBuffer, 0); + } + } + else { + H5Pset_fapl_dsm(mFAPL, MPI_COMM_WORLD, mDSMBuffer, 0); + } closeFAPL = true; } @@ -180,4 +555,5 @@ void XdmfHDF5WriterDSM::visit(XdmfArray & array, herr_t status = H5Pclose(mFAPL); mFAPL = -1; } + } diff --git a/core/XdmfHDF5WriterDSM.hpp b/core/XdmfHDF5WriterDSM.hpp index 1a89f000..9316d59f 100644 --- a/core/XdmfHDF5WriterDSM.hpp +++ b/core/XdmfHDF5WriterDSM.hpp @@ -30,6 +30,9 @@ class H5FDdsmManager; // Includes #include "XdmfHDF5Writer.hpp" +#include +#include +#include /** * @brief Traverse the Xdmf graph and write heavy data stored in @@ -69,7 +72,7 @@ public: * @skipline array * @until initialize * @skipline for - * @until setHeavyDataController + * @until insert * @skipline accept * * Python @@ -83,7 +86,7 @@ public: * @skipline writeController * @until exampleWriter * @skipline setMode - * @until setHeavyDataController + * @until insert * @skipline accept * @skipline segfault * @skipline exampleWriter @@ -102,7 +105,7 @@ public: * Currently the write requires all cores to write and will freeze otherwise. This version creates a DSM buffer in the provided com of the size provided. * * When created the manager has the following defaults: - * IsStandAlone =H5FD_DSM_TRUE + * IsStandAlone = H5FD_DSM_TRUE * H5FD_DSM_LOCK_ASYNCHRONOUS * * Example of use: @@ -119,7 +122,7 @@ public: * @skipline writeController * @until getBuffer * @skipline setMode - * @until setHeavyDataController + * @until insert * @skipline accept * @skipline dsmManager * @skipline exampleWriter->getManager() @@ -132,7 +135,7 @@ public: * @skipline coreSize * @until getBuffer * @skipline setMode - * @until setHeavyDataController + * @until insert * @skipline accept * @skipline segfault * @skipline exampleWriter @@ -146,12 +149,82 @@ public: New(const std::string & filePath, MPI_Comm comm, unsigned int bufferSize); + /** + * Contruct XdmfHDF5WriterDSM, nonthreaded version + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleWriter2 + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleWriter2 + * + * @param filePath The location of the hdf5 file to output to on disk. + * @param dsmBuffer The Buffer to write to. + * @param startCoreIndex The index of the first core in the server block + * @param endCoreIndex The index of the last core in the server block. + * @return a New XdmfHDF5WriterDSM + */ + static shared_ptr + New(const std::string & filePath, + XdmfDSMBuffer * const dsmBuffer, + int startCoreIndex, + int endCoreIndex); + + /** + * Contruct XdmfHDF5WriterDSM, nonthreaded version + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * + * Python + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * + * @param filePath The location of the hdf5 file to output to on disk. + * @param comm the communicator that the buffer will be created in. + * @param bufferSize the size of the created buffer. + * @param startCoreIndex The index of the first core in the server block + * @param endCoreIndex The index of the last core in the server block + * @return a New XdmfHDF5WriterDSM + */ + static shared_ptr + New(const std::string & filePath, + MPI_Comm comm, + unsigned int bufferSize, + int startCoreIndex, + int endCoreIndex); virtual ~XdmfHDF5WriterDSM(); /** - * Returns the current dsmManager for the Writer. If there is no manager then it returns null + * Deletes the manager that the writer contains. + * Used during cleanup. * * Example of Use: * @@ -160,9 +233,13 @@ public: * @dontinclude ExampleXdmfDSMSelfcontained.cpp * @skipline providedThreading * @until MPI_Comm_size + * @skipline testArray + * @until newSetPath * @skipline exampleWriter - * @skipline dsmManager - * @skipline exampleWriter->getManager() + * @skip /* + * @skipline writeController + * @until getBuffer + * @skipline exampleWriter->deleteManager * * Python * @@ -170,16 +247,16 @@ public: * @skipline comm * @until size*4 * @skipline coreSize - * @until setManager + * @until getBuffer * @skipline setMode - * @until setHeavyDataController + * @until insert * @skipline accept * @skipline segfault * @skipline exampleWriter - * - * @return the dsmManager of the Writer */ - H5FDdsmManager * getManager(); + void deleteManager(); + + void closeFile(); /** * Returns the current dsmBuffer the Writer. If there is no manager then it returns null @@ -207,7 +284,7 @@ public: * @skipline coreSize * @until setBuffer * @skipline setMode - * @until setHeavyDataController + * @until insert * @skipline accept * @skipline segfault * @skipline exampleWriter @@ -217,8 +294,7 @@ public: H5FDdsmBuffer * getBuffer(); /** - * Sets the Writer's dsmManager to the provided manager. - * Then the dsmBuffer controlled by the manager is set to the Writer + * Returns the current dsmManager for the Writer. If there is no manager then it returns null * * Example of Use: * @@ -227,12 +303,8 @@ public: * @dontinclude ExampleXdmfDSMSelfcontained.cpp * @skipline providedThreading * @until MPI_Comm_size - * @skipline testArray - * @until newSetPath * @skipline exampleWriter - * @skip /* - * @skipline writeController - * @until setManager + * @skipline dsmManager * @skipline exampleWriter->getManager() * * Python @@ -241,19 +313,165 @@ public: * @skipline comm * @until size*4 * @skipline coreSize - * @until If - * @skip delete - * @skipline writeController * @until setManager * @skipline setMode - * @until setHeavyDataController + * @until insert * @skipline accept * @skipline segfault * @skipline exampleWriter * - * @param newManager the manager to be set + * @return the dsmManager of the Writer */ - void setManager(H5FDdsmManager * newManager); + H5FDdsmManager * getManager(); + + /** + * Gets the buffer for the non-threaded version of DSM + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleBuffer + * @skipline exampleWriter + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * + * @return The XdmfDSMBuffer that is controlling the data for the DSM + */ + XdmfDSMBuffer * getServerBuffer(); + + /** + * Gets the communicator that the servers use to communicate between themselves. + * Will be MPI_COMM_NULL on worker cores. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline getServerComm + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline getServerComm + * + * @return The comm that the servers are using. + */ + MPI_Comm getServerComm(); + + /** + * Gets the manager for the non-threaded version of DSM + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleManager + * @skipline exampleWriter + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleManager + * + * @return The XdmfDSMManager that is controlling the DSM + */ + XdmfDSMManager * getServerManager(); + + /** + * Checks if the DSM is in server mode or not. + * True is server mode, false is threaded + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline getServerMode + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline getServerMode + * + * @return If the DSM is in server mode or not + */ + bool getServerMode(); + + /** + * Gets the Communicator that the workers are using to communicate between themselves + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline getWorkerComm + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline getWorkerComm + * + * @return The comm that the workers are using. + */ + MPI_Comm getWorkerComm(); /** * Sets the Writer's dsmBuffer to the provided buffer @@ -284,7 +502,7 @@ public: * @skipline writeController * @until setBuffer * @skipline setMode - * @until setHeavyDataController + * @until insert * @skipline accept * @skipline segfault * @skipline exampleWriter @@ -294,8 +512,40 @@ public: void setBuffer(H5FDdsmBuffer * newBuffer); /** - * Deletes the manager that the writer contains. - * Used during cleanup. + * Sets the Writer's dsmBuffer to the provided buffer + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleBuffer + * @skipline exampleWriter + * @skipline setBuffer + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleBuffer + * @skipline SetBuffer + * + * @param newBuffer A pointer to the buffer to be set + */ + void setBuffer(XdmfDSMBuffer * newBuffer); + + /** + * Sets the Writer's dsmManager to the provided manager. + * Then the dsmBuffer controlled by the manager is set to the Writer * * Example of Use: * @@ -309,8 +559,8 @@ public: * @skipline exampleWriter * @skip /* * @skipline writeController - * @until getBuffer - * @skipline exampleWriter->deleteManager + * @until setManager + * @skipline exampleWriter->getManager() * * Python * @@ -318,16 +568,201 @@ public: * @skipline comm * @until size*4 * @skipline coreSize - * @until getBuffer + * @until If + * @skip delete + * @skipline writeController + * @until setManager * @skipline setMode - * @until setHeavyDataController + * @until insert * @skipline accept * @skipline segfault * @skipline exampleWriter + * + * @param newManager the manager to be set */ - void deleteManager(); + void setManager(H5FDdsmManager * newManager); - void closeFile(); + /** + * Sets the Writer's dsmManager to the provided manager. + * Then the dsmBuffer controlled by the manager is set to the Writer + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline exampleManager + * @skipline exampleWriter + * @skipline setManager + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline exampleManager + * @skipline SetManager + * + * @param newManager A pointer the the manager to be set. + */ + void setManager(XdmfDSMManager * newManager); + /** + * Sets the comm that the servers will use to communicate with the other server cores. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline getServerComm + * @until setServerComm + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline getServerComm + * @skipline setServerComm + * + * @param comm The communicator that the server will be using to communicate with the other server cores. + */ + void setServerComm(MPI_Comm comm); + + /** + * Used to switch between server and threaded mode. + * True is server mode, false is threaded mode. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline setServerMode + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline setServerMode + * + * @param newMode The mode that the writer is to be set to. + */ + void setServerMode(bool newMode); + + /** + * Sets the comm that the workers will use to communicate with other worker cores + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline if + * @until { + * @skipline getWorkerComm + * @until setWorkerComm + * @skip Section + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skipline if + * @skipline getWorkerComm + * @skipline setWorkerComm + * + * @param comm The communicator that the worker will be using to communicate with the other worker cores. + */ + void setWorkerComm(MPI_Comm comm); + + /** + * Sends a stop command to all the server cores that the writer is connected to, ending the DSM. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline closeFile + * @skipline if + * @until stopDSM + * @skipline } + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skip Section + * @skipline if + * @skipline exampleWriter + * + */ + void stopDSM(); + + /** + * Restarts the DSM when called on server cores. + * + * Example of use: + * + * C++ + * + * @dontinclude ExampleXdmfDSMNoThread.cpp + * @skipline size + * @until MPI_Comm_size + * @skipline exampleWriter + * @skipline closeFile + * @skipline if + * @until stopDSM + * @skipline } + * @skipline restartDSM + * + * Python + * + * @dontinclude XdmfExampleDSMNoThread.cpp + * @skipline total + * @until exampleWriter + * @skip Section + * @skipline if + * @skipline exampleWriter + * @skipline restartDSM + * + */ + void restartDSM(); void openFile(); @@ -344,6 +779,17 @@ protected: MPI_Comm comm, unsigned int bufferSize); + XdmfHDF5WriterDSM(const std::string & filePath, + XdmfDSMBuffer * const dsmBuffer, + int startCoreIndex, + int endCoreIndex); + + XdmfHDF5WriterDSM(const std::string & filePath, + MPI_Comm comm, + unsigned int bufferSize, + int startCoreIndex, + int endCoreIndex); + virtual shared_ptr createHDF5Controller(const std::string & hdf5FilePath, const std::string & dataSetPath, @@ -362,6 +808,17 @@ private: H5FDdsmManager * mDSMManager; int mFAPL; + XdmfDSMBuffer * mDSMServerBuffer; + XdmfDSMManager * mDSMServerManager; + MPI_Comm mGroupComm; + MPI_Comm mServerComm; + MPI_Comm mWorkerComm; + int mStartCoreIndex; + int mEndCoreIndex; + int mRank; + int mGroupSize; + bool mServerMode; + }; #endif /* XDMFHDF5WRITERDSM_HPP_ */ diff --git a/core/XdmfWriter.cpp b/core/XdmfWriter.cpp index 8224355d..0df47eb1 100644 --- a/core/XdmfWriter.cpp +++ b/core/XdmfWriter.cpp @@ -71,10 +71,7 @@ public: void closeFile() { - // remove xpath info as it will not be valid for the new file mXPath.clear(); - mXPathCount = 0; - mXPathString = ""; //this section writes to file std::ofstream fileStream; diff --git a/core/tests/Cxx/TestXdmfHDF5WriterDSM.cpp b/core/tests/Cxx/TestXdmfHDF5WriterDSM.cpp index f593a5e0..60db98f6 100644 --- a/core/tests/Cxx/TestXdmfHDF5WriterDSM.cpp +++ b/core/tests/Cxx/TestXdmfHDF5WriterDSM.cpp @@ -62,12 +62,12 @@ int main(int argc, char *argv[]) std::vector(1, 1), std::vector(1, 3), std::vector(1, size*3), - NULL); + dsmBuffer); array->setHeavyDataController(controller); // Create DSM Writer and write to DSM space. shared_ptr writer = - XdmfHDF5WriterDSM::New("dsm.h5", NULL); + XdmfHDF5WriterDSM::New("dsm.h5", dsmBuffer); writer->setMode(XdmfHeavyDataWriter::Hyperslab); array->accept(writer); diff --git a/examples/Cxx/DSMLoopTest.cpp b/examples/Cxx/DSMLoopTest.cpp new file mode 100644 index 00000000..d3b0096d --- /dev/null +++ b/examples/Cxx/DSMLoopTest.cpp @@ -0,0 +1,310 @@ +#include +#include +#include +#include +#include +#include "XdmfArray.hpp" +#include "XdmfArrayType.hpp" +#include "XdmfHDF5WriterDSM.hpp" +#include "XdmfHDF5ControllerDSM.hpp" + +int main(int argc, char *argv[]) +{ + int size, id, dsmSize; + dsmSize = 64;//The total size of the DSM being created + MPI_Status status; + MPI_Comm comm = MPI_COMM_WORLD; + + MPI_Init(&argc, &argv); + + MPI_Comm_rank(comm, &id); + MPI_Comm_size(comm, &size); + + + std::vector outputVector; + + shared_ptr testArray = XdmfArray::New(); + testArray->initialize(0); + + std::string newPath = "dsm"; + std::string newSetPath = "data"; + + // Change this to determine the number of cores used as servers + int numServersCores = 8; + // Change this to determine the size of the arrays generated when initializing + int writeArraySize = 4; + + std::vector writeStartVector; + std::vector writeStrideVector; + std::vector writeCountVector; + std::vector writeDataSizeVector; + + std::vector readStartVector; + std::vector readStrideVector; + std::vector readCountVector; + std::vector readDataSizeVector; + + std::vector readOutputCountVector; + + shared_ptr readArray = XdmfArray::New(); + readArray->initialize(0); + + shared_ptr readController; + shared_ptr readOutputController; + shared_ptr writeController; + + MPI_Comm workerComm; + + MPI_Group workers, dsmgroup; + + MPI_Comm_group(comm, &dsmgroup); + int * ServerIds = (int *)calloc((numServersCores), sizeof(int)); + unsigned int index = 0; + for(int i=size-numServersCores ; i <= size-1 ; ++i) + { + ServerIds[index++] = i; + } + + MPI_Group_excl(dsmgroup, index, ServerIds, &workers); + int testval = MPI_Comm_create(comm, workers, &workerComm); + cfree(ServerIds); + + shared_ptr exampleWriter = XdmfHDF5WriterDSM::New(newPath, comm, dsmSize/numServersCores, size-numServersCores, size-1); + + exampleWriter->setMode(XdmfHeavyDataWriter::Hyperslab); + + //split out sub-comm for the worker cores + //server cores will not progress to this point until after the servers are done running + + if (id < size - numServersCores) + { + // Split the comm even further + + MPI_Comm readComm, writeComm; + + MPI_Group readingCores, writingCores; + + MPI_Comm_group(workerComm, &workers); + int * ServerIds = (int *)calloc(((size - numServersCores) / 2), sizeof(int)); + unsigned int index = 0; + for(int i=0 ; i < (int)((size - numServersCores) / 2) ; ++i) + { + ServerIds[index++] = i; + } + + MPI_Group_excl(workers, index, ServerIds, &readingCores); + testval = MPI_Comm_create(workerComm, readingCores, &readComm); + MPI_Group_incl(workers, index, ServerIds, &writingCores); + testval = MPI_Comm_create(workerComm, writingCores, &writeComm); + cfree(ServerIds); + + std::cout << "initializing" << std::endl; + + // Initialize values + // Writer is first + if (id < (int)((size - numServersCores) / 2)) + { + for (unsigned int i = 1; i <= writeArraySize; ++i) + { + testArray->pushBack(i*(id+1)); + } + writeStartVector.push_back(id*writeArraySize); + writeStrideVector.push_back(1); + writeCountVector.push_back(writeArraySize); + writeDataSizeVector.push_back(writeArraySize*(int)((size-numServersCores) / 2)); + writeController = XdmfHDF5ControllerDSM::New( + newPath, + newSetPath, + XdmfArrayType::Int32(), + writeStartVector, + writeStrideVector, + writeCountVector, + writeDataSizeVector, + exampleWriter->getServerBuffer(), + size-numServersCores, + size-1); + + testArray->insert(writeController); + + readStartVector.push_back(0); + readStrideVector.push_back(1); + readCountVector.push_back(0); + readDataSizeVector.push_back(writeArraySize*(int)((size-numServersCores) / 2)); + + readController = XdmfHDF5ControllerDSM::New( + newPath, + newSetPath, + XdmfArrayType::Int32(), + readStartVector, + readStrideVector, + readCountVector, + readDataSizeVector, + exampleWriter->getServerBuffer(), + size-numServersCores, + size-1); + + readArray->insert(readController); + + for (unsigned int i = 0; igetSize(); ++i) + { + std::cout << "core #" << id <<" testArray[" << i << "] = " << testArray->getValue(i) << std::endl; + } + } + } + } + else + { + // Reader is second + readStartVector.push_back(0); + readStrideVector.push_back(1); + readCountVector.push_back(writeArraySize*(int)((size-numServersCores) / 2)); + readDataSizeVector.push_back(writeArraySize*(int)((size-numServersCores) / 2)); + + readController = XdmfHDF5ControllerDSM::New( + newPath, + newSetPath, + XdmfArrayType::Int32(), + readStartVector, + readStrideVector, + readCountVector, + readDataSizeVector, + exampleWriter->getServerBuffer(), + size-numServersCores, + size-1); + + readArray->insert(readController); + + readOutputCountVector.push_back(0); + + readOutputController = XdmfHDF5ControllerDSM::New( + newPath, + newSetPath, + XdmfArrayType::Int32(), + readStartVector, + readStrideVector, + readOutputCountVector, + readDataSizeVector, + exampleWriter->getServerBuffer(), + size-numServersCores, + size-1);; + + writeStartVector.push_back(0); + writeStrideVector.push_back(1); + writeCountVector.push_back(0); + writeDataSizeVector.push_back(writeArraySize*(int)((size-numServersCores) / 2)); + writeController = XdmfHDF5ControllerDSM::New( + newPath, + newSetPath, + XdmfArrayType::Int32(), + writeStartVector, + writeStrideVector, + writeCountVector, + writeDataSizeVector, + exampleWriter->getServerBuffer(), + size-numServersCores, + size-1); + + testArray->insert(writeController); + } + + testArray->accept(exampleWriter); + + MPI_Barrier(workerComm); + + // This is the loop that manipulates the data + for (unsigned int iteration = 0; iteration < 10; ++iteration) + { + std::cout << "loop iteration " << iteration << " on core " << id << std::endl; + + // Read in to the first set of cores + if (id >= (int)((size - numServersCores) / 2)) + { + // Controllers are accessed like this since the writer removes them and creates its own. + shared_dynamic_cast(readArray->getHeavyDataController(0))->setWorkerComm(readComm); + printf("read on core %d\n", id); + readArray->read(); + MPI_Barrier(readComm); + + if (id == (int)((size - numServersCores) / 2)) + { + for(unsigned int i=0; igetSize(); ++i) + { + int tempVal = readArray->getValue(i); + tempVal = tempVal * 2; + readArray->insert(i, tempVal); + // Pull the value from the array in order to ensure the change has happened + std::cout << "core #" << id <<" readArray[" << i << "] = " << readArray->getValue(i) << std::endl; + } + } + } + shared_dynamic_cast(readArray->getHeavyDataController(0))->setWorkerComm(workerComm); + + if (id > (int)((size - numServersCores) / 2)) + { + // Only the first read core should write out data + readArray->removeHeavyDataController(0); + readArray->insert(readOutputController); + } + + readArray->accept(exampleWriter); + + if (id > (int)((size - numServersCores) / 2)) + { + // Only the first read core should write out data + readArray->removeHeavyDataController(0); + readArray->insert(readController); + } + + MPI_Barrier(workerComm); + + std::cout << "on writing cores" << std::endl; + + if (id < (int)((size - numServersCores) / 2)) + { + shared_dynamic_cast(testArray->getHeavyDataController(0))->setWorkerComm(writeComm); + testArray->read(); + MPI_Barrier(writeComm); + for (unsigned int i = 0; igetSize(); ++i) + { + int tempVal = testArray->getValue(i); + tempVal = tempVal * 3; + testArray->insert(i, tempVal); + // Pull the value from the array in order to ensure the change has happened + std::cout << "core #" << id <<" testArray[" << i << "] = " << testArray->getValue(i) << std::endl; + } + } + } + } + writeController->setWorkerComm(workerComm); + testArray->accept(exampleWriter); + } + + } + + + + + if (id == 0) + { + exampleWriter->stopDSM(); + } + + MPI_Barrier(comm); + + //the dsmManager must be deleted or else there will be a segfault + exampleWriter->deleteManager(); + + MPI_Finalize(); + + return 0; +} diff --git a/examples/Cxx/ExampleXdmfDSMNoThread.cpp b/examples/Cxx/ExampleXdmfDSMNoThread.cpp new file mode 100644 index 00000000..72755a0a --- /dev/null +++ b/examples/Cxx/ExampleXdmfDSMNoThread.cpp @@ -0,0 +1,537 @@ +#include +#include +#include +#include "XdmfArray.hpp" +#include "XdmfArrayType.hpp" +#include "XdmfHDF5WriterDSM.hpp" +#include "XdmfHDF5ControllerDSM.hpp" +#include "XdmfError.hpp" + +int main(int argc, char *argv[]) +{ + int size, id, dsmSize; + dsmSize = 64;//The total size of the DSM being created + MPI_Status status; + MPI_Comm comm = MPI_COMM_WORLD; + + MPI_Init(&argc, &argv); + + MPI_Comm_rank(comm, &id); + MPI_Comm_size(comm, &size); + + + std::vector outputVector; + + shared_ptr testArray = XdmfArray::New(); + + for (unsigned int i = 1; i <= 4; ++i) + { + testArray->pushBack(i*(id+1)); + } + + std::string newPath = "dsm"; + std::string newSetPath = "data"; + + int numServersCores = 4; + + std::vector writeStartVector; + writeStartVector.push_back(id*4); + //writeStartVector.push_back(id); + std::vector writeStrideVector; + writeStrideVector.push_back(1); + //writeStrideVector.push_back(size-3); + std::vector writeCountVector; + writeCountVector.push_back(4); + std::vector writeDataSizeVector; + writeDataSizeVector.push_back(4*(size-numServersCores)); + + MPI_Comm workerComm; + + MPI_Group workers, dsmgroup; + + MPI_Comm_group(comm, &dsmgroup); + int * ServerIds = (int *)calloc((numServersCores), sizeof(int)); + unsigned int index = 0; + for(int i=size-numServersCores ; i <= size-1 ; ++i) + { + ServerIds[index++] = i; + } + + MPI_Group_excl(dsmgroup, index, ServerIds, &workers); + int testval = MPI_Comm_create(comm, workers, &workerComm); + cfree(ServerIds); + + + shared_ptr exampleWriter = XdmfHDF5WriterDSM::New(newPath, comm, dsmSize/numServersCores, size-numServersCores, size-1); +/* + shared_ptr exampleController = XdmfHDF5ControllerDSM::New( + newPath, + newSetPath, + XdmfArrayType::Int32(), + writeStartVector, + writeStrideVector, + writeCountVector, + writeDataSizeVector, + comm, + dsmSize/numServersCores, + size-numServersCores, + size-1); +*/ + + // Split out sub-comm for the worker cores + // Server cores will not progress to this point until after the servers are done running + + if (id < size - numServersCores) + { + // This section is to demonstrate the functionality of the XdmfDSM classes + + exampleWriter->setServerMode(true); + bool exampleServerMode = exampleWriter->getServerMode(); + + /* + MPI_Comm exampleServerComm = exampleWriter->getServerComm(); + exampleWriter->setServerComm(exampleServerComm); + */ + + MPI_Comm exampleWorkerComm = exampleWriter->getWorkerComm(); + exampleWriter->setWorkerComm(exampleWorkerComm); + +/* + MPI_Comm exampleServerComm = exampleController->getServerComm(); + exampleController->setServerComm(exampleServerComm); + + MPI_Comm exampleWorkerComm = exampleController->getWorkerComm(); + exampleController->setWorkerComm(exampleWorkerComm); +*/ + + shared_ptr writeController = XdmfHDF5ControllerDSM::New( + newPath, + newSetPath, + XdmfArrayType::Int32(), + writeStartVector, + writeStrideVector, + writeCountVector, + writeDataSizeVector, + exampleWriter->getServerBuffer(), + size-numServersCores, + size-1); + + shared_ptr exampleWriter2 = XdmfHDF5WriterDSM::New(newPath, exampleWriter->getServerBuffer(), size-numServersCores, size-1); + + writeController->setServerMode(true); + bool exampleControllerServerMode = writeController->getServerMode(); + + XdmfDSMManager * exampleManager; + + exampleManager = exampleWriter->getServerManager(); + exampleWriter->setManager(exampleManager); +/* + exampleManager = exampleController->getServerManager(); + exampleController->setManager(exampleManager); +*/ + + int intraId = exampleManager->GetUpdatePiece(); + int intraSize = exampleManager->GetUpdateNumPieces(); + + MPI_Comm exampleManagerComm = exampleManager->GetMpiComm(); + exampleManager->SetMpiComm(exampleManagerComm); + + int exampleBufferSize = exampleManager->GetLocalBufferSizeMBytes(); + exampleManager->SetLocalBufferSizeMBytes(exampleBufferSize); + + bool exampleIsServer = exampleManager->GetIsServer(); + exampleManager->SetIsServer(exampleIsServer); + + int exampleType = exampleManager->GetDsmType(); + exampleManager->SetDsmType(XDMF_DSM_TYPE_UNIFORM); + + long exampleBlockLength = exampleManager->GetBlockLength(); + exampleManager->SetBlockLength(exampleBlockLength); + + int exampleCommType = exampleManager->GetInterCommType(); + exampleManager->SetInterCommType(XDMF_DSM_COMM_MPI); + + bool exampleManagerConnectionStatus = exampleManager->GetIsConnected(); + + /* + exampleManager->Destroy(); + exampleManager->Create(size - numServerCores, size - 1); + */ + + XdmfDSMBuffer * exampleBuffer; + + exampleBuffer = exampleWriter->getServerBuffer(); + exampleWriter->setBuffer(exampleBuffer); +/* + exampleBuffer = exampleController->getServerBuffer(); + exampleController->setBuffer(exampleBuffer); +*/ + exampleManager->SetDsmBuffer(exampleBuffer); + exampleBuffer = exampleManager->GetDsmBuffer(); + + bool exampleIsConnected = exampleBuffer->GetIsConnected(); + exampleBuffer->SetIsConnected(exampleIsConnected); + + char * exampleDataPointer = exampleBuffer->GetDataPointer(); + + int exampleDSMType = exampleBuffer->GetDsmType(); + exampleBuffer->SetDsmType(XDMF_DSM_TYPE_UNIFORM); + + bool exampleBufferIsServer = exampleBuffer->GetIsServer(); + exampleBuffer->SetIsServer(exampleIsServer); + + int exampleBufferStart = exampleBuffer->GetStartAddress(); + int exampleBufferEnd = exampleBuffer->GetEndAddress(); + + int exampleServerStart = exampleBuffer->GetStartServerId(); + int exampleServerEnd = exampleBuffer->GetEndServerId(); + + long exampleBufferLength = exampleBuffer->GetLength(); + long exampleTotalBufferLength = exampleBuffer->GetTotalLength(); + + long exampleBufferBlockLength = exampleBuffer->GetBlockLength(); + exampleBuffer->SetBlockLength(exampleBufferBlockLength); + + /* + exampleBuffer->ConfigureUniform(exampleBuffer->GetComm(), dsmSize/numServersCores, size - numServersCores, size - 1); + */ + + if (id == 0) + { + exampleBuffer->SendCommandHeader(XDMF_DSM_LOCK_ACQUIRE, 1, 0, 0, XDMF_DSM_INTRA_COMM); + } + + if (id == 1) + { + int probeComm; + exampleBuffer->ProbeCommandHeader(&probeComm); + + if (probeComm == XDMF_DSM_INTER_COMM) + { + std::cout << "InterComm" << std::endl; + } + else + { + std::cout << "IntraComm" << std::endl; + } + int length; + int address; + int opcode; + int source; + exampleBuffer->ReceiveCommandHeader(&opcode, &source, &length, &address, XDMF_DSM_INTRA_COMM, 0); + } + + + if (id == 0) + { + char * sentData = "datastring"; + exampleBuffer->SendData(1, sentData, 0, XDMF_DSM_PUT_DATA_TAG, 0, XDMF_DSM_INTER_COMM); + } + + if (id == 1) + { + int length; + int address; + char * recvData; + exampleBuffer->ReceiveData(0, recvData, length, XDMF_DSM_PUT_DATA_TAG, address, XDMF_DSM_INTER_COMM); + } + + if (id == 0) + { + int sentData = 1; + exampleBuffer->SendAcknowledgment(1, sentData, XDMF_DSM_PUT_DATA_TAG, XDMF_DSM_INTER_COMM); + } + + if (id == 1) + { + int recvData; + exampleBuffer->ReceiveAcknowledgment(0, recvData, XDMF_DSM_PUT_DATA_TAG, XDMF_DSM_INTER_COMM); + } + + MPI_Comm readComm, writeComm; + + MPI_Group readingCores, writingCores; + + MPI_Comm_group(workerComm, &workers); + int * ServerIds = (int *)calloc(((size - numServersCores) / 2), sizeof(int)); + unsigned int index = 0; + for(int i=0 ; i < (int)((size - numServersCores) / 2) ; ++i) + { + ServerIds[index++] = i; + } + + MPI_Group_excl(workers, index, ServerIds, &writingCores); + testval = MPI_Comm_create(workerComm, writingCores, &writeComm); + MPI_Group_incl(workers, index, ServerIds, &readingCores); + testval = MPI_Comm_create(workerComm, readingCores, &readComm); + cfree(ServerIds); + + + if (id < (int)((size - numServersCores) / 2)) + { + exampleBuffer->GetComm()->DupComm(readComm); + exampleBuffer->ReceiveInfo(); + } + else + { + exampleBuffer->GetComm()->DupComm(writeComm); + exampleBuffer->SendInfo(); + } + + exampleBuffer->GetComm()->DupComm(workerComm); + + int broadcastComm = XDMF_DSM_INTER_COMM; + exampleBuffer->BroadcastComm(&broadcastComm, 0); + + if (id == 0) + { + int returnCode; + exampleBuffer->BufferServiceLoop(&returnCode); + int serviceOut = exampleBuffer->BufferService(&returnCode); + } + + if (id == 1) + { + exampleBuffer->SendCommandHeader(XDMF_DSM_OPCODE_DONE, 0, 0, 0, XDMF_DSM_INTER_COMM); + exampleBuffer->SendCommandHeader(XDMF_DSM_OPCODE_DONE, 0, 0, 0, XDMF_DSM_INTER_COMM); + } + + int core0StartAddress = 0; + int core0EndAddress = 0; + exampleBuffer->GetAddressRangeForId(0, &core0StartAddress, &core0EndAddress); + + int correspondingId = exampleBuffer->AddressToId(500); + + int dsmData = 5; + if (sizeof(int)/sizeof(char) + core0StartAddress < core0EndAddress) + { + exampleBuffer->Put(0, sizeof(int)/sizeof(char), &dsmData); + exampleBuffer->Get(0, sizeof(int)/sizeof(char), &dsmData); + } + else + { + // Error occured + XdmfError::message(XdmfError::FATAL, "Address out of range"); + } + + XdmfDSMCommMPI * exampleDSMComm = exampleBuffer->GetComm(); + exampleBuffer->SetComm(exampleDSMComm); + + int exampleIntraID = exampleDSMComm->GetId(); + int exampleIntraSize = exampleDSMComm->GetIntraSize(); + int exampleInterSize = exampleDSMComm->GetInterSize(); + + int exampleInterCommType = exampleDSMComm->GetInterCommType(); + + exampleDSMComm->Init(); + + MPI_Comm exampleIntraComm = exampleDSMComm->GetIntraComm(); + exampleDSMComm->DupComm(workerComm); + + /* + bool connectingGroup; + char * portString; + if (id < 5) + { + connectingGroup = true; + } + else + { + connectingGroup = false; + } + + if (!connectingGroup) + { + exampleDSMComm->OpenPort(); + portString = exampleDSMComm->GetDsmMasterHostName(); + // Send the port string to the connecting group + exampleDSMComm->Accept(); + + // When done with connection; + exampleDSMComm->ClosePort(); + } + + if (connectingGroup) + { + // Recieve string from Master group + exampleDSMComm->SetDsmMasterHostName(portString); + exampleDSMComm->Connect(); + + // When done with connection + exampleDSMComm->Disconnect(); + } + + if (connectingGroup) + { + // Recieve string from Master group + exampleDSMComm->SetDsmMasterHostName(portString); + exampleManager->Connect(); + + // When done with connection + exampleManager->Disconnect(); + } + + */ + + + // This is the end of the Demonstration + + exampleWriter->setMode(XdmfHeavyDataWriter::Hyperslab); + + testArray->insert(writeController); + + for (unsigned int i = 0; igetHeavyDataController(0)->getDimensions(); + for (unsigned int j=0; jgetHeavyDataController(0)->getSize() << std::endl; + std::cout << "Controller starts" << std::endl; + outputVector = testArray->getHeavyDataController(0)->getStart(); + for (unsigned int j=0; jgetHeavyDataController(0)->getStride(); + for (unsigned int j=0; jgetSize(); ++j) + { + std::cout << "core #" << id <<" testArray[" << j << "] = " << testArray->getValue(j) << std::endl; + } + } + } + testArray->accept(exampleWriter); + + std::vector readStartVector; + readStartVector.push_back(4*(size - id - 1 - numServersCores)); + std::vector readStrideVector; + readStrideVector.push_back(1); + std::vector readCountVector; + readCountVector.push_back(4); + std::vector readDataSizeVector; + readDataSizeVector.push_back(4*(size-numServersCores)); + + shared_ptr readArray = XdmfArray::New(); + + readArray->initialize(0); + readArray->reserve(testArray->getSize()); + + shared_ptr readController = XdmfHDF5ControllerDSM::New( + newPath, + newSetPath, + XdmfArrayType::Int32(), + readStartVector, + readStrideVector, + readCountVector, + readDataSizeVector, + exampleWriter->getServerBuffer(), + size-numServersCores, + size-1); + + readArray->insert(readController); + + if (id == 0) + { + printf("\n\n\n"); + } + + std::cout << "testing read" << std::endl; + readArray->read(); + + + for (unsigned int i = 0; igetHeavyDataController(0)->getDimensions(); + for (unsigned int j=0; jgetHeavyDataController(0)->getSize() << std::endl; + std::cout << "Controller starts" << std::endl; + outputVector = readArray->getHeavyDataController(0)->getStart(); + for (unsigned int j=0; jgetHeavyDataController(0)->getStride(); + for (unsigned int j=0; jgetSize(); ++j) + { + std::cout << "core #" << id <<" readArray[" << j << "] = " << readArray->getValue(j) << std::endl; + } + } + } + + MPI_Barrier(workerComm); + + // End of Work Section + } + + exampleWriter->closeFile(); + + if (id == 0) + { + exampleWriter->stopDSM(); + /* + exampleController->stopDSM(); + XdmfDSMBuffer closeBuffer = exampleWriter->getServerBuffer(); + closeBuffer->SendDone(); + */ + } + + XdmfDSMCommMPI * exampleDSMComm = exampleWriter->getServerBuffer()->GetComm(); + MPI_Comm exampleInterComm = exampleDSMComm->GetInterComm(); + exampleDSMComm->DupInterComm(comm); + + /* + exampleWriter->restartDSM(); + exampleController->restartDSM(); + */ + + MPI_Barrier(comm); + + //the dsmManager must be deleted or else there will be a segfault + exampleWriter->deleteManager(); + + MPI_Finalize(); + + return 0; +} diff --git a/examples/Python/XdmfExampleDSMNoThread.py b/examples/Python/XdmfExampleDSMNoThread.py new file mode 100644 index 00000000..31f0c7bd --- /dev/null +++ b/examples/Python/XdmfExampleDSMNoThread.py @@ -0,0 +1,402 @@ +from Xdmf import * +from mpi4py.MPI import * + +if __name__ == "__main__": + #The total size of the DSM being created + dsmSize = 64; + comm = COMM_WORLD; + + id = comm.Get_rank() + size = comm.Get_size() + + testArray = XdmfArray.New() + testArray.initializeAsInt32(0) + newPath = "dsm" + newSetPath = "data" + + numServersCores = 4 + + for i in range(1,5): + testArray.pushBackAsInt32(i*(id+1)) + + + writeStartVector = UInt32Vector() + writeStartVector.push_back(id*4) + #writeStartVector.push_back(id); + writeStrideVector = UInt32Vector() + writeStrideVector.push_back(1) + #writeStrideVector.push_back(size-3); + writeCountVector = UInt32Vector() + writeCountVector.push_back(4) + writeDataSizeVector = UInt32Vector() + writeDataSizeVector.push_back(4*(size-numServersCores)) + + + ServerIds = [] + + for i in range(size-numServersCores, size): + ServerIds.append(i) + + workers = comm.Get_group().Excl(ServerIds) + + workerComm = comm.Create(workers) + + exampleWriter = XdmfHDF5WriterDSM.New(newPath, comm, dsmSize/numServersCores, size-numServersCores, size-1) + + ''' + exampleController = XdmfHDF5ControllerDSM.New( + newPath, + newSetPath, + XdmfArrayType.Int32(), + writeStartVector, + writeStrideVector, + writeCountVector, + writeDataSizeVector, + comm, + dsmSize/numServersCores, + size-numServersCores, + size-1) + ''' + + + + # Split out sub-comm for the worker cores + # Server cores will not progress to this point until after the servers are done running + + + if (id < size - numServersCores): + + + # This section is to demonstrate the functionality of the XdmfDSM classes + + exampleWriter.setServerMode(True) + exampleServerMode = exampleWriter.getServerMode() + + exampleServerComm = exampleWriter.getServerComm() + exampleWriter.setServerComm(exampleServerComm) + + exampleWorkerComm = exampleWriter.getWorkerComm() + exampleWriter.setWorkerComm(exampleWorkerComm) + + ''' + exampleServerComm = exampleController.getServerComm() + exampleController.setServerComm(exampleServerComm) + + exampleWorkerComm = exampleController.getWorkerComm() + exampleController.setWorkerComm(exampleWorkerComm) + ''' + + writeController = XdmfHDF5ControllerDSM.New( + newPath, + newSetPath, + XdmfArrayType.Int32(), + writeStartVector, + writeStrideVector, + writeCountVector, + writeDataSizeVector, + exampleWriter.getServerBuffer(), + size-numServersCores, + size-1); + + exampleWriter2 = XdmfHDF5WriterDSM.New(newPath, exampleWriter.getServerBuffer(), size-numServersCores, size-1); + + writeController.setServerMode(True) + exampleControllerServerMode = writeController.getServerMode() + + exampleManager = exampleWriter.getServerManager() + exampleWriter.setManager(exampleManager) + + ''' + exampleManager = exampleController.getServerManager() + exampleController.setManager(exampleManager) + ''' + + intraId = exampleManager.GetUpdatePiece() + intraSize = exampleManager.GetUpdateNumPieces() + + exampleManagerComm = exampleManager.GetMpiComm() + exampleManager.SetMpiComm(exampleManagerComm) + + exampleBufferSize = exampleManager.GetLocalBufferSizeMBytes() + exampleManager.SetLocalBufferSizeMBytes(exampleBufferSize) + + exampleIsServer = exampleManager.GetIsServer() + exampleManager.SetIsServer(exampleIsServer) + + exampleType = exampleManager.GetDsmType() + exampleManager.SetDsmType(XDMF_DSM_TYPE_UNIFORM) + + exampleBlockLength = exampleManager.GetBlockLength() + exampleManager.SetBlockLength(exampleBlockLength) + + exampleCommType = exampleManager.GetInterCommType() + exampleManager.SetInterCommType(XDMF_DSM_COMM_MPI) + + exampleManagerConnectionStatus = exampleManager.GetIsConnected() + + ''' + exampleManager.Destroy() + exampleManager.Create(size - numServerCores, size - 1) + ''' + + exampleBuffer = exampleWriter.getServerBuffer() + exampleWriter.setBuffer(exampleBuffer) + + ''' + exampleBuffer = exampleController.getServerBuffer() + exampleController.setBuffer(exampleBuffer) + ''' + + exampleManager.SetDsmBuffer(exampleBuffer) + exampleBuffer = exampleManager.GetDsmBuffer() + + exampleIsConnected = exampleBuffer.GetIsConnected() + exampleBuffer.SetIsConnected(exampleIsConnected) + + exampleDataPointer = exampleBuffer.GetDataPointer() + + exampleDSMType = exampleBuffer.GetDsmType() + exampleBuffer.SetDsmType(XDMF_DSM_TYPE_UNIFORM) + + exampleBufferIsServer = exampleBuffer.GetIsServer() + exampleBuffer.SetIsServer(exampleIsServer) + + exampleBufferStart = exampleBuffer.GetStartAddress() + exampleBufferEnd = exampleBuffer.GetEndAddress() + + exampleServerStart = exampleBuffer.GetStartServerId() + exampleServerEnd = exampleBuffer.GetEndServerId() + + exampleBufferLength = exampleBuffer.GetLength() + exampleTotalBufferLength = exampleBuffer.GetTotalLength() + + exampleBufferBlockLength = exampleBuffer.GetBlockLength() + exampleBuffer.SetBlockLength(exampleBufferBlockLength) + + ''' + exampleBuffer.ConfigureUniform(exampleBuffer.GetComm(), dsmSize/numServersCores, size - numServersCores, size - 1) + ''' + sendingCore = -1 + + if (id == sendingCore): + exampleBuffer.SendCommandHeader(XDMF_DSM_OPCODE_DONE, 1, 0, 0, XDMF_DSM_INTER_COMM) + + if (id == sendingCore): + sentData = "datastring" + exampleBuffer.SendData(1, sentData, 0, XDMF_DSM_PUT_DATA_TAG, 0, XDMF_DSM_INTER_COMM) + + if (id == sendingCore): + sentData = 1 + exampleBuffer.SendAcknowledgment(1, sentData, XDMF_DSM_PUT_DATA_TAG, XDMF_DSM_INTER_COMM) + + ServerIds = [] + + for i in range(0, (size - numServersCores) / 2): + ServerIds.append(i) + + readingCores = workerComm.Get_group().Excl(ServerIds) + + readComm = workerComm.Create(readingCores) + + writingCores = workerComm.Get_group().Incl(ServerIds) + + writeComm = workerComm.Create(writingCores) + + if (id < (int)((size - numServersCores) / 2)): + exampleBuffer.GetComm().DupComm(writeComm) + exampleBuffer.ReceiveInfo() + else: + exampleBuffer.GetComm().DupComm(readComm) + exampleBuffer.SendInfo() + + exampleBuffer.GetComm().DupComm(workerComm) + + if (id == 0): + exampleBuffer.BufferServiceLoop() + serviceOut = exampleBuffer.BufferService() + + if (id == 1): + exampleBuffer.SendCommandHeader(XDMF_DSM_OPCODE_DONE, 0, 0, 0, XDMF_DSM_INTER_COMM) + exampleBuffer.SendCommandHeader(XDMF_DSM_OPCODE_DONE, 0, 0, 0, XDMF_DSM_INTER_COMM) + + + correspondingId = exampleBuffer.AddressToId(500) + + exampleDSMComm = exampleBuffer.GetComm() + exampleBuffer.SetComm(exampleDSMComm) + + exampleIntraID = exampleDSMComm.GetId() + exampleIntraSize = exampleDSMComm.GetIntraSize() + exampleInterSize = exampleDSMComm.GetInterSize() + + exampleInterCommType = exampleDSMComm.GetInterCommType() + + exampleDSMComm.Init() + + + exampleIntraComm = exampleDSMComm.GetIntraComm() + exampleDSMComm.DupComm(exampleIntraComm.Dup()) + + print type(exampleDSMComm.GetDsmMasterHostName()) + testName = "test" + exampleDSMComm.SetDsmMasterHostName(testName) + print exampleDSMComm.GetDsmMasterHostName() + + ''' + + connectingGroup = True + if (id < 5): + connectingGroup = True; + else: + connectingGroup = False; + + portString = "" + + if (!connectingGroup): + exampleDSMComm.OpenPort() + portString = exampleDSMComm.GetDsmMasterHostName() + // Send the port string to the connecting group + exampleDSMComm.Accept() + // When done with connection + exampleDSMComm.ClosePort() + + if (connectingGroup): + // Recieve string from Master group + exampleDSMComm.SetDsmMasterHostName(portString) + exampleDSMComm.Connect() + // When done with connection + exampleDSMComm.Disconnect() + + if (connectingGroup): + // Recieve string from Master group + exampleDSMComm.SetDsmMasterHostName(portString); + exampleManager.Connect(); + // When done with connection + exampleManager.Disconnect(); + + ''' + + # This is the end of the Demonstration + + + exampleWriter.setMode(XdmfHeavyDataWriter.Hyperslab) + + testArray.insert(writeController) + + for i in range(0, size-numServersCores): + workerComm.Barrier() + if (i == id): + print "Core # " + str(id) + print "Controller stats" + print "datasetpath = " + testArray.getHeavyDataController(0).getDataSetPath() + print "filepath = " + testArray.getHeavyDataController(0).getFilePath() + outputVector = testArray.getHeavyDataController(0).getDataspaceDimensions() + print "Data space dimensions" + for j in range(0, outputVector.size()): + print "[" + str(j) + "] =" + str(outputVector[j]) + print "Controller Dimensions" + outputVector = testArray.getHeavyDataController(0).getDimensions() + for j in range(0, outputVector.size()): + print "[" + str(j) + "] =" + str(outputVector[j]) + print "Controller size" + str(testArray.getHeavyDataController(0).getSize()) + print "Controller starts" + outputVector = testArray.getHeavyDataController(0).getStart() + for j in range(0, outputVector.size()): + print "[" + str(j) + "] =" + str(outputVector[j]) + print "Controller strides" + outputVector = testArray.getHeavyDataController(0).getStride() + for j in range(0, outputVector.size()): + print "[" + str(j) + "] =" + str(outputVector[j]) + for j in range(0, testArray.getSize()): + print "core #" + str(id) + " testArray[" + str(j) + "] = " + str(testArray.getValueAsInt32(j)) + testArray.accept(exampleWriter) + + readStartVector = UInt32Vector() + readStartVector.push_back(4*(size - id - 1 - numServersCores)) + readStrideVector = UInt32Vector() + readStrideVector.push_back(1) + readCountVector = UInt32Vector() + readCountVector.push_back(4) + readDataSizeVector = UInt32Vector() + readDataSizeVector.push_back(4*(size-numServersCores)) + + readArray = XdmfArray.New() + + readArray.initializeAsInt32(0) + readArray.reserve(testArray.getSize()) + + readController = XdmfHDF5ControllerDSM.New( + newPath, + newSetPath, + XdmfArrayType.Int32(), + readStartVector, + readStrideVector, + readCountVector, + readDataSizeVector, + exampleWriter.getServerBuffer(), + size-numServersCores, + size-1) + + readArray.insert(readController) + + if (id == 0): + print "\n\n\n" + + print "testing read" + readArray.read() + + + for i in range (0, size): + workerComm.Barrier() + if (i == id): + print "Core # " + str(id) + print "Controller stats" + print "datasetpath = " + readArray.getHeavyDataController(0).getDataSetPath() + print "filepath = " + readArray.getHeavyDataController(0).getFilePath() + outputVector = readArray.getHeavyDataController(0).getDataspaceDimensions() + print "Data space dimensions" + for j in range(0, outputVector.size()): + print "[" + str(j) + "] =" + str(outputVector[j]) + print "Controller Dimensions" + outputVector = readArray.getHeavyDataController(0).getDimensions() + for j in range(0, outputVector.size()): + print "[" + str(j) + "] =" + str(outputVector[j]) + print "Controller size" + str(readArray.getHeavyDataController(0).getSize()) + print "Controller starts" + outputVector = readArray.getHeavyDataController(0).getStart() + for j in range(0, outputVector.size()): + print "[" + str(j) + "] =" + str(outputVector[j]) + print "Controller strides" + outputVector = readArray.getHeavyDataController(0).getStride() + for j in range(0, outputVector.size()): + print "[" + str(j) + "] =" + str(outputVector[j]) + for j in range (0, readArray.getSize()): + print "core #" + str(id) + " readArray[" + str(j) + "] = " + str(readArray.getValueAsInt32(j)) + + workerComm.Barrier() + + # End of Work Section + + + if (id == 0): + exampleWriter.stopDSM() + ''' + exampleController.stopDSM() + closeBuffer = exampleWriter.getServerBuffer() + closeBuffer.SendDone() + ''' + + exampleDSMComm = exampleWriter.getServerBuffer().GetComm() + + exampleInterComm = exampleDSMComm.GetInterComm() + exampleDSMComm.DupInterComm(exampleInterComm.Dup()) + + ''' + exampleWriter.restartDSM() + exampleController.restartDSM() + ''' + + exampleWriter.deleteManager() + ''' + exampleController.deleteManager() + ''' diff --git a/tests/Cxx/HugeWriteArray.cpp b/tests/Cxx/HugeWriteArray.cpp index 8a6f62a5..b77b6eec 100644 --- a/tests/Cxx/HugeWriteArray.cpp +++ b/tests/Cxx/HugeWriteArray.cpp @@ -5,10 +5,16 @@ #include #include #include +#include int main(int, char **) { - + timeb tb; + ftime(&tb); + int start = tb.millitm + (tb.time & 0xfffff) * 1000; + + + shared_ptr writtenArray = XdmfArray::New(); for (int i = 0; i < 9000000; i++) { @@ -77,5 +83,14 @@ int main(int, char **) assert(i == writtenArray->getValue(i)); } + + ftime(&tb); + int end = tb.millitm + (tb.time & 0xfffff) * 1000; + + int nSpan = end - start; + if(nSpan < 0) + nSpan += 0x100000 * 1000; + std::cout << nSpan << std::endl; + return 0; } diff --git a/tests/Cxx/XdmfPostFixCalc.cpp b/tests/Cxx/XdmfPostFixCalc.cpp index c649612b..a12a9f3f 100644 --- a/tests/Cxx/XdmfPostFixCalc.cpp +++ b/tests/Cxx/XdmfPostFixCalc.cpp @@ -170,7 +170,7 @@ double parse(std::string expression, std::map variables) //string is parsed left to right //elements of the same priority are evaluated right to left - for (unsigned int i = 0; i < expression.size(); i++) + for (int i = 0; i < expression.size(); i++) { if (validDigitChars.find(expression[i]) != std::string::npos)//found to be a digit { @@ -407,7 +407,7 @@ double function(std::vector valueVector, std::string functionName) double sum(std::vector values) { double total = 0.0; - for (unsigned int i = 0; i < values.size(); i++) + for (int i = 0; i < values.size(); i++) { total += values[i]; } @@ -439,7 +439,7 @@ shared_ptr parse(std::string expression, std::map calculation(shared_ptr val1, shared_ptrresize(val1->getSize()+val2->getSize(), sampleValue); } //determine ratio of array sizes - int arrayRatio1 = (int)floor(static_cast(val1->getSize())/val2->getSize()); - int arrayRatio2 = (int)floor(static_cast(val2->getSize())/val1->getSize()); + int arrayRatio1 = floor(static_cast(val1->getSize())/val2->getSize()); + int arrayRatio2 = floor(static_cast(val2->getSize())/val1->getSize()); if (arrayRatio1 < 1) { arrayRatio1 = 1; @@ -996,28 +996,28 @@ shared_ptr calculation(shared_ptr val1, shared_ptrgetSize()/arrayRatio1; - if (((amountWritten * arrayRatio1) + i) < (int)val1->getSize()) + if (((amountWritten * arrayRatio1) + i) < val1->getSize()) { amountWritten++; } - if (amountWritten > (int)floor(val2->getSize()/arrayRatio2)) + if (amountWritten > floor(val2->getSize()/arrayRatio2)) { - arrayExcess1 += amountWritten - (int)floor(val2->getSize()/arrayRatio2); - amountWritten = (int)floor(val2->getSize()/arrayRatio2); + arrayExcess1 += amountWritten - floor(val2->getSize()/arrayRatio2); + amountWritten = floor(val2->getSize()/arrayRatio2); } returnArray->insert(i, val1, i, amountWritten, stride, arrayRatio1); } else //second array takes the rest { int amountWritten = val2->getSize()/arrayRatio2; - if (((amountWritten * arrayRatio2) + i) < (int)val2->getSize()) + if (((amountWritten * arrayRatio2) + i) < val2->getSize()) { amountWritten++; } - if (amountWritten > (int)floor(val1->getSize()/arrayRatio1)) + if (amountWritten > floor(val1->getSize()/arrayRatio1)) { - arrayExcess2 += amountWritten - (int)floor(val1->getSize()/arrayRatio1); - amountWritten = (int)floor(val1->getSize()/arrayRatio1); + arrayExcess2 += amountWritten - floor(val1->getSize()/arrayRatio1); + amountWritten = floor(val1->getSize()/arrayRatio1); } returnArray->insert(i, val2, i-arrayRatio1, amountWritten, stride, arrayRatio2); } @@ -1064,9 +1064,9 @@ shared_ptr function(std::vector > valueVector, shared_ptr sum(std::vector > values) { double total = 0.0; - for (unsigned int i = 0; i < values.size(); i++) + for (int i = 0; i < values.size(); i++) { - for (unsigned int j = 0; j < values[i]->getSize(); j++) + for (int j = 0; j < values[i]->getSize(); j++) { total += values[i]->getValue(j); } @@ -1080,7 +1080,7 @@ shared_ptr ave(std::vector > values) { double total = sum(values)->getValue(0);; int totalSize = 0; - for (unsigned int i = 0; i < values.size(); i++) + for (int i = 0; i < values.size(); i++) { totalSize += values[i]->getSize(); } @@ -1100,9 +1100,9 @@ shared_ptr maximum(std::vector > values) else { double maxVal = values[0]->getValue(0); - for (unsigned int i = 0; i < values.size(); i++) + for (int i = 0; i < values.size(); i++) { - for (unsigned int j = 0; j < values[i]->getSize(); j++) + for (int j = 0; j < values[i]->getSize(); j++) { if (maxVal < values[i]->getValue(j)) { diff --git a/utils/XdmfUtils.i b/utils/XdmfUtils.i index c5a8e70f..2df4ae48 100644 --- a/utils/XdmfUtils.i +++ b/utils/XdmfUtils.i @@ -16,6 +16,9 @@ swig -v -c++ -python -o XdmfUtilsPython.cpp XdmfUtils.i #include #include #include + #include + #include + #include #include #include #include -- GitLab