Commit 63b6854f authored by Jerry Clarke's avatar Jerry Clarke
Browse files

when using pthreads, HDF5 was stepping on data structures.

If HDF was compiled --threadsafe, DSM would Deadlock on MPIRecv().
Solution is to stay out of HDF5 Library on DSM Service.
parent f323e243
......@@ -45,6 +45,7 @@ XdmfDsm::XdmfDsm() {
// For Alignment
this->Storage->SetNumberType(XDMF_INT64_TYPE);
this->SetLength(XDMF_DSM_DEFAULT_LENGTH);
this->DataPointer = (XdmfByte *)this->Storage->GetDataPointer();
this->StartAddress = 0;
this->EndAddress = this->StartAddress + this->Length - 1;
this->Comm = 0;
......@@ -61,6 +62,7 @@ XdmfDsm::Copy(XdmfDsm *Source){
this->DsmType = Source->DsmType;
this->Storage = Source->GetStorage();
this->StorageIsMine = 0;
this->DataPointer = (XdmfByte *)this->Storage->GetDataPointer();
// For Alignment
this->Length = Source->Length;
this->StartAddress = Source->StartAddress;
......@@ -78,6 +80,7 @@ XdmfInt32
XdmfDsm::SetStorage(XdmfArray *aStorage){
if(this->Storage && this->StorageIsMine) delete this->Storage;
this->Storage = aStorage;
this->DataPointer = (XdmfByte *)this->Storage->GetDataPointer();
return(XDMF_SUCCESS);
}
......@@ -170,6 +173,7 @@ XdmfDsm::SetLength(XdmfInt64 aLength){
return(XDMF_FAIL);
}
this->Length = aLength;
this->DataPointer = (XdmfByte *)this->Storage->GetDataPointer();
return(XDMF_SUCCESS);
}
......
......@@ -126,6 +126,7 @@ protected:
XdmfArray *Storage;
XdmfDsmComm *Comm;
XdmfDsmMsg *Msg;
XdmfByte *DataPointer;
};
#endif // __XdmfDsm_h
......@@ -46,6 +46,7 @@ XdmfDsmBufferServiceThread(void *DsmObj){
XdmfDsmBuffer::XdmfDsmBuffer() {
XdmfInt64 i;
this->ThreadDsmReady = 0;
this->DataPointer = 0;
this->Locks = new XdmfInt64[XDMF_DSM_MAX_LOCKS];
for(i=0;i < XDMF_DSM_MAX_LOCKS;i++) this->Locks[i] = -1;
}
......@@ -147,7 +148,11 @@ XdmfDsmBuffer::Service(XdmfInt32 *ReturnOpcode){
XdmfErrorMessage("Length too long");
return(XDMF_FAIL);
}
datap = (XdmfByte *)this->Storage->GetDataPointer();
// datap = (XdmfByte *)this->Storage->GetDataPointer();
// Stay out of HDF library. If it is threadsafe if will
// deadlock on mpi_recv. If it is not threadsafe it will
// get corrupted
datap = this->DataPointer;
datap += Address - this->StartAddress;
this->Msg->SetTag(XDMF_DSM_COMMAND_TAG);
status = this->ReceiveData(who, datap, aLength);
......@@ -164,7 +169,11 @@ XdmfDsmBuffer::Service(XdmfInt32 *ReturnOpcode){
XdmfErrorMessage("Server Start = " << this->StartAddress << " End = " << this->EndAddress);
return(XDMF_FAIL);
}
datap = (XdmfByte *)this->Storage->GetDataPointer();
// datap = (XdmfByte *)this->Storage->GetDataPointer();
// Stay out of HDF library. If it is threadsafe if will
// deadlock on mpi_recv. If it is not threadsafe it will
// get corrupted
datap = this->DataPointer;
datap += Address - this->StartAddress;
this->Msg->SetTag(XDMF_DSM_RESPONSE_TAG);
status = this->SendData(who, datap, aLength);
......@@ -341,7 +350,11 @@ XdmfDsmBuffer::Put(XdmfInt64 Address, XdmfInt64 aLength, void *Data){
XdmfByte *dp;
// cout << "That's me!!" << endl;
dp = (XdmfByte *)this->Storage->GetDataPointer();
// dp = (XdmfByte *)this->Storage->GetDataPointer();
// Stay out of HDF library. If it is threadsafe if will
// deadlock on mpi_recv. If it is not threadsafe it will
// get corrupted
dp = this->DataPointer;
dp += Address - this->StartAddress;
memcpy(dp, datap, len);
......@@ -387,7 +400,11 @@ XdmfDsmBuffer::Get(XdmfInt64 Address, XdmfInt64 aLength, void *Data){
XdmfByte *dp;
// cout << "That's me!!" << endl;
dp = (XdmfByte *)this->Storage->GetDataPointer();
// dp = (XdmfByte *)this->Storage->GetDataPointer();
// Stay out of HDF library. If it is threadsafe if will
// deadlock on mpi_recv. If it is not threadsafe it will
// get corrupted
dp = this->DataPointer;
dp += Address - this->StartAddress;
memcpy(datap, dp, len);
......
......@@ -87,6 +87,12 @@ XdmfDsmCommMpi::Receive(XdmfDsmMsg *Msg){
}
status = MPI_Get_count(&SendRecvStatus, MPI_UNSIGNED_CHAR, &MessageLength);
XdmfDebug("::::: (" << this->Id << ") Received " << MessageLength << " bytes from " << SendRecvStatus.MPI_SOURCE);
/*
if(this->Id == 0){
cout << "::::: (" << this->Id << ") Received " << MessageLength << " bytes from " << SendRecvStatus.MPI_SOURCE << endl;
cout << "::::: (" << this->Id << ") Expected Length = " << Msg->Length << endl;
}
*/
Msg->SetSource(SendRecvStatus.MPI_SOURCE);
Msg->SetLength(MessageLength);
if(status != MPI_SUCCESS){
......@@ -102,6 +108,11 @@ XdmfDsmCommMpi::Send(XdmfDsmMsg *Msg){
if(XdmfDsmComm::Send(Msg) != XDMF_SUCCESS) return(XDMF_FAIL);
XdmfDebug("::::: (" << this->Id << ") Sending " << Msg->Length << " bytes to " << Msg->Dest << " Tag = " << Msg->Tag);
/*
if(this->Id == 1){
cout << "::::: (" << this->Id << ") Sending " << Msg->Length << " bytes to " << Msg->Dest << " Tag = " << Msg->Tag << endl;
}
*/
status = MPI_Send(Msg->Data, Msg->Length, MPI_UNSIGNED_CHAR, Msg->Dest, Msg->Tag, this->Comm);
// status = MPI_Ssend(Msg->Data, Msg->Length, MPI_UNSIGNED_CHAR, Msg->Dest, Msg->Tag, this->Comm);
if(status != MPI_SUCCESS){
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment