XdmfDSMManager.cpp 8.09 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
/*****************************************************************************/
/*                                    XDMF                                   */
/*                       eXtensible Data Model and Format                    */
/*                                                                           */
/*  Id : XdmfDSMManager.cpp                                                  */
/*                                                                           */
/*  Author:                                                                  */
/*     Andrew Burns                                                          */
/*     andrew.j.burns2@us.army.mil                                           */
/*     US Army Research Laboratory                                           */
/*     Aberdeen Proving Ground, MD                                           */
/*                                                                           */
/*     Copyright @ 2013 US Army Research Laboratory                          */
/*     All Rights Reserved                                                   */
/*     See Copyright.txt for details                                         */
/*                                                                           */
/*     This software is distributed WITHOUT ANY WARRANTY; without            */
/*     even the implied warranty of MERCHANTABILITY or FITNESS               */
/*     FOR A PARTICULAR PURPOSE.  See the above copyright notice             */
/*     for more information.                                                 */
/*                                                                           */
/*****************************************************************************/

/*=========================================================================
  This code is derived from an earlier work and is distributed
  with permission from, and thanks to ...
=========================================================================*/

/*============================================================================

  Project                 : H5FDdsm
  Module                  : H5FDdsmManger.cxx

  Authors:
     John Biddiscombe     Jerome Soumagne
     biddisco@cscs.ch     soumagne@cscs.ch

  Copyright (C) CSCS - Swiss National Supercomputing Centre.
  You may use modify and and distribute this code freely providing
  1) This copyright notice appears on all copies of source code
  2) An acknowledgment appears with any substantial usage of the code
  3) If this code is contributed to any other open source project, it
  must not be reformatted such that the indentation, bracketing or
  overall style is modified significantly.

  This software is distributed WITHOUT ANY WARRANTY; without even the
  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

  This work has received funding from the European Community's Seventh
  Framework Programme (FP7/2007-2013) under grant agreement 225967 âxtMuSEâOC

============================================================================*/

#include <XdmfDSMManager.hpp>
#include <XdmfDSMBuffer.hpp>
#include <XdmfDSMCommMPI.hpp>
#include <XdmfError.hpp>
#include <mpi.h>

60
#ifndef _WIN32
61
62
63
  #include <unistd.h>
#endif

64
65
XdmfDSMManager::XdmfDSMManager()
{
66
67
68
69
70
71
72
73
74
75
76
  this->MpiComm                 = MPI_COMM_NULL;
  this->UpdatePiece             = 0;
  this->UpdateNumPieces         = 0;
  this->LocalBufferSizeMBytes   = 128;

  this->DsmBuffer               = NULL;
  this->DsmComm                 = NULL;
  this->IsServer                = true;
  this->DsmType                 = XDMF_DSM_TYPE_UNIFORM;
  this->BlockLength             = XDMF_DSM_DEFAULT_BLOCK_LENGTH;
  this->InterCommType           = XDMF_DSM_COMM_MPI;
77
78
79
80
}

XdmfDSMManager::~XdmfDSMManager()
{
81
  this->Destroy();
82
83
84
}

void
85
XdmfDSMManager::Connect(bool persist)
86
{
87
  int status;
88

89
90
91
  do {
    try {
      status = this->DsmBuffer->GetComm()->Connect();
92
    }
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
    catch (XdmfError e) {
      throw e;
    }
    if (status == MPI_SUCCESS) {
      dynamic_cast<XdmfDSMBuffer*> (this->DsmBuffer)->SetIsConnected(true);
      try {
        this->DsmBuffer->ReceiveInfo();
      }
      catch (XdmfError e) {
        throw e;
      }
    }
    else {
#ifdef _WIN32
  Sleep(1000);
  // Since windows has a different sleep command
#else
  sleep(1);
#endif
    }
  } while (persist && (status != MPI_SUCCESS));
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
}

void
XdmfDSMManager::Create(int startId, int endId)
{
  if (!this->DsmBuffer) {

    MPI_Comm_size(this->MpiComm, &this->UpdateNumPieces);
    MPI_Comm_rank(this->MpiComm, &this->UpdatePiece);
    //
    // Create DSM communicator
    //
    switch (this->GetInterCommType()) {
    case XDMF_DSM_COMM_MPI:
      this->DsmComm = new XdmfDSMCommMPI();
      break;
    default:
      try {
        XdmfError::message(XdmfError::FATAL, "DSM communication type not supported");
      }
      catch (XdmfError e) {
        throw e;
      }
    }
    this->DsmComm->DupComm(this->MpiComm);
    this->DsmComm->Init();
    //
    // Create the DSM buffer
    //
    this->DsmBuffer = new XdmfDSMBuffer();
    //
    this->DsmBuffer->SetIsServer(this->IsServer);
    // Uniform Dsm : every node has a buffer the same size. (Addresses are sequential)
    long length = (long) (this->GetLocalBufferSizeMBytes())*1024LU*1024LU;
    switch (this->DsmType) {
    case XDMF_DSM_TYPE_UNIFORM:
    case XDMF_DSM_TYPE_UNIFORM_RANGE:
      this->DsmBuffer->ConfigureUniform(this->DsmComm, length, startId, endId);
      break;
    case XDMF_DSM_TYPE_BLOCK_CYCLIC:
      this->DsmBuffer->ConfigureUniform(this->DsmComm, length, startId, endId, this->BlockLength, false);
      break;
    case XDMF_DSM_TYPE_BLOCK_RANDOM:
      this->DsmBuffer->ConfigureUniform(this->DsmComm, length, startId, endId, this->BlockLength, true);
      break;
    default:
      try {
        XdmfError(XdmfError::FATAL, "DSM configuration type not supported");
      }
      catch (XdmfError e) {
        throw e;
      }
    }
  }
}

void
171
XdmfDSMManager::Destroy()
172
{
173
174
175
176
177
178
179
180
181
182
183
  // Watch out that all processes have empty message queues
  // Should be already done during the disconnection
  if (this->DsmBuffer) {
    delete this->DsmBuffer;
    this->DsmBuffer = NULL;
    // Will be replaced by an Xdmf version
    // H5FD_dsm_set_manager(NULL);
  }
  if (this->DsmComm) {
    delete this->DsmComm;
    this->DsmComm = NULL;
184
185
186
187
188
189
  }
}

void
XdmfDSMManager::Disconnect()
{
190
191
192
193
194
195
196
197
  // Disconnecting is done manually
  try {
    this->DsmBuffer->GetComm()->Disconnect();
  }
  catch (XdmfError e) {
    throw e;
  }
  dynamic_cast<XdmfDSMBuffer*> (this->DsmBuffer)->SetIsConnected(false);
198
}
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311

long
XdmfDSMManager::GetBlockLength()
{
  return this->BlockLength;
}

XdmfDSMBuffer *
XdmfDSMManager::GetDsmBuffer()
{
  return this->DsmBuffer;
}

int
XdmfDSMManager::GetDsmType()
{
  return this->DsmType;
}

int
XdmfDSMManager::GetInterCommType()
{
  return this->InterCommType;
}

bool
XdmfDSMManager::GetIsConnected()
{
  if (this->DsmBuffer) {
    return this->DsmBuffer->GetIsConnected();
  }
  else {
    return false;
  }
}

bool
XdmfDSMManager::GetIsServer()
{
  return this->IsServer;
}

unsigned int
XdmfDSMManager::GetLocalBufferSizeMBytes()
{
  return this->LocalBufferSizeMBytes;
}

MPI_Comm
XdmfDSMManager::GetMpiComm()
{
  return this->MpiComm;
}

int
XdmfDSMManager::GetUpdatePiece()
{
  return this->UpdatePiece;
}

int
XdmfDSMManager::GetUpdateNumPieces()
{
  return this->UpdateNumPieces;
}

void
XdmfDSMManager::SetBlockLength(long newSize)
{
  this->BlockLength = newSize;
}

void
XdmfDSMManager::SetDsmBuffer(XdmfDSMBuffer * newBuffer)
{
  this->DsmBuffer = newBuffer;
}

void
XdmfDSMManager::SetDsmType(int newType)
{
  this->DsmType = newType;
}

void
XdmfDSMManager::SetIsServer(bool newStatus)
{
  this->IsServer = newStatus;
}

void
XdmfDSMManager::SetInterCommType(int newType)
{
  this->InterCommType = newType;
}

void
XdmfDSMManager::SetLocalBufferSizeMBytes(unsigned int newSize)
{
  this->LocalBufferSizeMBytes = newSize;
}

void
XdmfDSMManager::SetMpiComm(MPI_Comm comm)
{
  if (comm != this->MpiComm) {
    this->MpiComm = comm;
    if (this->MpiComm != MPI_COMM_NULL) {
      MPI_Comm_size(this->MpiComm, &this->UpdateNumPieces);
      MPI_Comm_rank(this->MpiComm, &this->UpdatePiece);
    }
  }
}