mpi.c 17.5 KB
Newer Older
Andy Cedilnik's avatar
Andy Cedilnik committed
1 2 3
/* -*- c -*- *******************************************************/
/*
 * Copyright (C) 2003 Sandia Corporation
4 5 6 7
 * Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
 * the U.S. Government retains certain rights in this software.
 *
 * This source code is released under the New BSD License.
Andy Cedilnik's avatar
Andy Cedilnik committed
8 9
 */

10
#include <IceTMPI.h>
Andy Cedilnik's avatar
Andy Cedilnik committed
11

12
#include <IceTDevCommunication.h>
13
#include <IceTDevDiagnostics.h>
14 15
#include <IceTDevPorting.h>
#include <IceTDevState.h>
Andy Cedilnik's avatar
Andy Cedilnik committed
16 17

#include <stdlib.h>
18 19 20 21 22
#include <string.h>

#ifdef DEBUG
#define BREAK_ON_MPI_ERROR
#endif
Andy Cedilnik's avatar
Andy Cedilnik committed
23

24 25 26 27
#if MPI_VERSION >= 2
#define ICET_USE_MPI_IN_PLACE
#endif

28 29
#define ICET_MPI_REQUEST_MAGIC_NUMBER ((IceTEnum)0xD7168B00)

30 31
#define ICET_MPI_TEMP_BUFFER_0  (ICET_COMMUNICATION_LAYER_START | (IceTEnum)0x00)

32 33 34
static IceTCommunicator MPIDuplicate(IceTCommunicator self);
static IceTCommunicator MPISubset(IceTCommunicator self,
                                  int count,
35
                                  const IceTInt32 *ranks);
36 37 38 39 40
static void MPIDestroy(IceTCommunicator self);
static void MPIBarrier(IceTCommunicator self);
static void MPISend(IceTCommunicator self,
                    const void *buf,
                    int count,
41
                    IceTEnum datatype,
42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61
                    int dest,
                    int tag);
static void MPIRecv(IceTCommunicator self,
                    void *buf,
                    int count,
                    IceTEnum datatype,
                    int src,
                    int tag);
static void MPISendrecv(IceTCommunicator self,
                        const void *sendbuf,
                        int sendcount,
                        IceTEnum sendtype,
                        int dest,
                        int sendtag,
                        void *recvbuf,
                        int recvcount,
                        IceTEnum recvtype,
                        int src,
                        int recvtag);
static void MPIGather(IceTCommunicator self,
62 63 64
                      const void *sendbuf,
                      int sendcount,
                      IceTEnum datatype,
65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101
                      void *recvbuf,
                      int root);
static void MPIGatherv(IceTCommunicator self,
                       const void *sendbuf,
                       int sendcount,
                       IceTEnum datatype,
                       void *recvbuf,
                       const int *recvcounts,
                       const int *recvoffsets,
                       int root);
static void MPIAllgather(IceTCommunicator self,
                         const void *sendbuf,
                         int sendcount,
                         IceTEnum datatype,
                         void *recvbuf);
static void MPIAlltoall(IceTCommunicator self,
                        const void *sendbuf,
                        int sendcount,
                        IceTEnum datatype,
                        void *recvbuf);
static IceTCommRequest MPIIsend(IceTCommunicator self,
                                const void *buf,
                                int count,
                                IceTEnum datatype,
                                int dest,
                                int tag);
static IceTCommRequest MPIIrecv(IceTCommunicator self,
                                void *buf,
                                int count,
                                IceTEnum datatype,
                                int src,
                                int tag);
static void MPIWaitone(IceTCommunicator self, IceTCommRequest *request);
static int  MPIWaitany(IceTCommunicator self,
                       int count, IceTCommRequest *array_of_requests);
static int MPIComm_size(IceTCommunicator self);
static int MPIComm_rank(IceTCommunicator self);
Andy Cedilnik's avatar
Andy Cedilnik committed
102

103
typedef struct IceTMPICommRequestInternalsStruct {
Andy Cedilnik's avatar
Andy Cedilnik committed
104
    MPI_Request request;
105
} *IceTMPICommRequestInternals;
Andy Cedilnik's avatar
Andy Cedilnik committed
106

107 108 109 110 111
static MPI_Request getMPIRequest(IceTCommRequest icet_request)
{
    if (icet_request == ICET_COMM_REQUEST_NULL) {
        return MPI_REQUEST_NULL;
    }
Andy Cedilnik's avatar
Andy Cedilnik committed
112

113
    if (icet_request->magic_number != ICET_MPI_REQUEST_MAGIC_NUMBER) {
114 115
        icetRaiseError(ICET_INVALID_VALUE,
                       "Request object is not from the MPI communicator.");
116 117
        return MPI_REQUEST_NULL;
    }
Andy Cedilnik's avatar
Andy Cedilnik committed
118

119 120 121 122
    return (((IceTMPICommRequestInternals)icet_request->internals)->request);
}

static void setMPIRequest(IceTCommRequest icet_request, MPI_Request mpi_request)
Andy Cedilnik's avatar
Andy Cedilnik committed
123
{
124
    if (icet_request == ICET_COMM_REQUEST_NULL) {
125 126
        icetRaiseError(ICET_SANITY_CHECK_FAIL,
                       "Cannot set MPI request in null request.");
127
        return;
Andy Cedilnik's avatar
Andy Cedilnik committed
128
    }
129 130

    if (icet_request->magic_number != ICET_MPI_REQUEST_MAGIC_NUMBER) {
131 132
        icetRaiseError(ICET_SANITY_CHECK_FAIL,
                       "Request object is not from the MPI communicator.");
133
        return;
Andy Cedilnik's avatar
Andy Cedilnik committed
134
    }
135 136 137

    (((IceTMPICommRequestInternals)icet_request->internals)->request)
        = mpi_request;
Andy Cedilnik's avatar
Andy Cedilnik committed
138
}
139 140

static IceTCommRequest create_request(void)
Andy Cedilnik's avatar
Andy Cedilnik committed
141
{
142 143 144
    IceTCommRequest request;

    request = (IceTCommRequest)malloc(sizeof(struct IceTCommRequestStruct));
145
    if (request == NULL) {
146 147
        icetRaiseError(ICET_OUT_OF_MEMORY,
                       "Could not allocate memory for IceTCommRequest");
148 149 150
        return NULL;
    }

151 152
    request->magic_number = ICET_MPI_REQUEST_MAGIC_NUMBER;
    request->internals=malloc(sizeof(struct IceTMPICommRequestInternalsStruct));
153 154
    if (request->internals == NULL) {
        free(request);
155 156
        icetRaiseError(ICET_OUT_OF_MEMORY,
                       "Could not allocate memory for IceTCommRequest");
157 158
        return NULL;
    }
159 160 161 162

    setMPIRequest(request, MPI_REQUEST_NULL);

    return request;
Andy Cedilnik's avatar
Andy Cedilnik committed
163 164
}

165 166 167 168
static void destroy_request(IceTCommRequest request)
{
    MPI_Request mpi_request = getMPIRequest(request);
    if (mpi_request != MPI_REQUEST_NULL) {
169 170 171
        icetRaiseError(ICET_SANITY_CHECK_FAIL,
                       "Destroying MPI request that is not NULL."
                       " Probably leaking MPI requests.");
172 173 174 175 176
    }

    free(request->internals);
    free(request);
}
Andy Cedilnik's avatar
Andy Cedilnik committed
177

178
#ifdef BREAK_ON_MPI_ERROR
Mathieu Malaterre's avatar
Mathieu Malaterre committed
179
static void ErrorHandler(MPI_Comm *comm, int *errorno, ...)
180
{
181
    char error_msg[MPI_MAX_ERROR_STRING];
182
    int mpi_error_len;
Mathieu Malaterre's avatar
Mathieu Malaterre committed
183
    (void)comm;
184

185
    MPI_Error_string(*errorno, error_msg, &mpi_error_len);
186

187
    icetRaiseError(ICET_INVALID_OPERATION, "MPI ERROR:\n%s", error_msg);
188 189 190 191
    icetDebugBreak();
}
#endif

Andy Cedilnik's avatar
Andy Cedilnik committed
192 193
IceTCommunicator icetCreateMPICommunicator(MPI_Comm mpi_comm)
{
194
    IceTCommunicator comm;
195 196 197
#ifdef BREAK_ON_MPI_ERROR
    MPI_Errhandler eh;
#endif
Andy Cedilnik's avatar
Andy Cedilnik committed
198

199 200 201 202 203
    if (mpi_comm == MPI_COMM_NULL) {
        return ICET_COMM_NULL;
    }

    comm = malloc(sizeof(struct IceTCommunicatorStruct));
204
    if (comm == NULL) {
205 206
        icetRaiseError(ICET_OUT_OF_MEMORY,
                       "Could not allocate memory for IceTCommunicator.");
207 208 209
        return NULL;
    }

210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
    comm->Duplicate = MPIDuplicate;
    comm->Subset = MPISubset;
    comm->Destroy = MPIDestroy;
    comm->Barrier = MPIBarrier;
    comm->Send = MPISend;
    comm->Recv = MPIRecv;
    comm->Sendrecv = MPISendrecv;
    comm->Gather = MPIGather;
    comm->Gatherv = MPIGatherv;
    comm->Allgather = MPIAllgather;
    comm->Alltoall = MPIAlltoall;
    comm->Isend = MPIIsend;
    comm->Irecv = MPIIrecv;
    comm->Wait = MPIWaitone;
    comm->Waitany = MPIWaitany;
    comm->Comm_size = MPIComm_size;
    comm->Comm_rank = MPIComm_rank;
227

228
    comm->data = malloc(sizeof(MPI_Comm));
229 230
    if (comm->data == NULL) {
        free(comm);
231 232
        icetRaiseError(ICET_OUT_OF_MEMORY,
                       "Could not allocate memory for IceTCommunicator.");
233 234
        return NULL;
    }
235
    MPI_Comm_dup(mpi_comm, (MPI_Comm *)comm->data);
Andy Cedilnik's avatar
Andy Cedilnik committed
236

237
#ifdef BREAK_ON_MPI_ERROR
238
#if MPI_VERSION < 2
239
    MPI_Errhandler_create(ErrorHandler, &eh);
240
    MPI_Errhandler_set(*((MPI_Comm *)comm->data), eh);
241
    MPI_Errhandler_free(&eh);
242
#else /* MPI_VERSION >= 2 */
243 244 245
    MPI_Comm_create_errhandler(ErrorHandler, &eh);
    MPI_Comm_set_errhandler(*((MPI_Comm *)comm->data), eh);
    MPI_Errhandler_free(&eh);
246
#endif /* MPI_VERSION >= 2 */
247 248
#endif

Andy Cedilnik's avatar
Andy Cedilnik committed
249 250 251 252 253
    return comm;
}

void icetDestroyMPICommunicator(IceTCommunicator comm)
{
254 255 256
    if (comm != ICET_COMM_NULL) {
        comm->Destroy(comm);
    }
Andy Cedilnik's avatar
Andy Cedilnik committed
257 258 259
}


260
#define MPI_COMM        (*((MPI_Comm *)self->data))
Andy Cedilnik's avatar
Andy Cedilnik committed
261

262
static IceTCommunicator MPIDuplicate(IceTCommunicator self)
Andy Cedilnik's avatar
Andy Cedilnik committed
263
{
264 265 266 267 268
    if (self != ICET_COMM_NULL) {
        return icetCreateMPICommunicator(MPI_COMM);
    } else {
        return ICET_COMM_NULL;
    }
Andy Cedilnik's avatar
Andy Cedilnik committed
269 270
}

271 272
static IceTCommunicator MPISubset(IceTCommunicator self,
                                  int count,
273
                                  const IceTInt32 *ranks)
274 275 276 277 278 279 280
{
    MPI_Group original_group;
    MPI_Group subset_group;
    MPI_Comm subset_comm;
    IceTCommunicator result;

    MPI_Comm_group(MPI_COMM, &original_group);
281
    MPI_Group_incl(original_group, count, (IceTInt32 *)ranks, &subset_group);
282 283 284 285 286 287 288 289 290 291 292 293 294 295 296
    MPI_Comm_create(MPI_COMM, subset_group, &subset_comm);

    result = icetCreateMPICommunicator(subset_comm);

    if (subset_comm != MPI_COMM_NULL) {
        MPI_Comm_free(&subset_comm);
    }

    MPI_Group_free(&subset_group);
    MPI_Group_free(&original_group);

    return result;
}

static void MPIDestroy(IceTCommunicator self)
Andy Cedilnik's avatar
Andy Cedilnik committed
297
{
298 299
    MPI_Comm_free((MPI_Comm *)self->data);
    free(self->data);
Andy Cedilnik's avatar
Andy Cedilnik committed
300 301 302
    free(self);
}

303
static void MPIBarrier(IceTCommunicator self)
304 305 306 307
{
    MPI_Barrier(MPI_COMM);
}

Ken Martin's avatar
Ken Martin committed
308
#define CONVERT_DATATYPE(icet_type, mpi_type)                                \
309
    switch (icet_type) {                                                     \
310
      case ICET_BOOLEAN:mpi_type = MPI_BYTE;    break;                       \
311 312 313 314 315 316
      case ICET_BYTE:   mpi_type = MPI_BYTE;    break;                       \
      case ICET_SHORT:  mpi_type = MPI_SHORT;   break;                       \
      case ICET_INT:    mpi_type = MPI_INT;     break;                       \
      case ICET_FLOAT:  mpi_type = MPI_FLOAT;   break;                       \
      case ICET_DOUBLE: mpi_type = MPI_DOUBLE;  break;                       \
      default:                                                               \
317 318 319
          icetRaiseError(ICET_INVALID_ENUM,                                  \
                         "MPI Communicator received bad data type 0x%X.",    \
                         icet_type);                                         \
320 321
          mpi_type = MPI_BYTE;                                               \
          break;                                                             \
Andy Cedilnik's avatar
Andy Cedilnik committed
322 323
    }

324 325 326 327 328 329
static void MPISend(IceTCommunicator self,
                    const void *buf,
                    int count,
                    IceTEnum datatype,
                    int dest,
                    int tag)
Andy Cedilnik's avatar
Andy Cedilnik committed
330
{
331
    MPI_Datatype mpidatatype;
Andy Cedilnik's avatar
Andy Cedilnik committed
332 333 334 335
    CONVERT_DATATYPE(datatype, mpidatatype);
    MPI_Send((void *)buf, count, mpidatatype, dest, tag, MPI_COMM);
}

336 337 338 339 340 341
static void MPIRecv(IceTCommunicator self,
                    void *buf,
                    int count,
                    IceTEnum datatype,
                    int src,
                    int tag)
Andy Cedilnik's avatar
Andy Cedilnik committed
342
{
343
    MPI_Datatype mpidatatype;
Andy Cedilnik's avatar
Andy Cedilnik committed
344
    CONVERT_DATATYPE(datatype, mpidatatype);
345
    MPI_Recv(buf, count, mpidatatype, src, tag, MPI_COMM, MPI_STATUS_IGNORE);
Andy Cedilnik's avatar
Andy Cedilnik committed
346 347
}

348 349 350 351 352 353 354 355 356 357 358
static void MPISendrecv(IceTCommunicator self,
                        const void *sendbuf,
                        int sendcount,
                        IceTEnum sendtype,
                        int dest,
                        int sendtag,
                        void *recvbuf,
                        int recvcount,
                        IceTEnum recvtype,
                        int src,
                        int recvtag)
Andy Cedilnik's avatar
Andy Cedilnik committed
359
{
360 361
    MPI_Datatype mpisendtype;
    MPI_Datatype mpirecvtype;
Andy Cedilnik's avatar
Andy Cedilnik committed
362 363 364 365
    CONVERT_DATATYPE(sendtype, mpisendtype);
    CONVERT_DATATYPE(recvtype, mpirecvtype);

    MPI_Sendrecv((void *)sendbuf, sendcount, mpisendtype, dest, sendtag,
366 367
                 recvbuf, recvcount, mpirecvtype, src, recvtag, MPI_COMM,
                 MPI_STATUS_IGNORE);
Andy Cedilnik's avatar
Andy Cedilnik committed
368 369
}

370 371 372 373 374 375
static void MPIGather(IceTCommunicator self,
                      const void *sendbuf,
                      int sendcount,
                      IceTEnum datatype,
                      void *recvbuf,
                      int root)
376 377 378 379 380
{
    MPI_Datatype mpitype;
    CONVERT_DATATYPE(datatype, mpitype);

    if (sendbuf == ICET_IN_PLACE_COLLECT) {
381
#ifdef ICET_USE_MPI_IN_PLACE
382
        sendbuf = MPI_IN_PLACE;
383 384 385 386 387 388 389 390 391
#else
        int rank;
        MPI_Comm_rank(MPI_COMM, &rank);
        sendbuf = icetGetStateBuffer(ICET_MPI_TEMP_BUFFER_0,
                                     sendcount*icetTypeWidth(datatype));
        memcpy((void *)sendbuf,
               ((const IceTByte *)recvbuf) + rank*sendcount,
               sendcount);
#endif
392 393 394 395 396 397 398
    }

    MPI_Gather((void *)sendbuf, sendcount, mpitype,
               recvbuf, sendcount, mpitype, root,
               MPI_COMM);
}

399 400 401 402 403 404 405 406
static void MPIGatherv(IceTCommunicator self,
                       const void *sendbuf,
                       int sendcount,
                       IceTEnum datatype,
                       void *recvbuf,
                       const int *recvcounts,
                       const int *recvoffsets,
                       int root)
407 408 409 410 411
{
    MPI_Datatype mpitype;
    CONVERT_DATATYPE(datatype, mpitype);

    if (sendbuf == ICET_IN_PLACE_COLLECT) {
412
#ifdef ICET_USE_MPI_IN_PLACE
413
        sendbuf = MPI_IN_PLACE;
414 415 416 417 418 419 420 421 422 423
#else
        int rank;
        MPI_Comm_rank(MPI_COMM, &rank);
        sendcount = recvcounts[rank];
        sendbuf = icetGetStateBuffer(ICET_MPI_TEMP_BUFFER_0,
                                     sendcount*icetTypeWidth(datatype));
        memcpy((void *)sendbuf,
               ((const IceTByte *)recvbuf) + recvoffsets[rank],
               sendcount);
#endif
424 425 426 427 428 429 430
    }

    MPI_Gatherv((void *)sendbuf, sendcount, mpitype,
                recvbuf, (int *)recvcounts, (int *)recvoffsets, mpitype,
                root, MPI_COMM);
}

431 432 433 434 435
static void MPIAllgather(IceTCommunicator self,
                         const void *sendbuf,
                         int sendcount,
                         IceTEnum datatype,
                         void *recvbuf)
Andy Cedilnik's avatar
Andy Cedilnik committed
436
{
437
    MPI_Datatype mpitype;
438 439 440
    CONVERT_DATATYPE(datatype, mpitype);

    if (sendbuf == ICET_IN_PLACE_COLLECT) {
441
#ifdef ICET_USE_MPI_IN_PLACE
442
        sendbuf = MPI_IN_PLACE;
443 444 445 446 447 448 449 450 451
#else
        int rank;
        MPI_Comm_rank(MPI_COMM, &rank);
        sendbuf = icetGetStateBuffer(ICET_MPI_TEMP_BUFFER_0,
                                     sendcount*icetTypeWidth(datatype));
        memcpy((void *)sendbuf,
               ((const IceTByte *)recvbuf) + rank*sendcount,
               sendcount);
#endif
452
    }
Andy Cedilnik's avatar
Andy Cedilnik committed
453 454

    MPI_Allgather((void *)sendbuf, sendcount, mpitype,
Ken Martin's avatar
Ken Martin committed
455 456
                  recvbuf, sendcount, mpitype,
                  MPI_COMM);
Andy Cedilnik's avatar
Andy Cedilnik committed
457 458
}

459 460 461 462 463
static void MPIAlltoall(IceTCommunicator self,
                        const void *sendbuf,
                        int sendcount,
                        IceTEnum datatype,
                        void *recvbuf)
464 465 466 467 468 469 470 471 472
{
    MPI_Datatype mpitype;
    CONVERT_DATATYPE(datatype, mpitype);

    MPI_Alltoall((void *)sendbuf, sendcount, mpitype,
                 recvbuf, sendcount, mpitype,
                 MPI_COMM);
}

473 474 475 476 477 478
static IceTCommRequest MPIIsend(IceTCommunicator self,
                                const void *buf,
                                int count,
                                IceTEnum datatype,
                                int dest,
                                int tag)
Andy Cedilnik's avatar
Andy Cedilnik committed
479 480
{
    IceTCommRequest icet_request;
481
    MPI_Request mpi_request;
482
    MPI_Datatype mpidatatype;
Andy Cedilnik's avatar
Andy Cedilnik committed
483 484 485

    CONVERT_DATATYPE(datatype, mpidatatype);
    MPI_Isend((void *)buf, count, mpidatatype, dest, tag, MPI_COMM,
486 487 488 489
              &mpi_request);

    icet_request = create_request();
    setMPIRequest(icet_request, mpi_request);
Andy Cedilnik's avatar
Andy Cedilnik committed
490 491 492 493

    return icet_request;
}

494 495 496 497 498 499
static IceTCommRequest MPIIrecv(IceTCommunicator self,
                                void *buf,
                                int count,
                                IceTEnum datatype,
                                int src,
                                int tag)
Andy Cedilnik's avatar
Andy Cedilnik committed
500 501
{
    IceTCommRequest icet_request;
502
    MPI_Request mpi_request;
503
    MPI_Datatype mpidatatype;
Andy Cedilnik's avatar
Andy Cedilnik committed
504 505 506

    CONVERT_DATATYPE(datatype, mpidatatype);
    MPI_Irecv(buf, count, mpidatatype, src, tag, MPI_COMM,
507 508 509 510
              &mpi_request);

    icet_request = create_request();
    setMPIRequest(icet_request, mpi_request);
Andy Cedilnik's avatar
Andy Cedilnik committed
511 512 513 514

    return icet_request;
}

515
static void MPIWaitone(IceTCommunicator self, IceTCommRequest *icet_request)
Andy Cedilnik's avatar
Andy Cedilnik committed
516
{
517
    MPI_Request mpi_request;
Andy Cedilnik's avatar
Andy Cedilnik committed
518

519 520 521
    /* To remove warning */
    (void)self;

522
    if (*icet_request == ICET_COMM_REQUEST_NULL) return;
Andy Cedilnik's avatar
Andy Cedilnik committed
523

524 525 526 527 528 529
    mpi_request = getMPIRequest(*icet_request);
    MPI_Wait(&mpi_request, MPI_STATUS_IGNORE);
    setMPIRequest(*icet_request, mpi_request);

    destroy_request(*icet_request);
    *icet_request = ICET_COMM_REQUEST_NULL;
Andy Cedilnik's avatar
Andy Cedilnik committed
530 531
}

532 533
static int  MPIWaitany(IceTCommunicator self,
                       int count, IceTCommRequest *array_of_requests)
Andy Cedilnik's avatar
Andy Cedilnik committed
534
{
535
    MPI_Request *mpi_requests;
536
    int idx;
Andy Cedilnik's avatar
Andy Cedilnik committed
537

538 539
    /* To remove warning */
    (void)self;
540 541

    mpi_requests = malloc(sizeof(MPI_Request)*count);
542
    if (mpi_requests == NULL) {
543 544
        icetRaiseError(ICET_OUT_OF_MEMORY,
                       "Could not allocate array for MPI requests.");
545 546 547
        return -1;
    }

548
    for (idx = 0; idx < count; idx++) {
549
        mpi_requests[idx] = getMPIRequest(array_of_requests[idx]);
Andy Cedilnik's avatar
Andy Cedilnik committed
550 551
    }

552 553 554
    MPI_Waitany(count, mpi_requests, &idx, MPI_STATUS_IGNORE);

    setMPIRequest(array_of_requests[idx], mpi_requests[idx]);
555 556
    destroy_request(array_of_requests[idx]);
    array_of_requests[idx] = ICET_COMM_REQUEST_NULL;
557 558

    free(mpi_requests);
Andy Cedilnik's avatar
Andy Cedilnik committed
559

560
    return idx;
Andy Cedilnik's avatar
Andy Cedilnik committed
561 562
}

563
static int MPIComm_size(IceTCommunicator self)
Andy Cedilnik's avatar
Andy Cedilnik committed
564 565 566 567 568 569
{
    int size;
    MPI_Comm_size(MPI_COMM, &size);
    return size;
}

570
static int MPIComm_rank(IceTCommunicator self)
Andy Cedilnik's avatar
Andy Cedilnik committed
571 572 573 574 575
{
    int rank;
    MPI_Comm_rank(MPI_COMM, &rank);
    return rank;
}