Commit 8e5447bf authored by Kenneth Moreland's avatar Kenneth Moreland
Browse files

Merge branch 'ScalingStudies'

parents 9fd33608 9e1c92fa
......@@ -29,73 +29,76 @@
#define ICET_MPI_TEMP_BUFFER_0 (ICET_COMMUNICATION_LAYER_START | (IceTEnum)0x00)
static IceTCommunicator Duplicate(IceTCommunicator self);
static void Destroy(IceTCommunicator self);
static void Barrier(IceTCommunicator self);
static void Send(IceTCommunicator self,
const void *buf,
int count,
IceTEnum datatype,
int dest,
int tag);
static void Recv(IceTCommunicator self,
void *buf,
int count,
IceTEnum datatype,
int src,
int tag);
static void Sendrecv(IceTCommunicator self,
const void *sendbuf,
int sendcount,
IceTEnum sendtype,
int dest,
int sendtag,
void *recvbuf,
int recvcount,
IceTEnum recvtype,
int src,
int recvtag);
static void Gather(IceTCommunicator self,
const void *sendbuf,
int sendcount,
IceTEnum datatype,
void *recvbuf,
int root);
static void Gatherv(IceTCommunicator self,
const void *sendbuf,
int sendcount,
static IceTCommunicator MPIDuplicate(IceTCommunicator self);
static IceTCommunicator MPISubset(IceTCommunicator self,
int count,
IceTInt32 *ranks);
static void MPIDestroy(IceTCommunicator self);
static void MPIBarrier(IceTCommunicator self);
static void MPISend(IceTCommunicator self,
const void *buf,
int count,
IceTEnum datatype,
void *recvbuf,
const int *recvcounts,
const int *recvoffsets,
int root);
static void Allgather(IceTCommunicator self,
int dest,
int tag);
static void MPIRecv(IceTCommunicator self,
void *buf,
int count,
IceTEnum datatype,
int src,
int tag);
static void MPISendrecv(IceTCommunicator self,
const void *sendbuf,
int sendcount,
IceTEnum sendtype,
int dest,
int sendtag,
void *recvbuf,
int recvcount,
IceTEnum recvtype,
int src,
int recvtag);
static void MPIGather(IceTCommunicator self,
const void *sendbuf,
int sendcount,
IceTEnum datatype,
void *recvbuf);
static void Alltoall(IceTCommunicator self,
const void *sendbuf,
int sendcount,
IceTEnum datatype,
void *recvbuf);
static IceTCommRequest Isend(IceTCommunicator self,
const void *buf,
int count,
IceTEnum datatype,
int dest,
int tag);
static IceTCommRequest Irecv(IceTCommunicator self,
void *buf,
int count,
IceTEnum datatype,
int src,
int tag);
static void Waitone(IceTCommunicator self, IceTCommRequest *request);
static int Waitany(IceTCommunicator self,
int count, IceTCommRequest *array_of_requests);
static int Comm_size(IceTCommunicator self);
static int Comm_rank(IceTCommunicator self);
void *recvbuf,
int root);
static void MPIGatherv(IceTCommunicator self,
const void *sendbuf,
int sendcount,
IceTEnum datatype,
void *recvbuf,
const int *recvcounts,
const int *recvoffsets,
int root);
static void MPIAllgather(IceTCommunicator self,
const void *sendbuf,
int sendcount,
IceTEnum datatype,
void *recvbuf);
static void MPIAlltoall(IceTCommunicator self,
const void *sendbuf,
int sendcount,
IceTEnum datatype,
void *recvbuf);
static IceTCommRequest MPIIsend(IceTCommunicator self,
const void *buf,
int count,
IceTEnum datatype,
int dest,
int tag);
static IceTCommRequest MPIIrecv(IceTCommunicator self,
void *buf,
int count,
IceTEnum datatype,
int src,
int tag);
static void MPIWaitone(IceTCommunicator self, IceTCommRequest *request);
static int MPIWaitany(IceTCommunicator self,
int count, IceTCommRequest *array_of_requests);
static int MPIComm_size(IceTCommunicator self);
static int MPIComm_rank(IceTCommunicator self);
typedef struct IceTMPICommRequestInternalsStruct {
MPI_Request request;
......@@ -189,33 +192,39 @@ static void ErrorHandler(MPI_Comm *comm, int *errorno, ...)
IceTCommunicator icetCreateMPICommunicator(MPI_Comm mpi_comm)
{
IceTCommunicator comm = malloc(sizeof(struct IceTCommunicatorStruct));
IceTCommunicator comm;
#ifdef BREAK_ON_MPI_ERROR
MPI_Errhandler eh;
#endif
if (mpi_comm == MPI_COMM_NULL) {
return ICET_COMM_NULL;
}
comm = malloc(sizeof(struct IceTCommunicatorStruct));
if (comm == NULL) {
icetRaiseError("Could not allocate memory for IceTCommunicator.",
ICET_OUT_OF_MEMORY);
return NULL;
}
comm->Duplicate = Duplicate;
comm->Destroy = Destroy;
comm->Barrier = Barrier;
comm->Send = Send;
comm->Recv = Recv;
comm->Sendrecv = Sendrecv;
comm->Gather = Gather;
comm->Gatherv = Gatherv;
comm->Allgather = Allgather;
comm->Alltoall = Alltoall;
comm->Isend = Isend;
comm->Irecv = Irecv;
comm->Wait = Waitone;
comm->Waitany = Waitany;
comm->Comm_size = Comm_size;
comm->Comm_rank = Comm_rank;
comm->Duplicate = MPIDuplicate;
comm->Subset = MPISubset;
comm->Destroy = MPIDestroy;
comm->Barrier = MPIBarrier;
comm->Send = MPISend;
comm->Recv = MPIRecv;
comm->Sendrecv = MPISendrecv;
comm->Gather = MPIGather;
comm->Gatherv = MPIGatherv;
comm->Allgather = MPIAllgather;
comm->Alltoall = MPIAlltoall;
comm->Isend = MPIIsend;
comm->Irecv = MPIIrecv;
comm->Wait = MPIWaitone;
comm->Waitany = MPIWaitany;
comm->Comm_size = MPIComm_size;
comm->Comm_rank = MPIComm_rank;
comm->data = malloc(sizeof(MPI_Comm));
if (comm->data == NULL) {
......@@ -243,25 +252,56 @@ IceTCommunicator icetCreateMPICommunicator(MPI_Comm mpi_comm)
void icetDestroyMPICommunicator(IceTCommunicator comm)
{
comm->Destroy(comm);
if (comm != ICET_COMM_NULL) {
comm->Destroy(comm);
}
}
#define MPI_COMM (*((MPI_Comm *)self->data))
static IceTCommunicator Duplicate(IceTCommunicator self)
static IceTCommunicator MPIDuplicate(IceTCommunicator self)
{
return icetCreateMPICommunicator(MPI_COMM);
if (self != ICET_COMM_NULL) {
return icetCreateMPICommunicator(MPI_COMM);
} else {
return ICET_COMM_NULL;
}
}
static void Destroy(IceTCommunicator self)
static IceTCommunicator MPISubset(IceTCommunicator self,
int count,
IceTInt32 *ranks)
{
MPI_Group original_group;
MPI_Group subset_group;
MPI_Comm subset_comm;
IceTCommunicator result;
MPI_Comm_group(MPI_COMM, &original_group);
MPI_Group_incl(original_group, count, ranks, &subset_group);
MPI_Comm_create(MPI_COMM, subset_group, &subset_comm);
result = icetCreateMPICommunicator(subset_comm);
if (subset_comm != MPI_COMM_NULL) {
MPI_Comm_free(&subset_comm);
}
MPI_Group_free(&subset_group);
MPI_Group_free(&original_group);
return result;
}
static void MPIDestroy(IceTCommunicator self)
{
MPI_Comm_free((MPI_Comm *)self->data);
free(self->data);
free(self);
}
static void Barrier(IceTCommunicator self)
static void MPIBarrier(IceTCommunicator self)
{
MPI_Barrier(MPI_COMM);
}
......@@ -281,41 +321,41 @@ static void Barrier(IceTCommunicator self)
break; \
}
static void Send(IceTCommunicator self,
const void *buf,
int count,
IceTEnum datatype,
int dest,
int tag)
static void MPISend(IceTCommunicator self,
const void *buf,
int count,
IceTEnum datatype,
int dest,
int tag)
{
MPI_Datatype mpidatatype;
CONVERT_DATATYPE(datatype, mpidatatype);
MPI_Send((void *)buf, count, mpidatatype, dest, tag, MPI_COMM);
}
static void Recv(IceTCommunicator self,
void *buf,
int count,
IceTEnum datatype,
int src,
int tag)
static void MPIRecv(IceTCommunicator self,
void *buf,
int count,
IceTEnum datatype,
int src,
int tag)
{
MPI_Datatype mpidatatype;
CONVERT_DATATYPE(datatype, mpidatatype);
MPI_Recv(buf, count, mpidatatype, src, tag, MPI_COMM, MPI_STATUS_IGNORE);
}
static void Sendrecv(IceTCommunicator self,
const void *sendbuf,
int sendcount,
IceTEnum sendtype,
int dest,
int sendtag,
void *recvbuf,
int recvcount,
IceTEnum recvtype,
int src,
int recvtag)
static void MPISendrecv(IceTCommunicator self,
const void *sendbuf,
int sendcount,
IceTEnum sendtype,
int dest,
int sendtag,
void *recvbuf,
int recvcount,
IceTEnum recvtype,
int src,
int recvtag)
{
MPI_Datatype mpisendtype;
MPI_Datatype mpirecvtype;
......@@ -327,12 +367,12 @@ static void Sendrecv(IceTCommunicator self,
MPI_STATUS_IGNORE);
}
static void Gather(IceTCommunicator self,
const void *sendbuf,
int sendcount,
IceTEnum datatype,
void *recvbuf,
int root)
static void MPIGather(IceTCommunicator self,
const void *sendbuf,
int sendcount,
IceTEnum datatype,
void *recvbuf,
int root)
{
MPI_Datatype mpitype;
CONVERT_DATATYPE(datatype, mpitype);
......@@ -356,14 +396,14 @@ static void Gather(IceTCommunicator self,
MPI_COMM);
}
static void Gatherv(IceTCommunicator self,
const void *sendbuf,
int sendcount,
IceTEnum datatype,
void *recvbuf,
const int *recvcounts,
const int *recvoffsets,
int root)
static void MPIGatherv(IceTCommunicator self,
const void *sendbuf,
int sendcount,
IceTEnum datatype,
void *recvbuf,
const int *recvcounts,
const int *recvoffsets,
int root)
{
MPI_Datatype mpitype;
CONVERT_DATATYPE(datatype, mpitype);
......@@ -388,11 +428,11 @@ static void Gatherv(IceTCommunicator self,
root, MPI_COMM);
}
static void Allgather(IceTCommunicator self,
const void *sendbuf,
int sendcount,
IceTEnum datatype,
void *recvbuf)
static void MPIAllgather(IceTCommunicator self,
const void *sendbuf,
int sendcount,
IceTEnum datatype,
void *recvbuf)
{
MPI_Datatype mpitype;
CONVERT_DATATYPE(datatype, mpitype);
......@@ -416,11 +456,11 @@ static void Allgather(IceTCommunicator self,
MPI_COMM);
}
static void Alltoall(IceTCommunicator self,
const void *sendbuf,
int sendcount,
IceTEnum datatype,
void *recvbuf)
static void MPIAlltoall(IceTCommunicator self,
const void *sendbuf,
int sendcount,
IceTEnum datatype,
void *recvbuf)
{
MPI_Datatype mpitype;
CONVERT_DATATYPE(datatype, mpitype);
......@@ -430,12 +470,12 @@ static void Alltoall(IceTCommunicator self,
MPI_COMM);
}
static IceTCommRequest Isend(IceTCommunicator self,
const void *buf,
int count,
IceTEnum datatype,
int dest,
int tag)
static IceTCommRequest MPIIsend(IceTCommunicator self,
const void *buf,
int count,
IceTEnum datatype,
int dest,
int tag)
{
IceTCommRequest icet_request;
MPI_Request mpi_request;
......@@ -451,12 +491,12 @@ static IceTCommRequest Isend(IceTCommunicator self,
return icet_request;
}
static IceTCommRequest Irecv(IceTCommunicator self,
void *buf,
int count,
IceTEnum datatype,
int src,
int tag)
static IceTCommRequest MPIIrecv(IceTCommunicator self,
void *buf,
int count,
IceTEnum datatype,
int src,
int tag)
{
IceTCommRequest icet_request;
MPI_Request mpi_request;
......@@ -472,7 +512,7 @@ static IceTCommRequest Irecv(IceTCommunicator self,
return icet_request;
}
static void Waitone(IceTCommunicator self, IceTCommRequest *icet_request)
static void MPIWaitone(IceTCommunicator self, IceTCommRequest *icet_request)
{
MPI_Request mpi_request;
......@@ -489,8 +529,8 @@ static void Waitone(IceTCommunicator self, IceTCommRequest *icet_request)
*icet_request = ICET_COMM_REQUEST_NULL;
}
static int Waitany(IceTCommunicator self,
int count, IceTCommRequest *array_of_requests)
static int MPIWaitany(IceTCommunicator self,
int count, IceTCommRequest *array_of_requests)
{
MPI_Request *mpi_requests;
int idx;
......@@ -520,14 +560,14 @@ static int Waitany(IceTCommunicator self,
return idx;
}
static int Comm_size(IceTCommunicator self)
static int MPIComm_size(IceTCommunicator self)
{
int size;
MPI_Comm_size(MPI_COMM, &size);
return size;
}
static int Comm_rank(IceTCommunicator self)
static int MPIComm_rank(IceTCommunicator self)
{
int rank;
MPI_Comm_rank(MPI_COMM, &rank);
......
......@@ -34,6 +34,12 @@ IceTCommunicator icetCommDuplicate()
return comm->Duplicate(comm);
}
IceTCommunicator icetCommSubset(int count, IceTInt32 *ranks)
{
IceTCommunicator comm = icetGetCommunicator();
return comm->Subset(comm, count, ranks);
}
void icetCommBarrier()
{
IceTCommunicator comm = icetGetCommunicator();
......
......@@ -50,6 +50,10 @@ struct IceTCommunicatorStruct {
struct IceTCommunicatorStruct *
(*Duplicate)(struct IceTCommunicatorStruct *self);
void (*Destroy)(struct IceTCommunicatorStruct *self);
struct IceTCommunicatorStruct *
(*Subset)(struct IceTCommunicatorStruct *self,
int count,
IceTInt32 *ranks);
void (*Barrier)(struct IceTCommunicatorStruct *self);
void (*Send)(struct IceTCommunicatorStruct *self,
const void *buf,
......@@ -123,6 +127,7 @@ struct IceTCommunicatorStruct {
};
typedef struct IceTCommunicatorStruct *IceTCommunicator;
#define ICET_COMM_NULL ((IceTCommunicator)NULL)
ICET_EXPORT IceTDouble icetWallTime(void);
......
......@@ -22,6 +22,8 @@ extern "C" {
/* All of these methods call the associated method in the communicator for
the current context. */
ICET_EXPORT IceTCommunicator icetCommDuplicate();
ICET_EXPORT IceTCommunicator icetCommSubset(int count,
IceTInt32 *ranks);
ICET_EXPORT void icetCommBarrier();
ICET_EXPORT void icetCommSend(const void *buf,
IceTSizeType count,
......
......@@ -92,6 +92,8 @@ static IceTBoolean g_do_magic_k_study;
static IceTInt g_max_magic_k;
static IceTBoolean g_do_image_split_study;
static IceTInt g_min_image_split;
static IceTBoolean g_do_scaling_study_factor_2;
static IceTBoolean g_do_scaling_study_factor_2_3;
static float g_color[4];
......@@ -116,10 +118,16 @@ static void usage(char *argv[])
printstat(" -radixk Use the radix-k single-image strategy.\n");
printstat(" -tree Use the tree single-image strategy.\n");
printstat(" -magic-k-study <num> Use the radix-k single-image strategy and repeat for\n"
" multiple values of k, up to <num>, doubling each time.\n");
" multiple values of k, up to <num>, doubling each time.\n");
printstat(" -max-image-split-study <num> Repeat the test for multiple maximum image\n"
" splits starting at <num> and doubling each time.\n");
printstat(" -h, -help Print this help message.\n");
" splits starting at <num> and doubling each time.\n");
printstat(" -scaling-study-factor-2 Perform a scaling study for all process counts\n"
" that are a factor of 2.\n");
printstat(" -scaling-study-factor-2-3 Perform a scaling study that includes all\n"
" process counts that are a factor of 2 plus all process\n"
" counts that are a factor of 3 plus most process counts\n"
" that have factors of 2 and 3.\n");
printstat(" -h, -help Print this help message.\n");
printstat("\nFor general testing options, try -h or -help before test name.\n");
}
......@@ -143,6 +151,8 @@ static void parse_arguments(int argc, char *argv[])
g_max_magic_k = 0;
g_do_image_split_study = ICET_FALSE;
g_min_image_split = 0;
g_do_scaling_study_factor_2 = ICET_FALSE;
g_do_scaling_study_factor_2_3 = ICET_FALSE;
for (arg = 1; arg < argc; arg++) {
if (strcmp(argv[arg], "-tilesx") == 0) {
......@@ -191,6 +201,10 @@ static void parse_arguments(int argc, char *argv[])
g_single_image_strategy = ICET_SINGLE_IMAGE_STRATEGY_RADIXK;
arg++;
g_min_image_split = atoi(argv[arg]);
} else if (strcmp(argv[arg], "-scaling-study-factor-2") == 0) {
g_do_scaling_study_factor_2 = ICET_TRUE;
} else if (strcmp(argv[arg], "-scaling-study-factor-2-3") == 0) {
g_do_scaling_study_factor_2_3 = ICET_TRUE;
} else if ( (strcmp(argv[arg], "-h") == 0)
|| (strcmp(argv[arg], "-help")) ) {
usage(argv);
......@@ -910,38 +924,8 @@ static int SimpleTimingDoRender()
return TEST_PASSED;
}
int SimpleTimingRun()
static int SimpleTimingDoParameterStudies()
{
IceTInt rank;
icetGetIntegerv(ICET_RANK, &rank);
if (rank == 0) {
printf("HEADER,"
"num processes,"
"multi-tile strategy,"
"single-image strategy,"
"tiles x,"
"tiles y,"
"width,"
"height,"
"transparent,"
"interlacing,"
"collection,"
"max image split,"
"frame,"
"render time,"
"buffer read time,"
"buffer write time,"
"compress time,"
"blend time,"
"draw time,"
"composite time,"
"collect time,"
"bytes sent,"
"frame time\n");
}
if (g_do_magic_k_study) {
IceTContext original_context = icetGetContext();
IceTInt magic_k;
......@@ -1023,6 +1007,224 @@ int SimpleTimingRun()
}
}
static IceTCommunicator MakeCommSubset(IceTInt size, IceTInt offset)
{
IceTInt32 *ranks;
IceTInt rank_index;
IceTCommunicator old_comm;
IceTCommunicator new_comm;
ranks = malloc(size*sizeof(IceTInt32));
for (rank_index = 0; rank_index < size; rank_index++) {
ranks[rank_index] = rank_index + offset;
}
old_comm = icetGetCommunicator();
new_comm = old_comm->Subset(old_comm, size, ranks);
free(ranks);
return new_comm;
}
static int SimpleTimingDoScalingStudyFactor2()
{
IceTInt size;