Commit 6e0d6c9b authored by Andrew J. Burns (Cont's avatar Andrew J. Burns (Cont

Added DSM capabilities to Fortran interface, simple array passing; Also added test for Fortran DSM

parent 8a80f9a7
......@@ -29,7 +29,7 @@ int main(int argc, char *argv[])
// Initializing objects
//since the start and end ids are larger than the size there are no buffers alloted
//thus, not blockage occurs
//thus, no blockage occurs
XdmfDSMCommMPI * testComm = new XdmfDSMCommMPI();
testComm->DupComm(comm);
testComm->Init();
......
......@@ -3,7 +3,7 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR})
option(XDMF_BUILD_EXODUS_IO OFF)
option(XDMF_BUILD_PARTITIONER OFF)
option(XDMF_BUILD_FORTRAN OFF)
option(XDMF_BUILD_DSM OFF)
set(XdmfUtilsSources
XdmfDiff
......@@ -15,6 +15,10 @@ if(XDMF_BUILD_FORTRAN)
set(XdmfUtilsSources ${XdmfUtilsSources} XdmfFortran)
endif(XDMF_BUILD_FORTRAN)
if(XDMF_BUILD_DSM)
add_definitions(-DXDMF_BUILD_DSM)
endif(XDMF_BUILD_DSM)
if(XDMF_BUILD_EXODUS_IO)
set(XDMF_SWIG_FLAGS ${XDMF_SWIG_FLAGS} -DXDMF_BUILD_EXODUS_IO)
find_package(Exodus REQUIRED)
......
......@@ -57,6 +57,11 @@
#include <stdio.h>
#include <stdarg.h>
#ifdef XDMF_BUILD_DSM
#include <mpi.h>
#include "XdmfHDF5WriterDSM.hpp"
#include "XdmfHDF5ControllerDSM.hpp"
#endif
template <typename T>
void
......@@ -300,6 +305,7 @@ XdmfFortran::XdmfFortran() :
mOrigin(shared_ptr<XdmfArray>()),
mDimensions(shared_ptr<XdmfArray>()),
mHeavyDataWriter(shared_ptr<XdmfHeavyDataWriter>()),
mDSMWriter(shared_ptr<XdmfHDF5WriterDSM>()),
mMaxFileSize(0),
mAllowSetSplitting(false)
{
......@@ -7536,10 +7542,318 @@ XdmfFortran::initHDF5(const char * const xmlFilePath,
shared_ptr<XdmfHDF5Writer> writer = XdmfHDF5Writer::New(xmlFilePath);
writer->setFileSizeLimit(mMaxFileSize);
writer->setAllowSetSplitting(mAllowSetSplitting);
writer->setReleaseData( release );
writer->setReleaseData(release);
mHeavyDataWriter = writer;
}
#ifdef XDMF_BUILD_DSM
void
XdmfFortran::initDSMServer(const char * const filePath,
MPI_Comm comm,
int bufferSize,
int startCoreIndex,
int endCoreIndex)
{
// Non-Threaded version
std::string writtenFile(filePath);
if (bufferSize > 0) {
mDSMWriter = XdmfHDF5WriterDSM::New(writtenFile, comm, (unsigned int) bufferSize, startCoreIndex, endCoreIndex);
mDSMWriter->setMode(XdmfHeavyDataWriter::Hyperslab);
}
else {
try {
XdmfError::message(XdmfError::FATAL,
"Error: Non-positive DSM buffer size.");
}
catch (XdmfError e) {
throw e;
}
}
}
void
XdmfFortran::acceptDSM(int numConnections)
{
if (mDSMWriter) {
mDSMWriter->getServerBuffer()->GetComm()->OpenPort();
mDSMWriter->getServerBuffer()->SendAccept(numConnections);
}
else {
try {
XdmfError::message(XdmfError::FATAL,
"Error: Attempting to accept connection when DSM is not set up.");
}
catch (XdmfError e) {
throw e;
}
}
}
void
XdmfFortran::closeDSMPort()
{
if (mDSMWriter) {
mDSMWriter->getServerBuffer()->GetComm()->ClosePort();
}
else {
try {
XdmfError::message(XdmfError::FATAL,
"Error: Attempting to close a port when DSM is not set up.");
}
catch (XdmfError e) {
throw e;
}
}
}
void
XdmfFortran::connectDSM(const char * const filePath,
MPI_Comm comm)
{
std::string writtenFile(filePath);
XdmfDSMCommMPI * dsmComm = new XdmfDSMCommMPI();
dsmComm->DupComm(comm);
dsmComm->Init();
XdmfDSMBuffer * dsmBuffer = new XdmfDSMBuffer();
dsmBuffer->SetIsServer(false);
dsmBuffer->SetComm(dsmComm);
dsmBuffer->SetIsConnected(true);
mDSMWriter = XdmfHDF5WriterDSM::New(writtenFile, dsmBuffer);
mDSMWriter->setMode(XdmfHeavyDataWriter::Hyperslab);
// Currently uses default config file name
mDSMWriter->getServerBuffer()->GetComm()->ReadDsmPortName();
mDSMWriter->getServerManager()->Connect();
// To check if the DSM writer is using server mode
// bool test = mDSMWriter->getServerMode();
}
MPI_Comm
XdmfFortran::getDSMInterComm()
{
// Sanity check
if (mDSMWriter) {
return mDSMWriter->getServerBuffer()->GetComm()->GetInterComm();
}
else {
return MPI_COMM_NULL;
}
}
MPI_Comm
XdmfFortran::getDSMIntraComm()
{
// Sanity check
if (mDSMWriter) {
return mDSMWriter->getServerBuffer()->GetComm()->GetIntraComm();
}
else {
return MPI_COMM_NULL;
}
}
// Call only on one core
void
XdmfFortran::stopDSM()
{
if (mDSMWriter) {
mDSMWriter->stopDSM();
}
else {
try {
XdmfError::message(XdmfError::FATAL,
"Error: Stop called when DSM not initialized.");
}
catch (XdmfError e) {
throw e;
}
}
mDSMWriter = shared_ptr<XdmfHDF5WriterDSM>();
}
void
XdmfFortran::readFromDSM(const char * const dsmDataSetPath,
const int arrayType,
void * values,
const int start,
const int stride,
const int dimensions,
const int dataspace)
{
if (mDSMWriter) {
shared_ptr<const XdmfArrayType> writtenArrayType = shared_ptr<const XdmfArrayType>();
switch(arrayType) {
case XDMF_ARRAY_TYPE_INT8:
writtenArrayType = XdmfArrayType::Int8();
break;
case XDMF_ARRAY_TYPE_INT16:
writtenArrayType = XdmfArrayType::Int16();
break;
case XDMF_ARRAY_TYPE_INT32:
writtenArrayType = XdmfArrayType::Int32();
break;
case XDMF_ARRAY_TYPE_INT64:
writtenArrayType = XdmfArrayType::Int64();
break;
case XDMF_ARRAY_TYPE_UINT8:
writtenArrayType = XdmfArrayType::UInt8();
break;
case XDMF_ARRAY_TYPE_UINT16:
writtenArrayType = XdmfArrayType::UInt16();
break;
case XDMF_ARRAY_TYPE_UINT32:
writtenArrayType = XdmfArrayType::UInt32();
break;
case XDMF_ARRAY_TYPE_FLOAT32:
writtenArrayType = XdmfArrayType::Float32();
break;
case XDMF_ARRAY_TYPE_FLOAT64:
writtenArrayType = XdmfArrayType::Float64();
break;
default:
try {
XdmfError::message(XdmfError::FATAL,
"Invalid array number type");
}
catch (XdmfError e) {
throw e;
}
}
std::vector<unsigned int> startVector;
startVector.push_back(start);
std::vector<unsigned int> strideVector;
strideVector.push_back(stride);
std::vector<unsigned int> dimVector;
dimVector.push_back(dimensions);
std::vector<unsigned int> dataVector;
dataVector.push_back(dataspace);
std::string writtenPath(dsmDataSetPath);
shared_ptr<XdmfHDF5ControllerDSM> writerController =
XdmfHDF5ControllerDSM::New(mDSMWriter->getFilePath(),
writtenPath,
writtenArrayType,
startVector,
strideVector,
dimVector,
dataVector,
mDSMWriter->getServerBuffer());
shared_ptr<XdmfArray> readArray = XdmfArray::New();
readArray->insert(writerController);
readArray->read();
readFromArray(readArray,
arrayType,
values,
dimensions,
0,
1,
1);
}
else {
try {
XdmfError::message(XdmfError::FATAL,
"Error: Attempting to read from DSM when DSM is not set up.");
}
catch (XdmfError e) {
throw e;
}
}
}
void
XdmfFortran::writeToDSM(const char * const dsmDataSetPath,
const int arrayType,
void * values,
const int start,
const int stride,
const int dimensions,
const int dataspace)
{
if (mDSMWriter) {
shared_ptr<const XdmfArrayType> writtenArrayType = shared_ptr<const XdmfArrayType>();
switch(arrayType) {
case XDMF_ARRAY_TYPE_INT8:
writtenArrayType = XdmfArrayType::Int8();
break;
case XDMF_ARRAY_TYPE_INT16:
writtenArrayType = XdmfArrayType::Int16();
break;
case XDMF_ARRAY_TYPE_INT32:
writtenArrayType = XdmfArrayType::Int32();
break;
case XDMF_ARRAY_TYPE_INT64:
writtenArrayType = XdmfArrayType::Int64();
break;
case XDMF_ARRAY_TYPE_UINT8:
writtenArrayType = XdmfArrayType::UInt8();
break;
case XDMF_ARRAY_TYPE_UINT16:
writtenArrayType = XdmfArrayType::UInt16();
break;
case XDMF_ARRAY_TYPE_UINT32:
writtenArrayType = XdmfArrayType::UInt32();
break;
case XDMF_ARRAY_TYPE_FLOAT32:
writtenArrayType = XdmfArrayType::Float32();
break;
case XDMF_ARRAY_TYPE_FLOAT64:
writtenArrayType = XdmfArrayType::Float64();
break;
default:
try {
XdmfError::message(XdmfError::FATAL,
"Invalid array number type");
}
catch (XdmfError e) {
throw e;
}
}
std::vector<unsigned int> startVector;
startVector.push_back(start);
std::vector<unsigned int> strideVector;
strideVector.push_back(stride);
std::vector<unsigned int> dimVector;
dimVector.push_back(dimensions);
std::vector<unsigned int> dataVector;
dataVector.push_back(dataspace);
std::string writtenPath(dsmDataSetPath);
shared_ptr<XdmfHDF5ControllerDSM> writerController =
XdmfHDF5ControllerDSM::New(mDSMWriter->getFilePath(),
writtenPath,
writtenArrayType,
startVector,
strideVector,
dimVector,
dataVector,
mDSMWriter->getServerBuffer());
shared_ptr<XdmfArray> writtenArray = XdmfArray::New();
writeToArray(writtenArray,
dimensions,
arrayType,
values);
writtenArray->insert(writerController);
writtenArray->accept(mDSMWriter);
}
else {
try {
XdmfError::message(XdmfError::FATAL,
"Error: Attempting to write to DSM when DSM is not set up.");
}
catch (XdmfError e) {
throw e;
}
}
}
#endif
void
XdmfFortran::read(const char * const xmlFilePath)
{
......@@ -7547,7 +7861,6 @@ XdmfFortran::read(const char * const xmlFilePath)
mDomain = shared_dynamic_cast<XdmfDomain>(reader->read( xmlFilePath ));
}
//temporary fix, hopefully
int
XdmfFortran::setTopologyPolyline(const unsigned int nodesPerElement,
......@@ -9519,6 +9832,114 @@ extern "C"
xdmfFortran->write(xmlFilePath, *datalimit, *release);
}
#ifdef XDMF_BUILD_DSM
void
XdmfInitDSMServer(long * pointer,
char * filePath,
MPI_Fint * comm,
int * bufferSize,
int * startCoreIndex,
int * endCoreIndex)
{
XdmfFortran * xdmfFortran = reinterpret_cast<XdmfFortran *>(*pointer);
MPI_Comm tempComm = MPI_Comm_f2c(*comm);
xdmfFortran->initDSMServer(filePath,
tempComm,
*bufferSize,
*startCoreIndex,
*endCoreIndex);
}
void
XdmfAcceptDSM(long * pointer, int * numConnections)
{
XdmfFortran * xdmfFortran = reinterpret_cast<XdmfFortran *>(*pointer);
xdmfFortran->acceptDSM(*numConnections);
}
void
XdmfCloseDSMPort(long * pointer)
{
XdmfFortran * xdmfFortran = reinterpret_cast<XdmfFortran *>(*pointer);
xdmfFortran->closeDSMPort();
}
void
XdmfConnectDSM(long * pointer,
char * filePath,
MPI_Fint * comm)
{
XdmfFortran * xdmfFortran = reinterpret_cast<XdmfFortran *>(*pointer);
MPI_Comm tempComm = MPI_Comm_f2c(*comm);
xdmfFortran->connectDSM(filePath, tempComm);
}
void
XdmfGetDSMInterComm(long * pointer, MPI_Fint * returnComm)
{
XdmfFortran * xdmfFortran = reinterpret_cast<XdmfFortran *>(*pointer);
MPI_Comm tempComm = xdmfFortran->getDSMInterComm();
*returnComm = MPI_Comm_c2f(tempComm);
}
void
XdmfGetDSMIntraComm(long * pointer, MPI_Fint * returnComm)
{
XdmfFortran * xdmfFortran = reinterpret_cast<XdmfFortran *>(*pointer);
MPI_Comm tempComm = xdmfFortran->getDSMIntraComm();
*returnComm = MPI_Comm_c2f(tempComm);
}
void
XdmfStopDSM(long * pointer)
{
XdmfFortran * xdmfFortran = reinterpret_cast<XdmfFortran *>(*pointer);
xdmfFortran->stopDSM();
}
void
XdmfReadFromDSM(long * pointer,
char * dsmDataSetPath,
int * arrayType,
void * values,
int * start,
int * stride,
int * dimensions,
int * dataspace)
{
XdmfFortran * xdmfFortran = reinterpret_cast<XdmfFortran *>(*pointer);
xdmfFortran->readFromDSM(dsmDataSetPath,
*arrayType,
values,
*start,
*stride,
*dimensions,
*dataspace);
}
void
XdmfWriteToDSM(long * pointer,
char * dsmDataSetPath,
int * arrayType,
void * values,
int * start,
int * stride,
int * dimensions,
int * dataspace)
{
XdmfFortran * xdmfFortran = reinterpret_cast<XdmfFortran *>(*pointer);
xdmfFortran->writeToDSM(dsmDataSetPath,
*arrayType,
values,
*start,
*stride,
*dimensions,
*dataspace);
}
#endif
void
XdmfWriteHDF5(long * pointer,
char * xmlFilePath,
......
......@@ -41,12 +41,16 @@ class XdmfRectilinearGrid;
class XdmfRegularGrid;
class XdmfUnstructuredGrid;
class XdmfHeavyDataWriter;
class XdmfHDF5WriterDSM;
//Includes
#include <stack>
#include <vector>
#include "XdmfUtils.hpp"
#include "XdmfSharedPtr.hpp"
#ifdef XDMF_BUILD_DSM
#include "mpi.h"
#endif
/**
* Array Type
......@@ -167,6 +171,15 @@ class XdmfHeavyDataWriter;
#define XdmfSetMaxFileSize xdmfsetmaxfilesize_
#define XdmfWrite xdmfwrite_
#define XdmfRead xdmfread_
#define XdmfInitDSMServer xdmfinitdsmserver_
#define XdmfAcceptDSM xdmfacceptdsm_
#define XdmfCloseDSMPort xdmfclosedsmport_
#define XdmfConnectDSM xdmfconnectdsm_
#define XdmfGetDSMInterComm xdmfgetdsmintercomm_
#define XdmfGetDSMIntraComm xdmfgetdsmintracomm_
#define XdmfStopDSM xdmfstopdsm_
#define XdmfReadFromDSM xdmfreadfromdsm_
#define XdmfWriteToDSM xdmfwritetodsm_
#define XdmfWriteHDF5 xdmfwritehdf5_
#define XdmfInitHDF5 xdmfinithdf5_
#define XdmfSetTopologyPolyline xdmfsettopologypolyline_
......@@ -2887,6 +2900,112 @@ public:
*/
void writeHDF5(const char * const xmlFilePath, const bool release);
#ifdef XDMF_BUILD_DSM
/**
* Starts up a dsm server on cores from the start index to the end index.
* The server cores are blocked and will not proceed past this point
* until the dsm is stopped.
*
* @param filePath A string denoting the file path to the virtual file.
* @param comm The Communicator between all cores,
* worker and server.
* @param bufferSize The size of the memory buffer allocated per core.
* @param startCoreIndex The core index on which the dsm section begins.
* @param endCoreIndex The core index on which the dsm section ends.
*/
void initDSMServer(const char * const filePath,
MPI_Comm comm,
int bufferSize,
int startCoreIndex,
int endCoreIndex);
/**
* Accepts new connections into an initialized dsm server.
*
* @param numConnections The number of incoming connections to accept.
*/
void acceptDSM(int numConnections);
/**
* Closes the currently open port to the comm.
*/
void closeDSMPort();
/**
* Connects to an initialized dsm server.
*
* @param filePath The file path that the DSMWriter will be writing to,
* should be the same as on the server side.
* @param comm The local communicator to be connected to the
* server communicator.
*/
void connectDSM(const char * const filePath, MPI_Comm comm);
/**
* Gets the communicator across the entire dsm and all connected to it.
*
* @return The overarching communicator.
*/
MPI_Comm getDSMInterComm();
/**
* Gets the local communicator that the worker cores share.
*
* @return The communicator for the local worker group
*/
MPI_Comm getDSMIntraComm();
/**
* Stops the currently running dsm server.
*/
void stopDSM();
/**
* Writes the provided values to the DSM using at the location generated by
* the provided start, stride, dimensions, and dataspace.
*
* @param dsmDataSetPath The path to the dataset being written to.
* Will overwrite current dsm if different.
* @param arrayType The data type of the data to be written to the set.
* @param values A pointer to the values to be written.
* @param start The starting index to write to.
* @param stride The increment between writing data.
* @param dimensions The number of data items written.
* @param dataspace The total number of data items in the dataset.
*/
void writeToDSM(const char * const dsmDataSetPath,
const int arrayType,
void * values,
const int start,
const int stride,
const int dimensions,
const int dataspace);
/**
* Reads the data in the dsm at the location generated from the
* start, stride, dimensions, and dataspace and places it into the
* provided pointer.
*
* @param dsmDataSetPath The path to the dataset being read.
* @param arrayType The data type of the data being read.
* @param values A pointer to the location that the data
* will be stored at.
* @param start The starting index of the read.
* @param stride The increment between read values.
* @param dimensions The amount of value to be read.
* @param dataspace The total number of data items in the dataset.
*/
void readFromDSM(const char * const dsmDataSetPath,
const int arrayType,
void * values,