Commit 6fd19037 authored by Kenneth Leiter's avatar Kenneth Leiter

ENH: Add DSM Support. Update swigging procedures to current code.

parent a3287659
......@@ -12,6 +12,10 @@ swig -v -c++ -python -o XdmfPython.cpp Xdmf.i
#include <XdmfCoreReader.hpp>
#include <XdmfHDF5Controller.hpp>
#include <XdmfHDF5Writer.hpp>
#ifdef XDMF_BUILD_DSM
#include <XdmfHDF5ControllerDSM.hpp>
#include <XdmfHDF5WriterDSM.hpp>
#endif
#include <XdmfHeavyDataController.hpp>
#include <XdmfHeavyDataWriter.hpp>
#include <XdmfInformation.hpp>
......
......@@ -58,7 +58,10 @@ set(XdmfCoreSources
)
if(XDMF_BUILD_DSM)
set(XdmfCoreSources ${XdmfCoreSources} XdmfHDF5WriterDSM)
set(XdmfCoreSources
${XdmfCoreSources}
XdmfHDF5ControllerDSM
XdmfHDF5WriterDSM)
endif(XDMF_BUILD_DSM)
add_library(XdmfCore ${XdmfCoreSources})
......
......@@ -13,6 +13,10 @@ swig -v -c++ -python -o XdmfCorePython.cpp XdmfCore.i
#include <XdmfHeavyDataWriter.hpp>
#include <XdmfHDF5Controller.hpp>
#include <XdmfHDF5Writer.hpp>
#ifdef XDMF_BUILD_DSM
#include <XdmfHDF5ControllerDSM.hpp>
#include <XdmfHDF5WriterDSM.hpp>
#endif
#include <XdmfInformation.hpp>
#include <XdmfItem.hpp>
#include <XdmfItemProperty.hpp>
......@@ -264,6 +268,10 @@ swig -v -c++ -python -o XdmfCorePython.cpp XdmfCore.i
%shared_ptr(XdmfCoreReader)
%shared_ptr(XdmfHDF5Controller)
%shared_ptr(XdmfHDF5Writer)
#ifdef XDMF_BUILD_DSM
%shared_ptr(XdmfHDF5ControllerDSM)
%shared_ptr(XdmfHDF5WriterDSM)
#endif
%shared_ptr(XdmfHeavyDataController)
%shared_ptr(XdmfHeavyDataWriter)
%shared_ptr(XdmfInformation)
......@@ -291,6 +299,11 @@ swig -v -c++ -python -o XdmfCorePython.cpp XdmfCore.i
%include XdmfHDF5Writer.hpp
%include XdmfWriter.hpp
#ifdef XDMF_BUILD_DSM
%include XdmfHDF5ControllerDSM.hpp
%include XdmfHDF5WriterDSM.hpp
#endif
%include XdmfArray.hpp
%include XdmfArrayType.hpp
......
......@@ -30,7 +30,12 @@ std::string XdmfHDF5Controller::getName() const
void XdmfHDF5Controller::read(XdmfArray * const array)
{
hid_t hdf5Handle = H5Fopen(mFilePath.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
this->read(array, H5P_DEFAULT);
}
void XdmfHDF5Controller::read(XdmfArray * const array, const int fapl)
{
hid_t hdf5Handle = H5Fopen(mFilePath.c_str(), H5F_ACC_RDONLY, fapl);
hid_t dataset = H5Dopen(hdf5Handle, mDataSetPath.c_str(), H5P_DEFAULT);
hid_t dataspace = H5Dget_space(dataset);
hssize_t numVals = H5Sget_simple_extent_npoints(dataspace);
......
......@@ -19,16 +19,24 @@ public:
/**
* Create a new controller for an hdf5 data set on disk.
*
* @param hdf5FilePath the location of the hdf5 file the data set resides in.
* @param dataSetPath the location of the dataset within the hdf5 file.
* @param size the size of the dataset to read.
* @param type the data type of the dataset to read.
*
* @return new HDF5 Controller.
*/
static boost::shared_ptr<XdmfHDF5Controller> New(const std::string & hdf5FilePath, const std::string & dataSetPath, const unsigned int size, const boost::shared_ptr<const XdmfArrayType> type);
std::string getName() const;
virtual std::string getName() const;
void read(XdmfArray * const array);
virtual void read(XdmfArray * const array);
protected:
XdmfHDF5Controller(const std::string & hdf5FilePath, const std::string & dataSetPath, const unsigned int size, const boost::shared_ptr<const XdmfArrayType> type);
void read(XdmfArray * const array, const int fapl);
private:
......
// Kenneth Leiter
// Xdmf Smart Pointer Test
#include <H5FDdsm.h>
#include <hdf5.h>
#include "XdmfHDF5ControllerDSM.hpp"
boost::shared_ptr<XdmfHDF5ControllerDSM> XdmfHDF5ControllerDSM::New(const std::string & hdf5FilePath, const std::string & dataSetPath, const unsigned int size, const boost::shared_ptr<const XdmfArrayType> type, H5FDdsmBuffer * const dsmBuffer)
{
boost::shared_ptr<XdmfHDF5ControllerDSM> p(new XdmfHDF5ControllerDSM(hdf5FilePath, dataSetPath, size, type, dsmBuffer));
return p;
}
XdmfHDF5ControllerDSM::XdmfHDF5ControllerDSM(const std::string & hdf5FilePath, const std::string & dataSetPath, const unsigned int size, const boost::shared_ptr<const XdmfArrayType> type, H5FDdsmBuffer * const dsmBuffer) :
XdmfHDF5Controller(hdf5FilePath, dataSetPath, size, type),
mDSMBuffer(dsmBuffer)
{
}
XdmfHDF5ControllerDSM::~XdmfHDF5ControllerDSM()
{
}
std::string XdmfHDF5ControllerDSM::getName() const
{
return "HDFDSM";
}
void XdmfHDF5ControllerDSM::read(XdmfArray * const array)
{
// Set file access property list for DSM
hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
// Use DSM driver
H5Pset_fapl_dsm(fapl, MPI_COMM_WORLD, mDSMBuffer);
// Read from DSM Buffer
XdmfHDF5Controller::read(array, fapl);
// Close file access property list
herr_t status = H5Pclose(fapl);
}
#ifndef XDMFHDF5CONTROLLERDSM_HPP_
#define XDMFHDF5CONTROLLERDSM_HPP_
// Forward Declarations
class H5FDdsmBuffer;
// Includes
#include "XdmfHDF5Controller.hpp"
/**
* @brief Couples an XdmfArray with HDF5 data stored in a DSM buffer.
*
* Serves as an interface between data stored in XdmfArrays and data stored in DSM buffers. When an Xdmf file is read from or
* written to a DSM buffer an XdmfHDF5ControllerDSM is attached to XdmfArrays. This allows data to be released
* from memory but still be accessible or have its location written to light data.
*/
class XdmfHDF5ControllerDSM : public XdmfHDF5Controller {
public:
virtual ~XdmfHDF5ControllerDSM();
/**
* Create a new controller for an hdf5 data set on disk.
*/
static boost::shared_ptr<XdmfHDF5ControllerDSM> New(const std::string & hdf5FilePath, const std::string & dataSetPath, const unsigned int size, const boost::shared_ptr<const XdmfArrayType> type, H5FDdsmBuffer * const dsmBuffer);
std::string getName() const;
void read(XdmfArray * const array);
protected:
XdmfHDF5ControllerDSM(const std::string & hdf5FilePath, const std::string & dataSetPath, const unsigned int size, const boost::shared_ptr<const XdmfArrayType> type, H5FDdsmBuffer * const dsmBuffer);
private:
XdmfHDF5ControllerDSM(const XdmfHDF5Controller & hdf5Controller); // Not implemented.
void operator=(const XdmfHDF5Controller & hdf5Controller); // Not implemented.
H5FDdsmBuffer * mDSMBuffer;
};
#endif /* XDMFHDF5CONTROLLER_HPP_ */
......@@ -23,7 +23,17 @@ XdmfHDF5Writer::~XdmfHDF5Writer()
{
}
void XdmfHDF5Writer::visit(XdmfArray & array, const boost::shared_ptr<XdmfBaseVisitor>)
boost::shared_ptr<XdmfHDF5Controller> XdmfHDF5Writer::createHDF5Controller(const std::string & hdf5FilePath, const std::string & dataSetPath, const unsigned int size, const boost::shared_ptr<const XdmfArrayType> type)
{
return XdmfHDF5Controller::New(hdf5FilePath, dataSetPath, size, type);
}
void XdmfHDF5Writer::visit(XdmfArray & array, const boost::shared_ptr<XdmfBaseVisitor> visitor)
{
this->write(array, H5P_DEFAULT);
}
void XdmfHDF5Writer::write(XdmfArray & array, const int fapl)
{
hid_t datatype = -1;
......@@ -100,11 +110,11 @@ void XdmfHDF5Writer::visit(XdmfArray & array, const boost::shared_ptr<XdmfBaseVi
if(H5Fis_hdf5(hdf5FilePath.c_str()) > 0)
{
hdf5Handle = H5Fopen(hdf5FilePath.c_str(), H5F_ACC_RDWR, H5P_DEFAULT);
hdf5Handle = H5Fopen(hdf5FilePath.c_str(), H5F_ACC_RDWR, fapl);
}
else
{
hdf5Handle = H5Fcreate(hdf5FilePath.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
hdf5Handle = H5Fcreate(hdf5FilePath.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, fapl);
}
hid_t dataset = H5Dopen(hdf5Handle, dataSetPath.str().c_str(), H5P_DEFAULT);
......@@ -164,7 +174,7 @@ void XdmfHDF5Writer::visit(XdmfArray & array, const boost::shared_ptr<XdmfBaseVi
// Attach a new controller to the array if needed.
if(mMode == Default || !array.getHeavyDataController())
{
boost::shared_ptr<XdmfHDF5Controller> newDataSetController = XdmfHDF5Controller::New(hdf5FilePath, dataSetPath.str(), array.getSize(), array.getArrayType());
boost::shared_ptr<XdmfHDF5Controller> newDataSetController = this->createHDF5Controller(hdf5FilePath, dataSetPath.str(), array.getSize(), array.getArrayType());
array.setHeavyDataController(newDataSetController);
mDataSetId++;
}
......
#ifndef XDMFHDF5WRITER_HPP_
#define XDMFHDF5WRITER_HPP_
// Forward Declarations
class XdmfArrayType;
class XdmfHDF5Controller;
// Includes
#include "XdmfHeavyDataWriter.hpp"
......@@ -28,12 +32,32 @@ public:
virtual ~XdmfHDF5Writer();
void visit(XdmfArray & array, const boost::shared_ptr<XdmfBaseVisitor> visitor);
virtual void visit(XdmfArray & array, const boost::shared_ptr<XdmfBaseVisitor> visitor);
protected:
XdmfHDF5Writer(const std::string & filePath);
/**
* Create a new HDF5 Controller that is able to read in a dataset after being written by this writer.
*
* @param hdf5FilePath the location of the hdf5 file the data set resides in.
* @param dataSetPath the location of the dataset within the hdf5 file.
* @param size the size of the dataset to read.
* @param type the data type of the dataset to read.
*
* @return new HDF5 Controller.
*/
virtual boost::shared_ptr<XdmfHDF5Controller> createHDF5Controller(const std::string & hdf5FilePath, const std::string & dataSetPath, const unsigned int size, const boost::shared_ptr<const XdmfArrayType> type);
/**
* Write the XdmfArray to a hdf5 file.
*
* @param array an XdmfArray to write to hdf5.
* @param fapl the file access property list for the hdf5 file on disk.
*/
void write(XdmfArray & array, const int fapl);
private:
XdmfHDF5Writer(const XdmfHDF5Writer & hdf5Writer); // Not implemented.
......
// Kenneth Leiter
// Xdmf Smart Pointer Test
#include <H5FDdsm.h>
#include <hdf5.h>
#include "XdmfHDF5ControllerDSM.hpp"
#include "XdmfHDF5WriterDSM.hpp"
boost::shared_ptr<XdmfHDF5WriterDSM> XdmfHDF5WriterDSM::New(const std::string & filePath, H5FDdsmBuffer * const dsmBuffer)
{
boost::shared_ptr<XdmfHDF5WriterDSM> p(new XdmfHDF5WriterDSM(filePath, dsmBuffer));
return p;
}
XdmfHDF5WriterDSM::XdmfHDF5WriterDSM(const std::string & filePath, H5FDdsmBuffer * const dsmBuffer) :
XdmfHDF5Writer(filePath),
mDSMBuffer(dsmBuffer)
{
}
XdmfHDF5WriterDSM::~XdmfHDF5WriterDSM()
{
}
boost::shared_ptr<XdmfHDF5Controller> XdmfHDF5WriterDSM::createHDF5Controller(const std::string & hdf5FilePath, const std::string & dataSetPath, const unsigned int size, const boost::shared_ptr<const XdmfArrayType> type)
{
return XdmfHDF5ControllerDSM::New(hdf5FilePath, dataSetPath, size, type, mDSMBuffer);
}
void XdmfHDF5WriterDSM::visit(XdmfArray & array, const boost::shared_ptr<XdmfBaseVisitor>)
{
// Set file access property list for DSM
hid_t fapl = H5Pcreate(H5P_FILE_ACCESS);
// Use DSM driver
H5Pset_fapl_dsm(fapl, MPI_COMM_WORLD, mDSMBuffer);
// Write to DSM Buffer
this->write(array, fapl);
// Close file access property list
herr_t status = H5Pclose(fapl);
}
#ifndef XDMFHDF5WRITERDSM_HPP_
#define XDMFHDF5WRITERDSM_HPP_
// Forward Declarations
class H5FDdsmBuffer;
// Includes
#include "XdmfHDF5Writer.hpp"
/**
* @brief Traverse the Xdmf graph and write heavy data stored in XdmfArrays to a DSM buffer.
*
* XdmfHDF5WriterDSM traverses an Xdmf graph structure and writes data stored in XdmfArrays to a DSM buffer.
* Writing begins by calling the accept() operation on any XdmfItem and supplying this writer
* as the parameter. The writer will write all XdmfArrays under the XdmfItem to a DSM Buffer
* It will also attach an XdmfHDF5Controller to all XdmfArrays that it writes to disk.
*
* This writer supports all heavy data writing modes listed in XdmfHeavyDataWriter.
*/
class XdmfHDF5WriterDSM : public XdmfHDF5Writer {
public:
/**
* Construct XdmfHDF5WriterDSM
*
* @param filePath the location of the hdf5 file to output to on disk.
* @param dsmBuffer the dsm buffer to write to.
* @return new XdmfHDF5WriterDSM.
*/
static boost::shared_ptr<XdmfHDF5WriterDSM> New(const std::string & filePath, H5FDdsmBuffer * const dsmBuffer);
virtual ~XdmfHDF5WriterDSM();
void visit(XdmfArray & array, const boost::shared_ptr<XdmfBaseVisitor> visitor);
protected:
XdmfHDF5WriterDSM(const std::string & filePath, H5FDdsmBuffer * const dsmBuffer);
virtual boost::shared_ptr<XdmfHDF5Controller> createHDF5Controller(const std::string & hdf5FilePath, const std::string & dataSetPath, const unsigned int size, const boost::shared_ptr<const XdmfArrayType> type);
private:
XdmfHDF5WriterDSM(const XdmfHDF5WriterDSM & hdf5Writer); // Not implemented.
void operator=(const XdmfHDF5WriterDSM & hdf5Writer); // Not implemented.
H5FDdsmBuffer * mDSMBuffer;
};
#endif /* XDMFHDF5WRITERDSM_HPP_ */
......@@ -21,7 +21,7 @@ class XdmfHeavyDataController {
public:
virtual ~XdmfHeavyDataController();
virtual ~XdmfHeavyDataController() = 0;
/**
* Get the path of the data set within the heavy data file owned by this controller.
......
......@@ -34,7 +34,7 @@ public:
Default, Overwrite, Append
};
virtual ~XdmfHeavyDataWriter();
virtual ~XdmfHeavyDataWriter() = 0;
/**
* Get the path to the heavy data file on disk this writer is writing to.
......
......@@ -16,7 +16,6 @@ ADD_TEST_CXX(TestXdmfHDF5Controller)
ADD_TEST_CXX(TestXdmfHDF5Writer)
if(XDMF_BUILD_DSM)
ADD_TEST_CXX(TestXdmfHDF5WriterDSM)
ADD_TEST_CXX(TestXdmfHDF5WriterDSMServer)
endif(XDMF_BUILD_DSM)
ADD_TEST_CXX(TestXdmfInformation)
......
#include <H5FDdsmManager.h>
#include "XdmfArray.hpp"
#include "XdmfArrayType.hpp"
#include "XdmfHDF5WriterDSM.hpp"
int main(int argc, char * argv[])
{
MPI_Init(&argc, &argv);
int rank, size;
MPI_Comm comm = MPI_COMM_WORLD;
MPI_Comm_rank(comm, &rank);
MPI_Comm_size(comm, &size);
// Create DSM Server
H5FDdsmManager * dsmServer = new H5FDdsmManager();
dsmServer->SetCommunicator(comm);
dsmServer->SetLocalBufferSizeMBytes(16);
dsmServer->SetDsmCommType(H5FD_DSM_COMM_SOCKET);
dsmServer->SetDsmIsServer(1);
dsmServer->SetServerHostName("default");
dsmServer->SetServerPort(22000);
dsmServer->CreateDSM();
dsmServer->PublishDSM();
// Create DSM Client
H5FDdsmManager * dsmClient = new H5FDdsmManager();
dsmClient->SetGlobalDebug(0);
dsmClient->SetCommunicator(comm);
dsmClient->SetDsmIsServer(0);
dsmClient->ReadDSMConfigFile();
dsmClient->CreateDSM();
// Connect to Server
dsmClient->ConnectDSM();
H5FDdsmBuffer * dsmBuffer = dsmClient->GetDSMHandle();
H5FD_dsm_set_mode(H5FD_DSM_MANUAL_SERVER_UPDATE, dsmBuffer);
// Get info from remote server
double remoteMB = dsmBuffer->GetTotalLength() / (1024.0 * 1024.0);
double numServers = dsmBuffer->GetEndServerId() + 1;
if (rank == 0)
{
std::cout << "DSM server memory size is : " << (int)remoteMB << " MB" << std::endl;
std::cout << "DSM server process count : " << (int)numServers << std::endl;
}
// Create Array
boost::shared_ptr<XdmfArray> array = XdmfArray::New();
array->initialize<int>(0);
array->pushBack(0);
array->pushBack(1);
array->pushBack(2);
// Create DSM Writer and write to DSM space.
boost::shared_ptr<XdmfHDF5WriterDSM> writer = XdmfHDF5WriterDSM::New("dsm", dsmBuffer);
array->accept(writer);
// Read data
boost::shared_ptr<XdmfArray> readArray = XdmfArray::New();
readArray->setHeavyDataController(array->getHeavyDataController());
readArray->read();
assert(readArray->getSize() == 3);
assert(readArray->getArrayType() == XdmfArrayType::Int32());
for(unsigned int i=0; i<readArray->getSize(); ++i)
{
assert(array->getValue<int>(i) == readArray->getValue<int>(i));
}
// Return to Server
H5FD_dsm_server_update(dsmBuffer);
// Closes ports or MPI communicators
dsmServer->UnpublishDSM();
delete dsmClient;
delete dsmServer;
MPI_Finalize();
return 0;
}
......@@ -42,17 +42,15 @@ if(XDMF_BUILD_TESTING)
add_subdirectory(tests)
endif(XDMF_BUILD_TESTING)
if(XDMF_WRAP_PYTHON)
set_source_files_properties(XdmfUtils.i PROPERTIES CPLUSPLUS ON SWIG_FLAGS "${XdmfSwigFlags}")
swig_add_module(XdmfUtils python XdmfUtils.i)
swig_link_libraries(XdmfUtils XdmfUtils ${PYTHON_LIBRARIES})
install(FILES ${CMAKE_BINARY_DIR}/XdmfUtils.py DESTINATION lib/python)
install(TARGETS ${SWIG_MODULE_XdmfUtils_REAL_NAME} DESTINATION lib/python)
endif(XDMF_WRAP_PYTHON)
if(XDMF_WRAP_JAVA)
XDMF_SWIG_JAVA(XdmfUtils XdmfJava)
set(XDMF_UTILS_JAVA_JAR ${CMAKE_BINARY_DIR}/XdmfUtils.jar CACHE INTERNAL "")
endif(XDMF_WRAP_JAVA)
foreach(source ${XdmfUtilsSources})
set(XdmfUtilsHeaders ${XdmfUtilsHeaders} ${source}.hpp)
endforeach(source ${XdmfUtilsSources})
install(FILES ${XdmfUtilsHeaders} DESTINATION include)
install(TARGETS XdmfUtils LIBRARY DESTINATION lib)
install(TARGETS ${XdmfUtilsExecutables} RUNTIME DESTINATION bin)
......@@ -12,6 +12,10 @@ swig -v -c++ -python -o XdmfUtilsPython.cpp XdmfUtils.i
#include <XdmfCoreReader.hpp>
#include <XdmfHDF5Controller.hpp>
#include <XdmfHDF5Writer.hpp>
#ifdef XDMF_BUILD_DSM
#include <XdmfHDF5ControllerDSM.hpp>
#include <XdmfHDF5WriterDSM.hpp>
#endif
#include <XdmfHeavyDataController.hpp>
#include <XdmfHeavyDataWriter.hpp>
#include <XdmfInformation.hpp>
......@@ -52,10 +56,14 @@ swig -v -c++ -python -o XdmfUtilsPython.cpp XdmfUtils.i
%import Xdmf.i
#ifdef SWIGPYTHON
%pythoncode {
from Xdmf import *
}
#endif /* SWIGPYTHON */
// Shared Pointer Templates
#ifdef XDMF_BUILD_EXODUS_IO
%shared_ptr(XdmfExodusReader)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment