Skip to content

GitLab

X
Xdmf
Commit 39a4f8ee authored by Kenneth Leiter's avatar Kenneth Leiter
Browse files
Options

ENH: Initial commit of binary heavy data file support.

parent 366b7b37
Hide whitespace changes
Inline Side-by-side
Showing with 600 additions and 221 deletions
core/CMakeLists.txt
View file @ 39a4f8ee
......@@ -6,6 +6,7 @@ endif (POLICY CMP0015)
include(CheckCXXSourceCompiles)
include(SetUpVersion)
include(TestBigEndian)
if(VERSION_CONTROL_AUTOUPDATE OR
NOT EXISTS ${CMAKE_BINARY_DIR}/XdmfVersion.hpp)
......@@ -76,6 +77,9 @@ if(LIBXML2_FOUND)
endif(LIBXML2_FOUND)
# Perform compile-time checks and generate XdmfConfig.hpp
TEST_BIG_ENDIAN(XDMF_BIG_ENDIAN)
unset(HAVE_BOOST_SHARED_DYNAMIC_CAST CACHE)
set(CMAKE_REQUIRED_INCLUDES ${CMAKE_REQUIRED_INCLUDES} ${Boost_INCLUDE_DIRS})
check_cxx_source_compiles("
......@@ -98,6 +102,7 @@ set(XdmfCoreSources
XdmfArray
XdmfArrayReference
XdmfArrayType
XdmfBinaryController
XdmfCoreItemFactory
XdmfCoreReader
XdmfError
......
core/XdmfArray.cpp
View file @ 39a4f8ee
......@@ -31,11 +31,36 @@
#include "XdmfArray.hpp"
#include "XdmfArrayType.hpp"
#include "XdmfArrayReference.hpp"
#include "XdmfBinaryController.hpp"
#include "XdmfHDF5Controller.hpp"
#include "XdmfHeavyDataController.hpp"
#include "XdmfVisitor.hpp"
#include "XdmfError.hpp"
namespace {
std::string
getFullHeavyDataPath(const std::string & filePath,
const std::map<std::string, std::string> & itemProperties)
{
// FIXME: for other OS (e.g. windows)
if(filePath.size() > 0 && filePath[0] != '/') {
// Dealing with a relative path for heavyData location
std::map<std::string, std::string>::const_iterator xmlDir =
itemProperties.find("XMLDir");
if(xmlDir == itemProperties.end()) {
XdmfError::message(XdmfError::FATAL,
"'XMLDir' not found in itemProperties in "
"XdmfArray::populateItem");
}
std::stringstream newHeavyDataPath;
newHeavyDataPath << xmlDir->second << filePath;
return newHeavyDataPath.str();
}
return filePath;
}
}
XDMF_CHILDREN_IMPLEMENTATION(XdmfArray,
XdmfHeavyDataController,
......@@ -524,14 +549,11 @@ public:
unsigned int
operator()(const boost::blank & array) const
{
if(mArray->mHeavyDataControllers.size()>0) {
int total = 0;
for (unsigned int i = 0; i < mArray->mHeavyDataControllers.size(); ++i) {
total += mArray->mHeavyDataControllers[i]->getSize();
}
return total;
unsigned int total = 0;
for (unsigned int i = 0; i < mArray->mHeavyDataControllers.size(); ++i) {
total += mArray->mHeavyDataControllers[i]->getSize();
}
return 0;
return total;
}
template<typename T>
......@@ -659,9 +681,8 @@ std::map<std::string, std::string>
XdmfArray::getItemProperties() const
{
std::map<std::string, std::string> arrayProperties;
if(mHeavyDataControllers.size()>0) {
arrayProperties.insert(std::make_pair("Format",
mHeavyDataControllers[0]->getName()));
if(mHeavyDataControllers.size() > 0) {
mHeavyDataControllers[0]->getProperties(arrayProperties);
}
else {
arrayProperties.insert(std::make_pair("Format", "XML"));
......@@ -738,7 +759,7 @@ XdmfArray::getHeavyDataController()
shared_ptr<const XdmfHeavyDataController>
XdmfArray::getHeavyDataController() const
{
if (mHeavyDataControllers.size()>0) {
if (mHeavyDataControllers.size() > 0) {
return mHeavyDataControllers[0];
}
else {
......@@ -747,7 +768,7 @@ XdmfArray::getHeavyDataController() const
}
void
XdmfArray::initialize(const shared_ptr<const XdmfArrayType> arrayType,
XdmfArray::initialize(const shared_ptr<const XdmfArrayType> & arrayType,
const unsigned int size)
{
if(arrayType == XdmfArrayType::Int8()) {
......@@ -795,7 +816,7 @@ XdmfArray::initialize(const shared_ptr<const XdmfArrayType> arrayType,
}
void
XdmfArray::initialize(const shared_ptr<const XdmfArrayType> arrayType,
XdmfArray::initialize(const shared_ptr<const XdmfArrayType> & arrayType,
const std::vector<unsigned int> & dimensions)
{
mDimensions = dimensions;
......@@ -1065,7 +1086,6 @@ XdmfArray::populateItem(const std::map<std::string, std::string> & itemPropertie
}
const std::string & formatVal = format->second;
if(formatVal.compare("HDF") == 0) {
contentIndex = 0;
int contentStep = 2;
......@@ -1088,20 +1108,9 @@ XdmfArray::populateItem(const std::map<std::string, std::string> & itemPropertie
std::string dataSetPath =
contentVals[contentIndex].substr(colonLocation+1);
// FIXME: for other OS (e.g. windows)
if(hdf5Path.size() > 0 && hdf5Path[0] != '/') {
// Dealing with a relative path for hdf5 location
std::map<std::string, std::string>::const_iterator xmlDir =
itemProperties.find("XMLDir");
if(xmlDir == itemProperties.end()) {
XdmfError::message(XdmfError::FATAL,
"'XMLDir' not found in itemProperties in "
"XdmfArray::populateItem");
}
std::stringstream newHDF5Path;
newHDF5Path << xmlDir->second << hdf5Path;
hdf5Path = newHDF5Path.str();
}
hdf5Path = getFullHeavyDataPath(hdf5Path,
itemProperties);
// Parse dimensions from the content
std::vector<unsigned int> contentDims;
......@@ -1126,8 +1135,7 @@ XdmfArray::populateItem(const std::map<std::string, std::string> & itemPropertie
contentStep = 1;
}
mHeavyDataControllers.push_back(
mHeavyDataControllers.push_back(
XdmfHDF5Controller::New(hdf5Path,
dataSetPath,
arrayType,
......@@ -1137,8 +1145,8 @@ XdmfArray::populateItem(const std::map<std::string, std::string> & itemPropertie
1),
contentDims,
contentDims)
);
contentIndex+=contentStep;
);
contentIndex += contentStep;
}
}
else if(formatVal.compare("XML") == 0) {
......@@ -1164,6 +1172,43 @@ XdmfArray::populateItem(const std::map<std::string, std::string> & itemPropertie
}
}
}
else if(formatVal.compare("Binary") == 0) {
XdmfBinaryController::Endian endian = XdmfBinaryController::NATIVE;
std::map<std::string, std::string>::const_iterator endianIter =
itemProperties.find("Endian");
if(endianIter != itemProperties.end()) {
if(endianIter->second.compare("Big") == 0) {
endian = XdmfBinaryController::BIG;
}
else if(endianIter->second.compare("Little") == 0) {
endian = XdmfBinaryController::LITTLE;
}
else if(endianIter->second.compare("Native") == 0) {
endian = XdmfBinaryController::NATIVE;
}
else {
XdmfError(XdmfError::FATAL,
"Invalid endianness type: " + endianIter->second);
}
}
unsigned int seek = 0;
std::map<std::string, std::string>::const_iterator seekIter =
itemProperties.find("Seek");
if(seekIter != itemProperties.end()) {
seek = std::atoi(seekIter->second.c_str());
}
const std::string binaryPath = getFullHeavyDataPath(contentVals[0],
itemProperties);
mHeavyDataControllers.push_back(XdmfBinaryController::New(binaryPath,
arrayType,
endian,
seek,
mDimensions));
}
else {
try {
XdmfError::message(XdmfError::FATAL,
......
core/XdmfArray.hpp
View file @ 39a4f8ee
......@@ -649,7 +649,7 @@ public:
* @param arrayType The type of array to initialize.
* @param size The number of values in the initialized array.
*/
void initialize(const shared_ptr<const XdmfArrayType> arrayType,
void initialize(const shared_ptr<const XdmfArrayType> & arrayType,
const unsigned int size = 0);
/**
......@@ -678,7 +678,7 @@ public:
* @param arrayType The type of array to initialize.
* @param dimensions The number dimensions of the initialized array.
*/
void initialize(const shared_ptr<const XdmfArrayType> arrayType,
void initialize(const shared_ptr<const XdmfArrayType> & arrayType,
const std::vector<unsigned int> & dimensions);
using XdmfItem::insert;
......
core/XdmfBinaryController.cpp 0 → 100644
View file @ 39a4f8ee
/*****************************************************************************/
/* Xdmf */
/* eXtensible Data Model and Format */
/* */
/* Id : XdmfBinaryController.cpp */
/* */
/* Author: */
/* Kenneth Leiter */
/* kenneth.leiter@arl.army.mil */
/* US Army Research Laboratory */
/* Aberdeen Proving Ground, MD */
/* */
/* Copyright @ 2011 US Army Research Laboratory */
/* All Rights Reserved */
/* See Copyright.txt for details */
/* */
/* This software is distributed WITHOUT ANY WARRANTY; without */
/* even the implied warranty of MERCHANTABILITY or FITNESS */
/* FOR A PARTICULAR PURPOSE. See the above copyright notice */
/* for more information. */
/* */
/*****************************************************************************/
#include <fstream>
#include <sstream>
#include "XdmfArray.hpp"
#include "XdmfArrayType.hpp"
#include "XdmfBinaryController.hpp"
#include "XdmfError.hpp"
namespace {
template<size_t T>
struct ByteSwaper {
static inline void swap(void * p){}
static inline void swap(void * p,
unsigned int length)
{
char * data = static_cast<char *>(p);
for(unsigned int i=0; i<length; ++i, data+=T){
ByteSwaper<T>::swap(data);
}
}
};
template<>
void ByteSwaper<2>::swap(void * p){
char one_byte;
char* data = static_cast<char*>(p);
one_byte = data[0]; data[0] = data[1]; data[1] = one_byte;
};
template<>
void ByteSwaper<4>::swap(void * p){
char one_byte;
char* data = static_cast<char*>(p);
one_byte = data[0]; data[0] = data[3]; data[3] = one_byte;
one_byte = data[1]; data[1] = data[2]; data[2] = one_byte;
};
template<>
void ByteSwaper<8>::swap(void * p){
char one_byte;
char* data = static_cast<char*>(p);
one_byte = data[0]; data[0] = data[7]; data[7] = one_byte;
one_byte = data[1]; data[1] = data[6]; data[6] = one_byte;
one_byte = data[2]; data[2] = data[5]; data[5] = one_byte;
one_byte = data[3]; data[3] = data[4]; data[4] = one_byte;
};
}
shared_ptr<XdmfBinaryController>
XdmfBinaryController::New(const std::string & filePath,
const shared_ptr<const XdmfArrayType> & type,
const Endian & endian,
const unsigned int seek,
const std::vector<unsigned int> & dimensions)
{
try {
shared_ptr<XdmfBinaryController>
p(new XdmfBinaryController(filePath,
type,
endian,
seek,
dimensions));
return p;
}
catch (XdmfError e) {
throw e;
}
}
XdmfBinaryController::XdmfBinaryController(const std::string & filePath,
const shared_ptr<const XdmfArrayType> & type,
const Endian & endian,
const unsigned int seek,
const std::vector<unsigned int> & dimensions) :
XdmfHeavyDataController(filePath,
"",
type,
dimensions),
mEndian(endian),
mSeek(seek)
{
}
XdmfBinaryController::~XdmfBinaryController()
{
}
XdmfBinaryController::Endian
XdmfBinaryController::getEndian() const
{
return mEndian;
}
std::string
XdmfBinaryController::getName() const
{
return "Binary";
}
void
XdmfBinaryController::getProperties(std::map<std::string, std::string> & collectedProperties) const
{
collectedProperties["Format"] = this->getName();
std::stringstream seekStream;
seekStream << mSeek;
collectedProperties["Seek"] = seekStream.str();
if(mEndian == BIG) {
collectedProperties["Endian"] = "Big";
}
else if(mEndian == LITTLE) {
collectedProperties["Endian"] = "Little";
}
}
unsigned int
XdmfBinaryController::getSeek() const
{
return mSeek;
}
void
XdmfBinaryController::read(XdmfArray * const array)
{
array->initialize(mType, mDimensions);
std::ifstream fileStream(mFilePath.c_str(),
std::ifstream::binary);
if(!fileStream.good()) {
XdmfError::message(XdmfError::FATAL,
"Error reading " + mFilePath +
" in XdmfBinaryController::read");
}
fileStream.seekg(mSeek);
if(!fileStream.good()) {
XdmfError::message(XdmfError::FATAL,
"Error seeking " + mFilePath +
" in XdmfBinaryController::read");
}
fileStream.read(static_cast<char *>(array->getValuesInternal()),
array->getSize() * mType->getElementSize());
#if defined(XDMF_BIG_ENDIAN)
const bool needByteSwap = mEndian == LITTLE;
#else
const bool needByteSwap = mEndian == BIG;
#endif // XDMF_BIG_ENDIAN
if(needByteSwap) {
switch(mType->getElementSize()){
case 1:
break;
case 2:
ByteSwaper<2>::swap(array->getValuesInternal(),
array->getSize());
break;
case 4:
ByteSwaper<4>::swap(array->getValuesInternal(),
array->getSize());
break;
case 8:
ByteSwaper<8>::swap(array->getValuesInternal(),
array->getSize());
break;
default:
XdmfError::message(XdmfError::FATAL,
"Cannot perform endianness swap for datatype");
break;
}
}
}
core/XdmfBinaryController.hpp 0 → 100644
View file @ 39a4f8ee
/*****************************************************************************/
/* XDMF */
/* eXtensible Data Model and Format */
/* */
/* Id : XdmfBinaryController.hpp */
/* */
/* Author: */
/* Kenneth Leiter */
/* kenneth.leiter@arl.army.mil */
/* US Army Research Laboratory */
/* Aberdeen Proving Ground, MD */
/* */
/* Copyright @ 2011 US Army Research Laboratory */
/* All Rights Reserved */
/* See Copyright.txt for details */
/* */
/* This software is distributed WITHOUT ANY WARRANTY; without */
/* even the implied warranty of MERCHANTABILITY or FITNESS */
/* FOR A PARTICULAR PURPOSE. See the above copyright notice */
/* for more information. */
/* */
/*****************************************************************************/
#ifndef XDMFBinaryCONTROLLER_HPP_
#define XDMFBinaryCONTROLLER_HPP_
// Includes
#include "XdmfCore.hpp"
#include "XdmfHeavyDataController.hpp"
/**
* @brief Couples an XdmfArray with Binary data stored on disk.
*
* Serves as an interface between data stored in XdmfArrays and data
* stored in binary files on disk. When an Xdmf file is read from or
* written to disk an XdmfBinaryController is attached to
* XdmfArrays. This allows data to be released from memory but still
* be accessible or have its location written to light data.
*/
class XDMFCORE_EXPORT XdmfBinaryController : public XdmfHeavyDataController {
public:
typedef enum Endian {
BIG,
LITTLE,
NATIVE
} Endian;
virtual ~XdmfBinaryController();
/**
* Create a new controller for an binary data set on disk.
*
* @param filePath the location of the binary file.
* @param type the data type of the dataset to read.
* @param endian the endianness of the data.
* @param seek in bytes to begin reading in file.
* @param dimensions the number of elements to select in each
* dimension from the hdf5 data set. (size in each dimension)
*
* @return New Binary Controller.
*/
static shared_ptr<XdmfBinaryController>
New(const std::string & filePath,
const shared_ptr<const XdmfArrayType> & type,
const Endian & endian,
const unsigned int seek,
const std::vector<unsigned int> & dimensions);
virtual Endian getEndian() const;
virtual std::string getName() const;
virtual void
getProperties(std::map<std::string, std::string> & collectedProperties) const;
virtual unsigned int getSeek() const;
virtual void read(XdmfArray * const array);
protected:
XdmfBinaryController(const std::string & filePath,
const shared_ptr<const XdmfArrayType> & type,
const Endian & endian,
const unsigned int seek,
const std::vector<unsigned int> & dimensions);
private:
XdmfBinaryController(const XdmfBinaryController &); // Not implemented.
void operator=(const XdmfBinaryController &); // Not implemented.
const Endian mEndian;
const unsigned int mSeek;
};
#endif /* XDMFBinaryCONTROLLER_HPP_ */
core/XdmfConfig.hpp.in
View file @ 39a4f8ee
......@@ -25,5 +25,6 @@
#define XDMFCONFIG_HPP_
#cmakedefine HAVE_BOOST_SHARED_DYNAMIC_CAST
#cmakedefine XDMF_BIG_ENDIAN
#endif /* XDMFSHAREDPTR_HPP_ */
core/XdmfHDF5Controller.cpp
View file @ 39a4f8ee
......@@ -36,7 +36,7 @@ std::map<std::string, unsigned int> XdmfHDF5Controller::mOpenFileUsage;
shared_ptr<XdmfHDF5Controller>
XdmfHDF5Controller::New(const std::string & hdf5FilePath,
const std::string & dataSetPath,
const shared_ptr<const XdmfArrayType> type,
const shared_ptr<const XdmfArrayType> & type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & dimensions,
......@@ -60,7 +60,7 @@ XdmfHDF5Controller::New(const std::string & hdf5FilePath,
XdmfHDF5Controller::XdmfHDF5Controller(const std::string & hdf5FilePath,
const std::string & dataSetPath,
const shared_ptr<const XdmfArrayType> type,
const shared_ptr<const XdmfArrayType> & type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & dimensions,
......@@ -68,11 +68,24 @@ XdmfHDF5Controller::XdmfHDF5Controller(const std::string & hdf5FilePath,
XdmfHeavyDataController(hdf5FilePath,
dataSetPath,
type,
start,
stride,
dimensions,
dataspaceDimensions)
dimensions),
mDataspaceDimensions(dataspaceDimensions),
mStart(start),
mStride(stride)
{
if(!(mStart.size() == mStride.size() &&
mStride.size() == mDimensions.size() &&
mDimensions.size() == mDataspaceDimensions.size())) {
try {
XdmfError::message(XdmfError::FATAL,
"mStart, mStride, mDimensions, and "
"mDataSpaceDimensions must all be of equal length in "
"XdmfHDF5Controller constructor");
}
catch (XdmfError e) {
throw e;
}
}
}
XdmfHDF5Controller::~XdmfHDF5Controller()
......@@ -91,6 +104,12 @@ XdmfHDF5Controller::closeFiles()
mOpenFileUsage.clear();
}
std::vector<unsigned int>
XdmfHDF5Controller::getDataspaceDimensions() const
{
return mDataspaceDimensions;
}
std::string
XdmfHDF5Controller::getName() const
{
......@@ -103,6 +122,24 @@ XdmfHDF5Controller::getMaxOpenedFiles()
return XdmfHDF5Controller::mMaxOpenedFiles;
}
void
XdmfHDF5Controller::getProperties(std::map<std::string, std::string> & collectedProperties) const
{
collectedProperties["Format"] = this->getName();
}
std::vector<unsigned int>
XdmfHDF5Controller::getStart() const
{
return mStart;
}
std::vector<unsigned int>
XdmfHDF5Controller::getStride() const
{
return mStride;
}
void
XdmfHDF5Controller::read(XdmfArray * const array)
{
......
core/XdmfHDF5Controller.hpp
View file @ 39a4f8ee
......@@ -78,7 +78,7 @@ public:
static shared_ptr<XdmfHDF5Controller>
New(const std::string & hdf5FilePath,
const std::string & dataSetPath,
const shared_ptr<const XdmfArrayType> type,
const shared_ptr<const XdmfArrayType> & type,
const std::vector<unsigned int> & start,
const std::vector<unsigned int> & stride,
const std::vector<unsigned int> & dimensions,
......@@ -103,6 +103,35 @@ public:
*/
static void closeFiles();
/**
* Get the dimensions of the dataspace owned by this
* controller. This is the dimension of the entire heavy dataset,
* which may be larger than the dimensions of the array (if reading
* a piece of a larger dataset).
*
* Example of use:
*