An update will be applied December 9th, between 12PM and 1:00PM EST (UTC -5:00). The site may be slow during that time.

Commit 08bfa4eb authored by Kenneth Leiter's avatar Kenneth Leiter
Browse files

ENH: Complete hyperslab reading.

Still an issue with invalid read of size 1 in hdf5 in valgrind with the new test.
This has to do with overwriting a dataset with a smaller sized dataset when calling
H5Dset_extent(). It is deep within hdf5 when dealing with chunked data. No errors
from hdf5 returned however.
parent b56afa1a
......@@ -464,7 +464,7 @@ std::string
XdmfArray::getDimensionsString() const
{
const std::vector<unsigned int> & dimensions = this->getDimensions();
return GetValuesString().getValuesString(&dimensions[0],
return GetValuesString().getValuesString(&dimensions[0],
dimensions.size());
}
......@@ -491,9 +491,7 @@ XdmfArray::getItemProperties() const
else {
arrayProperties["Format"] = "XML";
}
std::stringstream size;
size << this->getSize();
arrayProperties["Dimensions"] = size.str();
arrayProperties["Dimensions"] = this->getDimensionsString();
if(mName.compare("") != 0) {
arrayProperties["Name"] = mName;
}
......@@ -605,8 +603,8 @@ XdmfArray::initialize(const boost::shared_ptr<const XdmfArrayType> arrayType,
const std::vector<unsigned int> & dimensions)
{
mDimensions = dimensions;
const unsigned int size = std::accumulate(mDimensions.begin(),
mDimensions.end(),
const unsigned int size = std::accumulate(dimensions.begin(),
dimensions.end(),
1,
std::multiplies<unsigned int>());
return this->initialize(arrayType, size);
......@@ -711,12 +709,14 @@ XdmfArray::populateItem(const std::map<std::string, std::string> & itemPropertie
newHDF5Path << xmlDir->second << hdf5Path;
hdf5Path = newHDF5Path.str();
}
mHeavyDataController =
mHeavyDataController =
XdmfHDF5Controller::New(hdf5Path,
dataSetPath,
arrayType,
std::vector<unsigned int>(1, 0),
std::vector<unsigned int>(1, 1),
std::vector<unsigned int>(mDimensions.size(),
0),
std::vector<unsigned int>(mDimensions.size(),
1),
mDimensions);
}
else {
......
......@@ -58,7 +58,7 @@ XdmfHDF5Controller::XdmfHDF5Controller(const std::string & hdf5FilePath,
mStart(start),
mStride(stride)
{
assert(mStart.size() == mStride.size() &&
assert(mStart.size() == mStride.size() &&
mStride.size() == mDimensions.size());
}
......@@ -99,11 +99,16 @@ XdmfHDF5Controller::read(XdmfArray * const array, const int fapl)
NULL);
hssize_t numVals = H5Sget_select_npoints(dataspace);
// hid_t memspace = H5Screate_simple(1, dims, NULL);
// hsize_t memStart[] = {0};
// hsize_t memStride[] = {1};
// hsize_t memCount[] = {10};
//status = H5Sselect_hyperslab(memspace, H5S_SELECT_SET, memStart, memStride, memCount, NULL);
hid_t memspace = H5Screate_simple(mDimensions.size(),
&count[0],
NULL);
/* status = H5Sselect_hyperslab(memspace,
H5S_SELECT_SET,
&memStart[0],
&memStride[0],
&memCount[0],
NULL);*/
hid_t datatype = H5Dget_type(dataset);
array->initialize(mType, mDimensions);
......@@ -112,14 +117,14 @@ XdmfHDF5Controller::read(XdmfArray * const array, const int fapl)
status = H5Dread(dataset,
datatype,
H5S_ALL,
memspace,
dataspace,
H5P_DEFAULT,
array->getValuesInternal());
status = H5Tclose(datatype);
status = H5Sclose(dataspace);
// status = H5Sclose(memspace);
status = H5Sclose(memspace);
status = H5Dclose(dataset);
status = H5Fclose(hdf5Handle);
}
......@@ -149,15 +149,16 @@ XdmfHDF5Writer::write(XdmfArray & array,
hid_t dataspace = H5S_ALL;
hid_t memspace = H5S_ALL;
std::vector<hsize_t> current_dims(dimensions.begin(), dimensions.end());
if(dataset < 0) {
std::vector<hsize_t> current_dims(dimensions.begin(), dimensions.end());
std::vector<hsize_t> maximum_dims(dimensions.size(), H5S_UNLIMITED);
memspace = H5Screate_simple(dimensions.size(),
&current_dims[0],
memspace = H5Screate_simple(dimensions.size(),
&current_dims[0],
&maximum_dims[0]);
hid_t property = H5Pcreate(H5P_DATASET_CREATE);
hsize_t chunkSize = 1024;
status = H5Pset_chunk(property, 1, &chunkSize);
std::vector<hsize_t> chunk_size(dimensions.size(), 1024);
status = H5Pset_chunk(property, dimensions.size(), &chunk_size[0]);
dataset = H5Dcreate(hdf5Handle,
dataSetPath.str().c_str(),
datatype,
......@@ -191,7 +192,17 @@ XdmfHDF5Writer::write(XdmfArray & array,
NULL) ;
}
else {
status = H5Dset_extent(dataset, &size);
// Overwriting - dataset rank must remain the same (hdf5 constraint)
hid_t dataspace = H5Dget_space(dataset);
const int ndims = H5Sget_simple_extent_ndims(dataspace);
assert(ndims == current_dims.size());
status = H5Dset_extent(dataset, &current_dims[0]);
if(status < 0) {
assert(false);
}
}
}
status = H5Dwrite(dataset,
......@@ -200,6 +211,11 @@ XdmfHDF5Writer::write(XdmfArray & array,
dataspace,
H5P_DEFAULT,
array.getValuesInternal());
if(status < 0) {
assert(false);
}
if(dataspace != H5S_ALL) {
status = H5Sclose(dataspace);
}
......@@ -213,7 +229,7 @@ XdmfHDF5Writer::write(XdmfArray & array,
H5Eset_auto2(0, old_func, old_client_data);
// Attach a new controller to the array
boost::shared_ptr<XdmfHDF5Controller> newDataController =
boost::shared_ptr<XdmfHDF5Controller> newDataController =
boost::shared_ptr<XdmfHDF5Controller>();
unsigned int newSize = array.getSize();
......@@ -227,7 +243,7 @@ XdmfHDF5Writer::write(XdmfArray & array,
std::vector<unsigned int>(1, 1),
std::vector<unsigned int>(1, newSize));
}
if(mMode == Default || !array.getHeavyDataController()) {
++mDataSetId;
}
......@@ -237,8 +253,10 @@ XdmfHDF5Writer::write(XdmfArray & array,
this->createHDF5Controller(hdf5FilePath,
dataSetPath.str(),
array.getArrayType(),
std::vector<unsigned int>(1, 0),
std::vector<unsigned int>(1, 1),
std::vector<unsigned int>(dimensions.size(),
0),
std::vector<unsigned int>(dimensions.size(),
1),
dimensions);
}
array.setHeavyDataController(newDataController);
......
......@@ -14,6 +14,7 @@ ADD_TEST_CXX(TestXdmfAttribute)
ADD_TEST_CXX(TestXdmfCurvilinearGrid)
ADD_TEST_CXX(TestXdmfGeometry)
ADD_TEST_CXX(TestXdmfGridCollection)
ADD_TEST_CXX(TestXdmfHDF5Hyperslab)
ADD_TEST_CXX(TestXdmfReader)
ADD_TEST_CXX(TestXdmfRegularGrid)
ADD_TEST_CXX(TestXdmfRectilinearGrid)
......@@ -42,6 +43,10 @@ CLEAN_TEST_CXX(TestXdmfGridCollection
TestXdmfGridCollectionHDF1.h5
TestXdmfGridCollectionHDF1.xmf
TestXdmfGridCollectionHDF2.xmf)
CLEAN_TEST_CXX(TestXdmfHDF5Hyperslab
TestXdmfHDF5Hyperslab.xmf
TestXdmfHDF5Hyperslab.h5
TestXdmfHDF5Hyperslab2.xmf)
CLEAN_TEST_CXX(TestXdmfReader
TestXdmfReader1.h5
TestXdmfReader1.xmf
......@@ -63,4 +68,4 @@ CLEAN_TEST_CXX(TestXdmfWriter
CLEAN_TEST_CXX(TestXdmfWriterHDF5ThenXML)
CLEAN_TEST_CXX(TestXdmfXPath
XdmfXPath1.xmf
XdmfXPath2.xmf)
XdmfXPath2.xmf)
\ No newline at end of file
#include "XdmfArray.hpp"
#include "XdmfHDF5Controller.hpp"
#include "XdmfHeavyDataWriter.hpp"
#include "XdmfReader.hpp"
#include "XdmfWriter.hpp"
int main(int, char *)
{
//
// Create 2x2 array.
//
boost::shared_ptr<XdmfArray> array = XdmfArray::New();
std::vector<unsigned int> dimensions(2, 2);
array->initialize<int>(dimensions);
int values[] = {0, 1, 2, 3};
array->insert(0, &values[0], 4);
assert(array->getSize() == 4);
dimensions = array->getDimensions();
assert(dimensions.size() == 2);
assert(dimensions[0] == 2 && dimensions[1] == 2);
//
// Write array to disk.
//
boost::shared_ptr<XdmfWriter> writer =
XdmfWriter::New("TestXdmfHDF5Hyperslab.xmf");
writer->setLightDataLimit(0);
array->accept(writer);
//
// Read array from disk.
//
boost::shared_ptr<XdmfReader> reader =
XdmfReader::New();
boost::shared_ptr<XdmfItem> item = reader->read("TestXdmfHDF5Hyperslab.xmf");
boost::shared_ptr<XdmfArray> readArray =
boost::shared_dynamic_cast<XdmfArray>(item);
assert(readArray);
assert(readArray->getSize() == 4);
std::vector<unsigned int> readDimensions = readArray->getDimensions();
assert(readDimensions.size() == 2);
assert(readDimensions[0] == 2 && readDimensions[1] == 2);
//
// Read heavy values from disk.
//
readArray->read();
assert(readArray->isInitialized());
assert(readArray->getSize() == 4);
readDimensions = readArray->getDimensions();
assert(readDimensions.size() == 2);
assert(readDimensions[0] == 2 && readDimensions[1] == 2);
for(int i=0; i<4; ++i) {
assert(readArray->getValue<int>(i) == i);
}
//
// Modify read array to do an overwrite of a different sized dataset.
//
dimensions = std::vector<unsigned int>(2,3);
readArray->resize(dimensions, 1000);
for(int i=0; i<9; ++i) {
readArray->insert(i, i);
}
assert(readArray->getSize() == 9);
dimensions = readArray->getDimensions();
assert(dimensions.size() == 2);
assert(dimensions[0] == 3 && dimensions[1] == 3);
boost::shared_ptr<XdmfWriter> writer2 =
XdmfWriter::New("TestXdmfHDF5Hyperslab2.xmf");
writer2->setLightDataLimit(0);
writer2->getHeavyDataWriter()->setMode(XdmfHeavyDataWriter::Overwrite);
readArray->accept(writer2);
//
// Read array from disk.
//
boost::shared_ptr<XdmfItem> item2 =
reader->read("TestXdmfHDF5Hyperslab2.xmf");
boost::shared_ptr<XdmfArray> readArray2 =
boost::shared_dynamic_cast<XdmfArray>(item2);
assert(readArray2);
assert(readArray2->getSize() == 9);
readDimensions = readArray2->getDimensions();
assert(readDimensions.size() == 2);
assert(readDimensions[0] == 3 && readDimensions[1] == 3);
//
// Read heavy values from disk.
//
readArray2->read();
assert(readArray2->isInitialized());
assert(readArray2->getSize() == 9);
readDimensions = readArray2->getDimensions();
assert(readDimensions.size() == 2);
assert(readDimensions[0] == 3 && readDimensions[1] == 3);
for(int i=0; i<9; ++i) {
assert(readArray2->getValue<int>(i) == i);
}
//
// Read hyperslab of data
//
// 0 1 2
// 3 4 5
// 6 7 8
//
// becomes...
//
// 4 5
// 7 8
//
const std::vector<unsigned int> start(2, 1);
const std::vector<unsigned int> stride(2,1);
const std::vector<unsigned int> count(2, 2);
boost::shared_ptr<XdmfHeavyDataController> controller =
readArray2->getHeavyDataController();
assert(controller);
boost::shared_ptr<XdmfHDF5Controller> hyperslabController =
XdmfHDF5Controller::New(controller->getFilePath(),
controller->getDataSetPath(),
controller->getType(),
start,
stride,
count);
boost::shared_ptr<XdmfArray> hyperslabArray = XdmfArray::New();
hyperslabArray->setHeavyDataController(hyperslabController);
assert(hyperslabArray->getSize() == 4);
std::vector<unsigned int> hyperslabDimensions =
hyperslabArray->getDimensions();
assert(hyperslabDimensions.size() == 2);
assert(hyperslabDimensions[0] == 2 && hyperslabDimensions[1] == 2);
hyperslabArray->read();
hyperslabDimensions = hyperslabArray->getDimensions();
assert(hyperslabDimensions.size() == 2);
assert(hyperslabDimensions[0] == 2 && hyperslabDimensions[1] == 2);
assert(hyperslabArray->getValue<int>(0) == 4);
assert(hyperslabArray->getValue<int>(1) == 5);
assert(hyperslabArray->getValue<int>(2) == 7);
assert(hyperslabArray->getValue<int>(3) == 8);
return 0;
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment