Commit 03e031e0 authored by Andrew J. Burns (Cont's avatar Andrew J. Burns (Cont

hdf5 write efficiency changes to XdmfFortran and added a test for functions

parent 6699fb88
......@@ -34,6 +34,7 @@
#include "XdmfHDF5Controller.hpp"
#include "XdmfHDF5Writer.hpp"
namespace {
const static unsigned int DEFAULT_CHUNK_SIZE = 0;
......@@ -248,20 +249,21 @@ XdmfHDF5Writer::visit(XdmfArray & array,
if (checkWritten == mImpl->mWrittenItems.end() || array.getItemTag() == "DataItem") {
//if it has children send the writer to them too.
try {
array.traverse(visitor);
array.traverse(visitor);
}
catch (XdmfError e) {
throw e;
}
//only do this if the object has not already been written
try {
this->write(array, H5P_DEFAULT);
}
catch (XdmfError e) {
throw e;
if (array.isInitialized()) {
//only do this if the object has not already been written
try {
this->write(array, H5P_DEFAULT);
}
catch (XdmfError e) {
throw e;
}
mImpl->mWrittenItems.insert(&array);
}
mImpl->mWrittenItems.insert(&array);
}
//if the object has already been written, just end, it already has the data
mImpl->mDepth--;
......@@ -1681,7 +1683,6 @@ XdmfHDF5Writer::write(XdmfArray & array,
}
}
status = H5Dwrite(dataset,
datatype,
memspace,
......@@ -1689,7 +1690,6 @@ XdmfHDF5Writer::write(XdmfArray & array,
H5P_DEFAULT,
curArray->getValuesInternal());
if(status < 0) {
try {
XdmfError::message(XdmfError::FATAL,
......@@ -1704,10 +1704,14 @@ XdmfHDF5Writer::write(XdmfArray & array,
if(dataspace != H5S_ALL) {
status = H5Sclose(dataspace);
}
if(memspace != H5S_ALL) {
status = H5Sclose(memspace);
}
status = H5Dclose(dataset);
//this is causing a lot of overhead
if(closeFile) {
mImpl->closeFile();
}
......
......@@ -30,6 +30,8 @@ ADD_TEST_CXX(TestXdmfVisitorValueCounter)
ADD_TEST_CXX(TestXdmfWriter)
ADD_TEST_CXX(TestXdmfWriterHDF5ThenXML)
ADD_TEST_CXX(TestXdmfXPath)
#removed due to long execution time
#ADD_TEST_CXX(HugeWriteArray)
# Add any cxx cleanup here:
# Note: We don't want to use a foreach loop to test the files incase we
......@@ -85,3 +87,10 @@ CLEAN_TEST_CXX(TestXdmfWriterHDF5ThenXML)
CLEAN_TEST_CXX(TestXdmfXPath
XdmfXPath1.xmf
XdmfXPath2.xmf)
#removed due to long execution time
#CLEAN_TEST_CXX(HugeWriteArray
# arraydata1.h5
# arraydata2.h5
# arraydata3.h5
# arraydata.h5
# arraydata.xmf)
......@@ -170,7 +170,7 @@ double parse(std::string expression, std::map<std::string, double> variables)
//string is parsed left to right
//elements of the same priority are evaluated right to left
for (int i = 0; i < expression.size(); i++)
for (unsigned int i = 0; i < expression.size(); i++)
{
if (validDigitChars.find(expression[i]) != std::string::npos)//found to be a digit
{
......@@ -211,7 +211,7 @@ double parse(std::string expression, std::map<std::string, double> variables)
//if it is grab the string between paranthesis
i = i + 2;
valueStart = i;
int numOpenParenthesis = 0;
unsigned int numOpenParenthesis = 0;
while ((expression[i] != ')' || numOpenParenthesis) && i < expression.size())
{
if (expression[i] == '(')
......@@ -407,7 +407,7 @@ double function(std::vector<double> valueVector, std::string functionName)
double sum(std::vector<double> values)
{
double total = 0.0;
for (int i = 0; i < values.size(); i++)
for (unsigned int i = 0; i < values.size(); i++)
{
total += values[i];
}
......@@ -439,7 +439,7 @@ shared_ptr<XdmfArray> parse(std::string expression, std::map<std::string, shared
//string is parsed left to right
//elements of the same priority are evaluated right to left
for (int i = 0; i < expression.size(); i++)
for (unsigned int i = 0; i < expression.size(); i++)
{
if (validDigitChars.find(expression[i]) != std::string::npos)//found to be a digit
{
......@@ -483,7 +483,7 @@ shared_ptr<XdmfArray> parse(std::string expression, std::map<std::string, shared
//if it is grab the string between paranthesis
i = i + 2;
valueStart = i;
int numOpenParenthesis = 0;
unsigned int numOpenParenthesis = 0;
while ((expression[i] != ')' || numOpenParenthesis) && i < expression.size())
{
if (expression[i] == '(')
......@@ -975,8 +975,8 @@ shared_ptr<XdmfArray> calculation(shared_ptr<XdmfArray> val1, shared_ptr<XdmfArr
returnArray->resize(val1->getSize()+val2->getSize(), sampleValue);
}
//determine ratio of array sizes
int arrayRatio1 = floor(static_cast<double>(val1->getSize())/val2->getSize());
int arrayRatio2 = floor(static_cast<double>(val2->getSize())/val1->getSize());
int arrayRatio1 = (int)floor(static_cast<double>(val1->getSize())/val2->getSize());
int arrayRatio2 = (int)floor(static_cast<double>(val2->getSize())/val1->getSize());
if (arrayRatio1 < 1)
{
arrayRatio1 = 1;
......@@ -996,28 +996,28 @@ shared_ptr<XdmfArray> calculation(shared_ptr<XdmfArray> val1, shared_ptr<XdmfArr
if (i<arrayRatio1)
{
int amountWritten = val1->getSize()/arrayRatio1;
if (((amountWritten * arrayRatio1) + i) < val1->getSize())
if ((unsigned int)((amountWritten * arrayRatio1) + i) < val1->getSize())
{
amountWritten++;
}
if (amountWritten > floor(val2->getSize()/arrayRatio2))
{
arrayExcess1 += amountWritten - floor(val2->getSize()/arrayRatio2);
amountWritten = floor(val2->getSize()/arrayRatio2);
arrayExcess1 += amountWritten - (int)floor(val2->getSize()/arrayRatio2);
amountWritten = (int)floor(val2->getSize()/arrayRatio2);
}
returnArray->insert(i, val1, i, amountWritten, stride, arrayRatio1);
}
else //second array takes the rest
{
int amountWritten = val2->getSize()/arrayRatio2;
if (((amountWritten * arrayRatio2) + i) < val2->getSize())
if ((unsigned int)((amountWritten * arrayRatio2) + i) < val2->getSize())
{
amountWritten++;
}
if (amountWritten > floor(val1->getSize()/arrayRatio1))
{
arrayExcess2 += amountWritten - floor(val1->getSize()/arrayRatio1);
amountWritten = floor(val1->getSize()/arrayRatio1);
arrayExcess2 += amountWritten - (int)floor(val1->getSize()/arrayRatio1);
amountWritten = (int)floor(val1->getSize()/arrayRatio1);
}
returnArray->insert(i, val2, i-arrayRatio1, amountWritten, stride, arrayRatio2);
}
......@@ -1064,9 +1064,9 @@ shared_ptr<XdmfArray> function(std::vector<shared_ptr<XdmfArray> > valueVector,
shared_ptr<XdmfArray> sum(std::vector<shared_ptr<XdmfArray> > values)
{
double total = 0.0;
for (int i = 0; i < values.size(); i++)
for (unsigned int i = 0; i < values.size(); i++)
{
for (int j = 0; j < values[i]->getSize(); j++)
for (unsigned int j = 0; j < values[i]->getSize(); j++)
{
total += values[i]->getValue<double>(j);
}
......@@ -1080,7 +1080,7 @@ shared_ptr<XdmfArray> ave(std::vector<shared_ptr<XdmfArray> > values)
{
double total = sum(values)->getValue<double>(0);;
int totalSize = 0;
for (int i = 0; i < values.size(); i++)
for (unsigned int i = 0; i < values.size(); i++)
{
totalSize += values[i]->getSize();
}
......@@ -1100,9 +1100,9 @@ shared_ptr<XdmfArray> maximum(std::vector<shared_ptr<XdmfArray> > values)
else
{
double maxVal = values[0]->getValue<double>(0);
for (int i = 0; i < values.size(); i++)
for (unsigned int i = 0; i < values.size(); i++)
{
for (int j = 0; j < values[i]->getSize(); j++)
for (unsigned int j = 0; j < values[i]->getSize(); j++)
{
if (maxVal < values[i]->getValue<double>(j))
{
......
from Xdmf import *
import timeit
import time
if __name__ == "__main__":
#for later use in determining actual time
# print timeit.Timer(timedWrite.write).timeit(1)
exampleHeavyWriter = XdmfHDF5Writer.New("timestamptest.h5")
#possible options
exampleHeavyWriter.setReleaseData(True)
exampleWriter = XdmfWriter.New("timestamptest.xmf", exampleHeavyWriter)
exampleWriter.setLightDataLimit(10)
primaryDomain = XdmfDomain.New()
numNodes = 30000
numAttrib = 10
gridsPerTimestamp = 10
numTimestamps = 100
#for 100 time steps
#each time step is a grid collection
#base structue is a spacial grid collection full of temporal grid collections
primaryCollection = XdmfGridCollection.New()
primaryDomain.insert(primaryCollection)
startclock = time.clock()
for i in range(numTimestamps):
#print "timestamp " + str(i)
timestampGrid = XdmfGridCollection.New()
timestampTime = XdmfTime.New(i)
timestampGrid.setTime(timestampTime)
#each time stamp has 10 grids
for j in range(gridsPerTimestamp):
sectionGrid = XdmfUnstructuredGrid.New()
sectionTime = XdmfTime.New(i)
sectionGrid.setTime(sectionTime)
#each grid has one topo, one geo, and 10 attrib
sectionGeometry = XdmfGeometry.New()
sectionGeometry.setType(XdmfGeometryType.XYZ())
sectionTopology = XdmfTopology.New()
sectionTopology.setType(XdmfTopologyType.Triangle())
#Fill each with 30,000 values
#for k in range(numNodes):
# sectionGeometry.pushBackAsFloat32(k)
# sectionTopology.pushBackAsFloat32(k)
sectionGeometry.resizeAsFloat32(30000)
sectionTopology.resizeAsFloat32(30000)
sectionGrid.setGeometry(sectionGeometry)
sectionGrid.setTopology(sectionTopology)
for k in range(numAttrib):
sectionAttribute = XdmfAttribute.New()
#for l in range(numNodes):
# sectionAttribute.pushBackAsFloat32(l)
sectionAttribute.resizeAsFloat32(30000)
sectionGrid.insert(sectionAttribute)
timestampGrid.insert(sectionGrid)
#exampleHeavyWriter.openFile()
#sectionGrid.accept(exampleHeavyWriter)
#exampleHeavyWriter.closeFile()
primaryCollection.insert(timestampGrid)
exampleHeavyWriter.openFile()
timestampGrid.accept(exampleHeavyWriter)
#primaryDomain.accept(exampleHeavyWriter)
#primaryDomain.accept(exampleWriter)
exampleHeavyWriter.closeFile()
exampleHeavyWriter.openFile()
primaryDomain.accept(exampleWriter)
exampleHeavyWriter.closeFile()
print (time.clock() - startclock)
......@@ -415,7 +415,9 @@ XdmfFortran::addGrid(const char * const name, const bool writeToHDF5)//unstructu
mDomain,
mGridCollections);
if (writeToHDF5 && mHeavyDataWriter != NULL) {
mHeavyDataWriter->openFile();
grid->accept(mHeavyDataWriter);
mHeavyDataWriter->closeFile();
}
}
......@@ -456,7 +458,9 @@ XdmfFortran::addGridCurvilinear(const char * const name, const bool writeToHDF5)
mDomain,
mGridCollections);
if (writeToHDF5 && mHeavyDataWriter != NULL) {
mHeavyDataWriter->openFile();
grid->accept(mHeavyDataWriter);
mHeavyDataWriter->closeFile();
}
}
......@@ -486,7 +490,9 @@ XdmfFortran::addGridRectilinear(const char * const name, const bool writeToHDF5)
mDomain,
mGridCollections);
if (writeToHDF5 && mHeavyDataWriter != NULL) {
mHeavyDataWriter->openFile();
grid->accept(mHeavyDataWriter);
mHeavyDataWriter->closeFile();
}
}
......@@ -535,7 +541,9 @@ XdmfFortran::addGridRegular(const char * const name, const bool writeToHDF5)
mDomain,
mGridCollections);
if (writeToHDF5 && mHeavyDataWriter != NULL) {
mHeavyDataWriter->openFile();
grid->accept(mHeavyDataWriter);
mHeavyDataWriter->closeFile();
}
}
......@@ -625,7 +633,9 @@ XdmfFortran::closeGridCollection(const bool writeToHDF5)
{
if(!mGridCollections.empty()) {
if (writeToHDF5 && mHeavyDataWriter != NULL) {
mHeavyDataWriter->openFile();
mGridCollections.top()->accept(mHeavyDataWriter);
mHeavyDataWriter->closeFile();
}
mGridCollections.pop();
}
......@@ -7919,7 +7929,9 @@ XdmfFortran::write(const char * const xmlFilePath, const int datalimit, const bo
writer->getHeavyDataWriter()->setReleaseData(release);
shared_dynamic_cast<XdmfHDF5Writer>(writer->getHeavyDataWriter())->setFileSizeLimit(mMaxFileSize);
shared_dynamic_cast<XdmfHDF5Writer>(writer->getHeavyDataWriter())->setAllowSetSplitting(mAllowSetSplitting);
shared_dynamic_cast<XdmfHDF5Writer>(writer->getHeavyDataWriter())->openFile();
mDomain->accept(writer);
shared_dynamic_cast<XdmfHDF5Writer>(writer->getHeavyDataWriter())->closeFile();
}
void
......@@ -7935,7 +7947,9 @@ XdmfFortran::writeHDF5(const char * const xmlFilePath, const bool release)
if (mHeavyDataWriter->getReleaseData() != release) {
mHeavyDataWriter->setReleaseData(release);
}
mHeavyDataWriter->openFile();
mDomain->accept(mHeavyDataWriter);
mHeavyDataWriter->closeFile();
}
void
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment