diff --git a/Documentation/release/dev/fix-vtkAMReXGridReader-windows-larger-than-2gb-files.md b/Documentation/release/dev/fix-vtkAMReXGridReader-windows-larger-than-2gb-files.md new file mode 100644 index 0000000000000000000000000000000000000000..55fb8c8578e954d52bd0c53c6d4d939955508454 --- /dev/null +++ b/Documentation/release/dev/fix-vtkAMReXGridReader-windows-larger-than-2gb-files.md @@ -0,0 +1,4 @@ +## Fix vtkAMReXGridReader when reading larger than 2GB files in Windows + +You can now open grid files larger than 2GB on Windows using the `vtkAMReXGridReader`. +Previously, a bug in the reader prevented it from loading arrays stored beyond the 2GB offset in the file. diff --git a/IO/AMR/vtkAMReXGridReaderInternal.cxx b/IO/AMR/vtkAMReXGridReaderInternal.cxx index 27d2471710f306d59090cb8b488d29e9e7c077b7..c662214ac9880718183314fd4ad069e4e78fd728 100644 --- a/IO/AMR/vtkAMReXGridReaderInternal.cxx +++ b/IO/AMR/vtkAMReXGridReaderInternal.cxx @@ -25,7 +25,7 @@ std::string ReadFile(const std::string& filename) if (stream) { stream.seekg(0, std::ios::end); - long flength = static_cast<long>(stream.tellg()); + const std::size_t flength = stream.tellg(); stream.seekg(0, std::ios::beg); std::vector<char> data(flength + 1 + (flength + 1) % 8); // padded for better alignment. stream.read(data.data(), flength); @@ -64,7 +64,7 @@ const std::vector<int>& RealDescriptor::orderarray() const& return ord; } -long RealDescriptor::numBytes() const +std::size_t RealDescriptor::numBytes() const { return (fr[0] + 7) >> 3; } @@ -964,7 +964,7 @@ void vtkAMReXGridReaderInternal::GetBlockAttribute( constexpr long ieee_float[] = { 32L, 8L, 23L, 0L, 1L, 9L, 0L, 0x7FL }; constexpr long ieee_double[] = { 64L, 11L, 52L, 0L, 1L, 12L, 0L, 0x3FFL }; - long offsetOfAttribute = this->GetOffsetOfAttribute(attribute); + vtkIdType offsetOfAttribute = this->GetOffsetOfAttribute(attribute); int theLevel = this->GetBlockLevel(blockIdx); int blockIdxWithinLevel = this->GetBlockIndexWithinLevel(blockIdx, theLevel); if (debugReader) @@ -991,7 +991,7 @@ void vtkAMReXGridReaderInternal::GetBlockAttribute( RealDescriptor* ird = this->ReadRealDescriptor(is); std::vector<int> boxArray(3 * dimension); std::vector<int> boxArrayDim(dimension); - long numberOfPoints = ReadBoxArray(is, boxArray.data(), boxArrayDim.data()); + const auto numberOfPoints = ReadBoxArray(is, boxArray.data(), boxArrayDim.data()); // int numberOfAttributes = this->ReadNumberOfAttributes(is); @@ -1000,7 +1000,7 @@ void vtkAMReXGridReaderInternal::GetBlockAttribute( // Jump to the desired attribute (offsetOfAttribute*(numberOfPoints*ird->numBytes())) // - Patrick O'Leary // - long linefeed = is.tellg(); + const std::size_t linefeed = is.tellg(); if (debugReader) { @@ -1122,7 +1122,7 @@ void vtkAMReXGridReaderInternal::GetExtraMultiFabBlockAttribute( RealDescriptor* ird = this->ReadRealDescriptor(is); std::vector<int> boxArray(3 * dimension); std::vector<int> boxArrayDim(dimension); - int numberOfPoints = ReadBoxArray(is, boxArray.data(), boxArrayDim.data()); + const auto numberOfPoints = ReadBoxArray(is, boxArray.data(), boxArrayDim.data()); // int numberOfAttributes = this->ReadNumberOfAttributes(is); @@ -1130,7 +1130,7 @@ void vtkAMReXGridReaderInternal::GetExtraMultiFabBlockAttribute( // Skip the Line Feed (linefeed+1) // Jump to the desired attribute (offsetOfAttribute*(numberOfPoints*ird->numBytes())) // - int linefeed = is.tellg(); + const std::size_t linefeed = is.tellg(); if (debugReader) { @@ -1148,7 +1148,8 @@ void vtkAMReXGridReaderInternal::GetExtraMultiFabBlockAttribute( { int compIndex = this->Header->extraMultiFabParsedVarNames[attributeName][i]; std::string compName = this->Header->extraMultiFabVariables[fabIndex][compIndex]; - int offsetOfAttribute = this->GetAttributeOffsetExtraMultiFab(compName.c_str(), fabIndex); + const vtkIdType offsetOfAttribute = + this->GetAttributeOffsetExtraMultiFab(compName.c_str(), fabIndex); is.seekg((linefeed + 1) + (offsetOfAttribute * (numberOfPoints * ird->numBytes()))); buffers[i].resize(numberOfPoints * ird->numBytes()); this->ReadBlockAttribute(is, numberOfPoints, ird->numBytes(), buffers[i].data()); @@ -1198,7 +1199,7 @@ void vtkAMReXGridReaderInternal::GetExtraMultiFabBlockAttribute( } } -int vtkAMReXGridReaderInternal::GetOffsetOfAttribute(const char* attribute) +vtkIdType vtkAMReXGridReaderInternal::GetOffsetOfAttribute(const char* attribute) { long i = 0, position = 0; bool found = false; @@ -1235,7 +1236,7 @@ int vtkAMReXGridReaderInternal::GetExtraMultiFabIndex(const char* attribute) return -1; } -int vtkAMReXGridReaderInternal::GetAttributeOffsetExtraMultiFab( +vtkIdType vtkAMReXGridReaderInternal::GetAttributeOffsetExtraMultiFab( const char* attribute, const int fabIndex) { std::string attr = attribute; @@ -1360,7 +1361,8 @@ RealDescriptor* vtkAMReXGridReaderInternal::ReadRealDescriptor(std::istream& is) return new RealDescriptor(fmt.data(), ord.data(), static_cast<int>(ord.size())); } -long vtkAMReXGridReaderInternal::ReadBoxArray(std::istream& is, int* boxArray, int* boxArrayDim) +std::size_t vtkAMReXGridReaderInternal::ReadBoxArray( + std::istream& is, int* boxArray, int* boxArrayDim) { char c; is >> c; // read '(' @@ -1385,7 +1387,7 @@ long vtkAMReXGridReaderInternal::ReadBoxArray(std::istream& is, int* boxArray, i // // block dimension - '(hi - lo + 1)' is the number of cells '+ 1' is the number of points // - long numberOfPoints = 1; + std::size_t numberOfPoints = 1; for (int i = 0; i < this->Header->dim; ++i) { boxArrayDim[i] = @@ -1437,18 +1439,17 @@ int vtkAMReXGridReaderInternal::ReadNumberOfAttributes(std::istream& is) } void vtkAMReXGridReaderInternal::ReadBlockAttribute( - std::istream& is, long numberOfPoints, long size, char* buffer) + std::istream& is, std::size_t numberOfPoints, std::size_t size, char* buffer) { is.read(buffer, numberOfPoints * size); } -void vtkAMReXGridReaderInternal::Convert( - void* out, const void* in, long nitems, const RealDescriptor& ord, const RealDescriptor& ird) +void vtkAMReXGridReaderInternal::Convert(void* out, const void* in, std::size_t nitems, + const RealDescriptor& ord, const RealDescriptor& ird) { if (ord == ird) { - size_t n = size_t(nitems); - memcpy(out, in, n * ord.numBytes()); + memcpy(out, in, nitems * ord.numBytes()); } else if (ord.formatarray() == ird.formatarray()) { @@ -1462,8 +1463,8 @@ void vtkAMReXGridReaderInternal::Convert( } } -void vtkAMReXGridReaderInternal::PermuteOrder( - void* out, const void* in, long nitems, const int* outord, const int* inord, int REALSIZE) +void vtkAMReXGridReaderInternal::PermuteOrder(void* out, const void* in, std::size_t nitems, + const int* outord, const int* inord, std::size_t REALSIZE) { char const* pin = static_cast<char const*>(in); char* pout = static_cast<char*>(out); @@ -1473,7 +1474,7 @@ void vtkAMReXGridReaderInternal::PermuteOrder( for (; nitems > 0; nitems--, pin += REALSIZE, pout += REALSIZE) { - for (int i = 0; i < REALSIZE; i++) + for (std::size_t i = 0; i < REALSIZE; i++) pout[outord[i]] = pin[inord[i]]; } } diff --git a/IO/AMR/vtkAMReXGridReaderInternal.h b/IO/AMR/vtkAMReXGridReaderInternal.h index b3cec3d88e4549127d936e29b70aa6add2a775ff..180fd3735184e2e449f81afc01d537ca255b2e72 100644 --- a/IO/AMR/vtkAMReXGridReaderInternal.h +++ b/IO/AMR/vtkAMReXGridReaderInternal.h @@ -59,7 +59,7 @@ public: const std::vector<long>& formatarray() const&; const int* order() const&; const std::vector<int>& orderarray() const&; - long numBytes() const; + std::size_t numBytes() const; bool operator==(const RealDescriptor& rd) const; private: @@ -188,7 +188,7 @@ public: int levelNumberOfFABOnDisk; std::string levelFabOnDiskPrefix; std::vector<std::string> levelFABFile; - std::vector<long> levelFileOffset; + std::vector<std::size_t> levelFileOffset; std::vector<std::vector<double>> levelMinimumsFAB; std::vector<std::vector<double>> levelMaximumsFAB; std::vector<double> levelFABArrayMinimum; @@ -232,8 +232,8 @@ public: int GetBlockIndexWithinLevel(int blockIdx, int level); void GetBlockAttribute(const char* attribute, int blockIdx, vtkDataSet* pDataSet); void GetExtraMultiFabBlockAttribute(const char* attribute, int blockIdx, vtkDataSet* pDataSet); - int GetOffsetOfAttribute(const char* attribute); - int GetAttributeOffsetExtraMultiFab(const char* attribute, int fabIndex); + vtkIdType GetOffsetOfAttribute(const char* attribute); + vtkIdType GetAttributeOffsetExtraMultiFab(const char* attribute, int fabIndex); int GetExtraMultiFabIndex(const char* attribute); void ReadFAB(std::istream& is); int ReadVersion(std::istream& is); @@ -242,19 +242,20 @@ public: void ReadFormat(std::istream& is, std::vector<long>& ar); void PrintFormat(std::vector<long>& ar); RealDescriptor* ReadRealDescriptor(std::istream& is); - long ReadBoxArray(std::istream& is, int* boxArray, int* boxArrayDim); + std::size_t ReadBoxArray(std::istream& is, int* boxArray, int* boxArrayDim); void PrintBoxArray(int* boxArray); int ReadNumberOfAttributes(std::istream& is); - void ReadBlockAttribute(std::istream& is, long numberOfPoints, long size, char* buffer); - void Convert( - void* out, const void* in, long nitems, const RealDescriptor& ord, const RealDescriptor& ird); - void PermuteOrder( - void* out, const void* in, long nitems, const int* outord, const int* inord, int REALSIZE); + void ReadBlockAttribute( + std::istream& is, std::size_t numberOfPoints, std::size_t size, char* buffer); + void Convert(void* out, const void* in, std::size_t nitems, const RealDescriptor& ord, + const RealDescriptor& ird); + void PermuteOrder(void* out, const void* in, std::size_t nitems, const int* outord, + const int* inord, std::size_t REALSIZE); template <typename T> void CreateVTKAttributeArray(vtkAOSDataArrayTemplate<T>* dataArray, const RealDescriptor* ord, - const RealDescriptor* ird, const std::vector<std::vector<char>>& buffers, int numberOfPoints, - const std::string& attribute); + const RealDescriptor* ird, const std::vector<std::vector<char>>& buffers, + std::size_t numberOfPoints, const std::string& attribute); bool headersAreRead; bool extraMultiFabHeadersAreRead; @@ -270,7 +271,8 @@ public: template <typename T> void vtkAMReXGridReaderInternal::CreateVTKAttributeArray(vtkAOSDataArrayTemplate<T>* dataArray, const RealDescriptor* ord, const RealDescriptor* ird, - const std::vector<std::vector<char>>& buffers, int numberOfPoints, const std::string& attribute) + const std::vector<std::vector<char>>& buffers, std::size_t numberOfPoints, + const std::string& attribute) { int nComps = static_cast<int>(this->Header->parsedVariableNames[attribute].size()); if (nComps == 0) // check if the variable is in an extra fab @@ -290,7 +292,7 @@ void vtkAMReXGridReaderInternal::CreateVTKAttributeArray(vtkAOSDataArrayTemplate this->Convert(arrayPtr, buffers[j].data(), numberOfPoints, *ord, *ird); // Copy to data array component - for (int i = 0; i < numberOfPoints; ++i) + for (std::size_t i = 0; i < numberOfPoints; ++i) { dataArray->SetTypedComponent(i, j, arrayPtr[i]); }