diff --git a/Documentation/release/dev/ensight-reader-improvements.md b/Documentation/release/dev/ensight-reader-improvements.md
new file mode 100644
index 0000000000000000000000000000000000000000..3781d2cb63e710456582d04ebab54ff4090cfafc
--- /dev/null
+++ b/Documentation/release/dev/ensight-reader-improvements.md
@@ -0,0 +1,7 @@
+## EnsightCombinedReader: Performance improvements pass
+
+Improve performances of the new vtkEnSightGoldCombinedReader in various ways, especially when reading large binary files containing nfaced/tetrahedron cells:
+* The tetrahedron cell-building logic now uses a tetrahedron-specific vtkUnstructredGrid::InsertNextCell call
+* Tetrahedron cells connectivity data are read as one block rather than cell-by-cell
+* Add new "skip" methods to vtkEnSightDataSet which implement dedicated logic to ignore file cell sections. The logic is duplicated a bit from the equivalent reading methods at the benefit of much faster parsing during the RequestInfo pass (to get the parts names)
+* Fix an int overflow that could occur while seeking ahead in large files
diff --git a/IO/EnSight/core/EnSightDataSet.cxx b/IO/EnSight/core/EnSightDataSet.cxx
index efca43a662db100e7485526c2617aa697012e921..86531f02d76c64ee71f4f55a6b20e6ba3eafa580 100644
--- a/IO/EnSight/core/EnSightDataSet.cxx
+++ b/IO/EnSight/core/EnSightDataSet.cxx
@@ -14,11 +14,8 @@
#include "vtkFloatArray.h"
#include "vtkIdTypeArray.h"
#include "vtkInformation.h"
-#include "vtkInformationVector.h"
#include "vtkLogger.h"
-#include "vtkMultiProcessController.h"
#include "vtkNew.h"
-#include "vtkObjectFactory.h"
#include "vtkPartitionedDataSet.h"
#include "vtkPartitionedDataSetCollection.h"
#include "vtkPointData.h"
@@ -37,6 +34,7 @@
#include <vtksys/SystemTools.hxx>
+#include <algorithm>
#include <cstdlib>
#include <numeric>
#include <regex>
@@ -1049,6 +1047,7 @@ bool EnSightDataSet::ReadGeometry(vtkPartitionedDataSetCollection* output,
vtkGenericWarningMacro("Part Id " << partId << " could not be found in PartInfoMap");
return false;
}
+
auto& partInfo = it->second;
result = this->GeometryFile.ReadNextLine(); // part description line
@@ -2501,11 +2500,11 @@ void EnSightDataSet::PassThroughUnstructuredGrid(const GridOptions& vtkNotUsed(o
{
if (elementType == ElementType::NSided)
{
- this->ReadNSidedSection(numCellsPerType[static_cast<int>(elementType)], nullptr);
+ this->SkipNSidedSection(numCellsPerType[static_cast<int>(elementType)]);
}
else if (elementType == ElementType::NFaced)
{
- this->ReadNFacedSection(numCellsPerType[static_cast<int>(elementType)], nullptr);
+ this->SkipNFacedSection(numCellsPerType[static_cast<int>(elementType)]);
}
else
{
@@ -2746,57 +2745,178 @@ void EnSightDataSet::ReadNSidedSection(int& numElements, vtkUnstructuredGrid* ou
delete[] numNodesPerElement;
}
+//------------------------------------------------------------------------------
+void EnSightDataSet::SkipNFacedSection(int& numElements)
+{
+ this->GeometryFile.ReadNumber(&numElements);
+
+ // (optional) Element IDs
+ if (this->ElementIdsListed)
+ {
+ this->GeometryFile.SkipNNumbers<int>(numElements);
+ }
+
+ // Number of faces per element
+ std::vector<int> numFacesPerElement(numElements, 0);
+ this->GeometryFile.ReadArray(numFacesPerElement.data(), numElements);
+
+ vtkIdType totalNumFaces = std::accumulate(
+ numFacesPerElement.begin(), numFacesPerElement.end(), static_cast<vtkIdType>(0));
+
+ if (this->GeometryFile.Format == FileType::ASCII)
+ {
+ for (vtkIdType i = 0; i < totalNumFaces; ++i)
+ {
+ // Skip 2 lines: number of point per face per element, face connectivity
+ this->GeometryFile.SkipLine();
+ this->GeometryFile.SkipLine();
+ }
+ }
+ else
+ {
+ std::vector<int> numNodesPerFacePerElement(totalNumFaces);
+ this->GeometryFile.ReadArray(numNodesPerFacePerElement.data(), totalNumFaces);
+
+ vtkIdType totalNumNodes = std::accumulate(numNodesPerFacePerElement.begin(),
+ numNodesPerFacePerElement.end(), static_cast<vtkIdType>(0));
+
+ this->GeometryFile.SkipNNumbers<int>(totalNumNodes);
+ }
+}
+
+//------------------------------------------------------------------------------
+void EnSightDataSet::SkipNSidedSection(int& numElements)
+{
+ this->GeometryFile.ReadNumber(&numElements);
+
+ if (this->ElementIdsListed)
+ {
+ this->GeometryFile.SkipNNumbers<int>(numElements);
+ }
+
+ if (this->GeometryFile.Format == FileType::ASCII)
+ {
+ // Skip 2 lines per element: number of nodes, node numbers for this element
+ for (int elementIdx = 0; elementIdx < numElements; ++elementIdx)
+ {
+ this->GeometryFile.SkipLine();
+ this->GeometryFile.SkipLine();
+ }
+ }
+ else
+ {
+ std::vector<int> numNodesPerElement(numElements);
+ this->GeometryFile.ReadArray(numNodesPerElement.data(), numElements);
+
+ vtkIdType totalNumNodes = std::accumulate(
+ numNodesPerElement.begin(), numNodesPerElement.end(), static_cast<vtkIdType>(0));
+ this->GeometryFile.SkipNNumbers<int>(totalNumNodes);
+ }
+}
+
//------------------------------------------------------------------------------
void EnSightDataSet::ReadNFacedSection(int& numElements, vtkUnstructuredGrid* output)
{
+ vtkLogScopeFunction(TRACE);
+
+ // Number of elements
this->GeometryFile.ReadNumber(&numElements);
+ // (optional) Element IDs
if (this->ElementIdsListed)
{
this->GeometryFile.SkipNNumbers<int>(numElements);
}
- // read number of faces per nfaced element
+ // Number of faces per element
std::vector<int> numFacesPerElement(numElements, 0);
this->GeometryFile.ReadArray(numFacesPerElement.data(), numElements);
- // read number of nodes per face of each element
- std::vector<std::vector<int>> nodesPerFacePerElement(numElements);
- std::vector<int> totalNodesPerElement(numElements, 0);
- for (int elem = 0; elem < numElements; elem++)
+ // Read the whole block in one go
+ vtkIdType totalNumFaces = std::accumulate(
+ numFacesPerElement.begin(), numFacesPerElement.end(), static_cast<vtkIdType>(0));
+
+ std::vector<int> numNodesPerFacePerElement(totalNumFaces);
+ this->GeometryFile.ReadArray(numNodesPerFacePerElement.data(), totalNumFaces);
+
+ vtkIdType totalNumNodes = std::accumulate(
+ numNodesPerFacePerElement.begin(), numNodesPerFacePerElement.end(), static_cast<vtkIdType>(0));
+ std::vector<int> faceNodesBuffer(totalNumNodes);
+
+ vtkIdType offset = 0;
+ for (vtkIdType i = 0; i < totalNumFaces; ++i)
{
- auto& numFaces = numFacesPerElement[elem];
- auto& nodesPerFace = nodesPerFacePerElement[elem];
- nodesPerFace.resize(numFaces);
- this->GeometryFile.ReadArray(nodesPerFace.data(), numFaces);
- totalNodesPerElement[elem] = std::accumulate(nodesPerFace.begin(), nodesPerFace.end(), 0);
+ this->GeometryFile.ReadArray(
+ faceNodesBuffer.data() + offset, numNodesPerFacePerElement[i], true);
+ offset += numNodesPerFacePerElement[i];
}
+ // Now build the actual cells
auto cellInfo = getVTKCellType(ElementType::NFaced);
- for (int elem = 0; elem < numElements; elem++)
+
+ auto numNodesInFaceIt = numNodesPerFacePerElement.begin();
+ auto nodeIt = faceNodesBuffer.begin();
+
+ // Break through all loops if numNodesInFaceIt or nodeIt reach the end of vector
+ bool endReached = false;
+ vtkNew<vtkCellArray> faceStream;
+
+ for (int elemIdx = 0; elemIdx < numElements; elemIdx++)
{
- auto& numNodesAllFaces = totalNodesPerElement[elem];
- auto& numFaces = numFacesPerElement[elem];
- auto arraySize = numNodesAllFaces + numFaces;
- std::vector<vtkIdType> nodeIds(arraySize, 0);
- int arrayIdx = 0;
- std::vector<int> tempIds;
- auto& numNodesPerFace = nodesPerFacePerElement[elem];
- for (int face = 0; face < numFaces; face++)
+ const int numFacesInElement = numFacesPerElement[elemIdx];
+ /// @note: we could save that value from the earlier "total" computation. It's not significant
+ // compared to the read time though
+ const vtkIdType numNodesInElement = std::accumulate(
+ numNodesInFaceIt, numNodesInFaceIt + numFacesInElement, static_cast<vtkIdType>(0));
+
+ std::vector<vtkIdType> uniqueCellIDs;
+ uniqueCellIDs.reserve(numNodesInElement);
+
+ faceStream->Reset();
+ faceStream->AllocateExact(numFacesInElement, numNodesInElement);
+
+ for (int faceIdx = 0; faceIdx < numFacesInElement; ++faceIdx)
{
- auto& numNodes = numNodesPerFace[face];
- nodeIds[arrayIdx++] = numNodes;
- tempIds.clear();
- tempIds.resize(numNodes);
- this->GeometryFile.ReadArray(tempIds.data(), numNodes, true);
- for (auto& nodeId : tempIds)
+ const int numNodesInFace = *numNodesInFaceIt;
+ faceStream->InsertNextCell(numNodesInFace);
+
+ for (int i = 0; i < numNodesInFace; ++i)
+ {
+ vtkIdType correctedId = (*nodeIt) - 1; // Ensight node IDs are 1-based
+ faceStream->InsertCellPoint(correctedId);
+
+ /// @note: We use an unsorted, unique vector instead of a set because:
+ // 1) This is a per-cell unique point list; we expect it to be relatively small
+ // 2) It allows us to use the insertNextCell call below which expects a contiguous container
+ if (std::find(uniqueCellIDs.begin(), uniqueCellIDs.end(), correctedId) ==
+ std::end(uniqueCellIDs))
+ {
+ uniqueCellIDs.push_back(correctedId);
+ }
+
+ ++nodeIt;
+ if (nodeIt == faceNodesBuffer.end())
+ {
+ endReached = true;
+ break;
+ }
+ }
+ ++numNodesInFaceIt;
+ if (endReached || numNodesInFaceIt == numNodesPerFacePerElement.end())
{
- nodeIds[arrayIdx++] = nodeId - 1;
+ endReached = true;
+ break;
}
}
+
if (output)
{
- output->InsertNextCell(cellInfo.first, numFaces, nodeIds.data());
+ output->InsertNextCell(
+ cellInfo.first, uniqueCellIDs.size(), uniqueCellIDs.data(), faceStream);
+ }
+ if (endReached)
+ {
+ break;
}
}
}
diff --git a/IO/EnSight/core/EnSightDataSet.h b/IO/EnSight/core/EnSightDataSet.h
index 13d1f3220ae7228301294d3431e780fd0be649b1..39e8eb67ffa287e4fd59452a1382bbe170140f56 100644
--- a/IO/EnSight/core/EnSightDataSet.h
+++ b/IO/EnSight/core/EnSightDataSet.h
@@ -280,6 +280,14 @@ private:
void ReadNSidedSection(int& numElements, vtkUnstructuredGrid* output);
void ReadNFacedSection(int& numElements, vtkUnstructuredGrid* output);
+ ///@{
+ /**
+ * Pass through element sections ignoring as much info as possible.
+ */
+ void SkipNSidedSection(int& numElements);
+ void SkipNFacedSection(int& numElements);
+ ///@}
+
void ReadVariableNodes(EnSightFile& file, const std::string& arrayName, int numComponents,
vtkPartitionedDataSetCollection* output, vtkDataArraySelection* selection,
bool isComplex = false, bool isReal = true);
diff --git a/IO/EnSight/core/EnSightFile.cxx b/IO/EnSight/core/EnSightFile.cxx
index 42cee83e919b75e2ef7ec1100eace20c116bf8b7..6a4c68eeeead88f3376f741e59615199aa7eff8a 100644
--- a/IO/EnSight/core/EnSightFile.cxx
+++ b/IO/EnSight/core/EnSightFile.cxx
@@ -542,6 +542,19 @@ std::pair<bool, std::string> EnSightFile::ReadNextLine(int size /* = MAX_LINE_LE
return result;
}
+//------------------------------------------------------------------------------
+void EnSightFile::SkipLine(vtkTypeInt64 size)
+{
+ if (this->Format == FileType::ASCII)
+ {
+ this->Stream->ignore(size, '\n');
+ }
+ else
+ {
+ this->MoveReadPosition(this->FortranSkipBytes * 2 + size);
+ }
+}
+
//------------------------------------------------------------------------------
std::pair<bool, std::string> EnSightFile::ReadLine(int size /* = MAX_LINE_LENGTH*/)
{
@@ -638,7 +651,7 @@ bool EnSightFile::DetectByteOrder(int* result)
}
//------------------------------------------------------------------------------
-void EnSightFile::MoveReadPosition(int numBytes)
+void EnSightFile::MoveReadPosition(vtkTypeInt64 numBytes)
{
this->Stream->seekg(numBytes, ios::cur);
}
diff --git a/IO/EnSight/core/EnSightFile.h b/IO/EnSight/core/EnSightFile.h
index c47bef212f81433bdfa83e49ee94bc72513be12a..1ff28867f8455b8c40eb31174cd77b7f354e7bed 100644
--- a/IO/EnSight/core/EnSightFile.h
+++ b/IO/EnSight/core/EnSightFile.h
@@ -8,6 +8,8 @@
#include "vtksys/FStream.hxx"
+#include <cassert>
+#include <limits>
#include <memory>
#include <sstream>
#include <string>
@@ -64,6 +66,7 @@ struct EnSightFile
Endianness ByteOrder = Endianness::Unknown;
int TimeSet = -1;
int FileSet = -1;
+ bool InBlockRead = false;
EnSightFile();
~EnSightFile();
@@ -133,6 +136,12 @@ struct EnSightFile
*/
std::pair<bool, std::string> ReadLine(int size = MAX_LINE_LENGTH);
+ /** Ignore the the next characters until either the line end delimiter is met or size characters
+ * have been ignored (if provided).
+ * For binary formats, ignore the next size characters + padding (but you should probably use
+ * another method)*/
+ void SkipLine(vtkTypeInt64 size = std::numeric_limits<std::streamsize>::max());
+
/**
* Skip the specified number of non-numeric lines when reading.
* WARNING: Should only be used for non-numeric lines, even in ASCII mode!
@@ -182,7 +191,7 @@ struct EnSightFile
/**
* Move the read position ahead n bytes.
*/
- void MoveReadPosition(int numBytes);
+ void MoveReadPosition(vtkTypeInt64 numBytes);
/**
* Get current position of reader in stream.
@@ -254,7 +263,7 @@ void EnSightFile::SkipNNumbers(vtkIdType n, int numsPerLine /* = 1 */)
// for float, 12 characters total
// there's also white space allowed between numbers
int size = getNumChars<T>() * numsPerLine + 10 * numsPerLine;
- int lineIdx = 0;
+ vtkIdType lineIdx = 0;
while (lineIdx < n)
{
auto result = this->ReadLine(size);
@@ -267,7 +276,7 @@ void EnSightFile::SkipNNumbers(vtkIdType n, int numsPerLine /* = 1 */)
}
else
{
- int numBytes = n * static_cast<int>(sizeof(T)) + this->FortranSkipBytes * 2;
+ vtkTypeInt64 numBytes = n * sizeof(T) + this->FortranSkipBytes * 2;
this->MoveReadPosition(numBytes);
}
}
@@ -285,6 +294,7 @@ bool EnSightFile::ReadNumber(T* result, bool padBeginning /* = true*/, bool padE
{
if (padBeginning)
{
+ assert(!this->InBlockRead);
this->MoveReadPosition(this->FortranSkipBytes);
}
if (!this->Stream->read((char*)result, sizeof(T)))
@@ -294,6 +304,7 @@ bool EnSightFile::ReadNumber(T* result, bool padBeginning /* = true*/, bool padE
}
if (padEnd)
{
+ assert(!this->InBlockRead);
this->MoveReadPosition(this->FortranSkipBytes);
}
if (this->ByteOrder == Endianness::Little)