Commit 21392cb6 authored by Utkarsh Ayachit's avatar Utkarsh Ayachit Committed by Kitware Robot
Browse files

Merge topic '19224-save-data-csv-release' into release

f43bbc48 remove `\t`s from writers.xml
cb20ad61 refactor vtkCSVWriter to fix #19224.
3dbed838 adding test for #19224
4d0c0424

 vtkPVMergeTables: improve code robustness
Acked-by: Kitware Robot's avatarKitware Robot <kwrobot@kitware.com>
Merge-request: !3479
parents 2f5c051d f43bbc48
......@@ -1520,22 +1520,22 @@
executed once for each timestep available from its input.</Documentation>
</IntVectorProperty>
<StringVectorProperty command="SetFileNameSuffix"
default_values="_%d"
label = "File name suffix"
default_values="_%d"
label = "File name suffix"
name="FileNameSuffix"
number_of_elements="1">
<Documentation>
The suffix to append to the file name when writing files at different timesteps.
(File extensions such as .csv should be excluded.)
The % format specifiers are used. For example, _%d will write files as FileName_0,
FileName_1, FileName_2, etc., and _%.3d will write files as FileName_000,
FileName_001, FileName_002 etc.
The suffix to append to the file name when writing files at different timesteps.
(File extensions such as .csv should be excluded.)
The % format specifiers are used. For example, _%d will write files as FileName_0,
FileName_1, FileName_2, etc., and _%.3d will write files as FileName_000,
FileName_001, FileName_002 etc.
</Documentation>
<Hints>
<PropertyWidgetDecorator type="EnableWidgetDecorator">
<Property name="WriteTimeSteps" function="boolean" />
</PropertyWidgetDecorator>
</Hints>
<Property name="WriteTimeSteps" function="boolean" />
</PropertyWidgetDecorator>
</Hints>
</StringVectorProperty>
<SubProxy>
<Proxy name="Writer"
......@@ -1592,22 +1592,22 @@
executed once for each timestep available from its input.</Documentation>
</IntVectorProperty>
<StringVectorProperty command="SetFileNameSuffix"
default_values="_%d"
label = "File name suffix"
default_values="_%d"
label = "File name suffix"
name="FileNameSuffix"
number_of_elements="1">
<Documentation>
The suffix to append to the file name when writing files at different timesteps.
(File extensions such as .csv should be excluded.)
The suffix to append to the file name when writing files at different timesteps.
(File extensions such as .csv should be excluded.)
The % format specifiers are used. For example, _%d will write files as FileName_0,
FileName_1, FileName_2, etc., and _%.3d will write files as FileName_000,
FileName_1, FileName_2, etc., and _%.3d will write files as FileName_000,
FileName_001, FileName_002 etc.
</Documentation>
<Hints>
<PropertyWidgetDecorator type="EnableWidgetDecorator">
<Property name="WriteTimeSteps" function="boolean" />
</PropertyWidgetDecorator>
</Hints>
<Property name="WriteTimeSteps" function="boolean" />
</PropertyWidgetDecorator>
</Hints>
</StringVectorProperty>
<SubProxy>
<Proxy name="Writer"
......
......@@ -6,4 +6,11 @@ vtk_add_test_cxx(vtkPVVTKExtensionsDefaultCxxTests tests
NO_VALID NO_OUTPUT
TestPVDArraySelection.cxx
)
if (PARAVIEW_USE_MPI)
vtk_add_test_mpi(vtkPVVTKExtensionsDefaultCxxTests tests
TESTING_DATA NO_VALID
TestCSVWriter.cxx
)
endif()
vtk_test_cxx_executable(vtkPVVTKExtensionsDefaultCxxTests tests)
/*=========================================================================
Program: ParaView
Module: TestCSVWriter.cxx
Copyright (c) Menno Deij - van Rijswijk, MARIN, The Netherlands
All rights reserved.
This software is distributed WITHOUT ANY WARRANTY; without even
the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE. See the above copyright notice for more information.
=========================================================================*/
#include <vtkCSVWriter.h>
#include <vtkDelimitedTextReader.h>
#include <vtkDoubleArray.h>
#include <vtkIntArray.h>
#include <vtkLogger.h>
#include <vtkMPIController.h>
#include <vtkNew.h>
#include <vtkTable.h>
#include <vtkTesting.h>
#include <string>
namespace
{
// ensure that the writer works when the columns are not in the same order on all ranks.
// also ensures partial arrays don't mess things up.
bool WriteCSV(const std::string& fname, int rank)
{
vtkNew<vtkTable> table;
vtkNew<vtkDoubleArray> col1;
col1->SetName("Column1");
col1->SetNumberOfTuples(10);
vtkNew<vtkIntArray> col2;
col2->SetName("Column2");
col2->SetNumberOfTuples(10);
vtkNew<vtkIntArray> col3;
col3->SetName("Column3-Partial");
col3->SetNumberOfTuples(10);
for (int cc = 0; cc < 10; ++cc)
{
const auto row = cc + rank * 10;
col1->SetValue(cc, row + 1.5);
col2->SetValue(cc, row * 100);
col3->SetValue(cc, 20);
}
if (rank == 0)
{
table->AddColumn(col1);
table->AddColumn(col3);
table->AddColumn(col2);
}
else
{
table->AddColumn(col2);
table->AddColumn(col1);
}
vtkNew<vtkCSVWriter> writer;
writer->SetFileName(fname.c_str());
writer->SetInputDataObject(table);
writer->Update();
return true;
}
#define VERITFY_EQ(x, y, txt) \
if ((x) != (y)) \
{ \
std::cerr << "ERROR: " << txt << endl \
<< "expected (" << (x) << "), got (" << (y) << ")" << endl; \
return false; \
}
bool ReadAndVerifyCSV(const std::string& fname, int rank, int numRanks)
{
if (rank != 0)
{
return true;
}
vtkNew<vtkDelimitedTextReader> reader;
reader->SetFileName(fname.c_str());
reader->SetHaveHeaders(true);
reader->SetDetectNumericColumns(true);
reader->Update();
auto table = reader->GetOutput();
VERITFY_EQ(table->GetNumberOfRows(), 10 * numRanks, "incorrect row count");
VERITFY_EQ(table->GetNumberOfColumns(), 2, "incorrect column count");
for (int irank = 0; irank < numRanks; ++irank)
{
for (vtkIdType cc = 0; cc < 10; ++cc)
{
const auto row = cc + irank * 10;
auto value1 = table->GetValueByName(row, "Column1");
auto value2 = table->GetValueByName(row, "Column2");
VERITFY_EQ(value1.ToDouble(), row + 1.5,
std::string("incorrect column1 values at row ") + std::to_string(row));
VERITFY_EQ(value2.ToInt(), row * 100,
std::string("incorrect column2 values at row ") + std::to_string(row));
}
}
return true;
}
} // end of namespace
int TestCSVWriter(int argc, char* argv[])
{
vtkMPIController* contr = vtkMPIController::New();
contr->Initialize(&argc, &argv);
vtkMultiProcessController::SetGlobalController(contr);
const int myRank = contr->GetLocalProcessId();
const int numRanks = contr->GetNumberOfProcesses();
vtkNew<vtkTesting> testing;
testing->AddArguments(argc, argv);
if (!testing->GetTempDirectory())
{
vtkLogF(ERROR, "no temp directory specified!");
contr->Finalize();
contr->Delete();
return EXIT_FAILURE;
}
std::string tname{ testing->GetTempDirectory() };
int success = WriteCSV(tname + "/TestCSVWriter.csv", myRank) &&
ReadAndVerifyCSV(tname + "/TestCSVWriter.csv", myRank, numRanks)
? 1
: 0;
int all_success;
contr->AllReduce(&success, &all_success, 1, vtkCommunicator::LOGICAL_AND_OP);
vtkMultiProcessController::SetGlobalController(nullptr);
contr->Finalize();
contr->Delete();
return all_success ? EXIT_SUCCESS : EXIT_FAILURE;
}
......@@ -30,6 +30,10 @@ OPTIONAL_DEPENDS
VTK::FiltersParallelMPI
VTK::ParallelMPI
TEST_DEPENDS
VTK::IOInfovis
VTK::TestingCore
VTK::TestingRendering
TEST_OPTIONAL_DEPENDS
VTK::ParallelMPI
TEST_LABELS
ParaView
......@@ -32,10 +32,12 @@
#include "vtkStreamingDemandDrivenPipeline.h"
#include "vtkTable.h"
#include <numeric>
#include <sstream>
#include <vector>
vtkStandardNewMacro(vtkCSVWriter);
vtkCxxSetObjectMacro(vtkCSVWriter, Controller, vtkMultiProcessController);
//-----------------------------------------------------------------------------
vtkCSVWriter::vtkCSVWriter()
{
......@@ -44,21 +46,22 @@ vtkCSVWriter::vtkCSVWriter()
this->UseStringDelimiter = true;
this->SetStringDelimiter("\"");
this->SetFieldDelimiter(",");
this->Stream = 0;
this->FileName = 0;
this->Precision = 5;
this->UseScientificNotation = true;
this->FieldAssociation = 0;
this->AddMetaData = false;
this->Controller = nullptr;
this->SetController(vtkMultiProcessController::GetGlobalController());
}
//-----------------------------------------------------------------------------
vtkCSVWriter::~vtkCSVWriter()
{
this->SetController(nullptr);
this->SetStringDelimiter(0);
this->SetFieldDelimiter(0);
this->SetFileName(0);
delete this->Stream;
}
//-----------------------------------------------------------------------------
......@@ -76,74 +79,37 @@ int vtkCSVWriter::ProcessRequest(
{
if (request->Has(vtkStreamingDemandDrivenPipeline::REQUEST_UPDATE_EXTENT()))
{
vtkMultiProcessController* controller = vtkMultiProcessController::GetGlobalController();
vtkInformation* inInfo = inputVector[0]->GetInformationObject(0);
inInfo->Set(vtkStreamingDemandDrivenPipeline::UPDATE_NUMBER_OF_PIECES(),
controller->GetNumberOfProcesses());
inInfo->Set(
vtkStreamingDemandDrivenPipeline::UPDATE_PIECE_NUMBER(), controller->GetLocalProcessId());
(this->Controller ? this->Controller->GetNumberOfProcesses() : 1));
inInfo->Set(vtkStreamingDemandDrivenPipeline::UPDATE_PIECE_NUMBER(),
(this->Controller ? this->Controller->GetLocalProcessId() : 0));
inInfo->Set(vtkStreamingDemandDrivenPipeline::UPDATE_NUMBER_OF_GHOST_LEVELS(), 0);
return 1;
}
return this->Superclass::ProcessRequest(request, inputVector, outputVector);
}
//-----------------------------------------------------------------------------
bool vtkCSVWriter::OpenFile(bool append)
{
if (!this->FileName)
{
vtkErrorMacro(<< "No FileName specified! Can't write!");
this->SetErrorCode(vtkErrorCode::NoFileNameError);
return false;
}
vtkDebugMacro(<< "Opening file for writing...");
ofstream* fptr = append == false ? new ofstream(this->FileName, ios::out)
: new ofstream(this->FileName, ios::out | ios::app);
if (fptr->fail())
{
vtkErrorMacro(<< "Unable to open file: " << this->FileName);
this->SetErrorCode(vtkErrorCode::CannotOpenFileError);
delete fptr;
return false;
}
this->Stream = fptr;
return true;
}
namespace
{
//-----------------------------------------------------------------------------
template <class iterT>
void vtkCSVWriterGetDataString(
iterT* iter, vtkIdType tupleIndex, ofstream* stream, vtkCSVWriter* writer, bool* first)
iterT* iter, vtkIdType tupleIndex, ofstream& stream, vtkCSVWriter* writer, bool* first)
{
int numComps = iter->GetNumberOfComponents();
vtkIdType index = tupleIndex * numComps;
for (int cc = 0; cc < numComps; cc++)
{
if ((index + cc) < iter->GetNumberOfValues())
if (!(*first))
{
if (*first == false)
{
(*stream) << writer->GetFieldDelimiter();
}
*first = false;
(*stream) << iter->GetValue(index + cc);
stream << writer->GetFieldDelimiter();
}
else
*first = false;
if ((index + cc) < iter->GetNumberOfValues())
{
if (*first == false)
{
(*stream) << writer->GetFieldDelimiter();
}
*first = false;
stream << iter->GetValue(index + cc);
}
}
}
......@@ -151,28 +117,20 @@ void vtkCSVWriterGetDataString(
//-----------------------------------------------------------------------------
template <>
void vtkCSVWriterGetDataString(vtkArrayIteratorTemplate<vtkStdString>* iter, vtkIdType tupleIndex,
ofstream* stream, vtkCSVWriter* writer, bool* first)
ofstream& stream, vtkCSVWriter* writer, bool* first)
{
int numComps = iter->GetNumberOfComponents();
vtkIdType index = tupleIndex * numComps;
for (int cc = 0; cc < numComps; cc++)
{
if ((index + cc) < iter->GetNumberOfValues())
if (!(*first))
{
if (*first == false)
{
(*stream) << writer->GetFieldDelimiter();
}
*first = false;
(*stream) << writer->GetString(iter->GetValue(index + cc));
stream << writer->GetFieldDelimiter();
}
else
(*first) = false;
if ((index + cc) < iter->GetNumberOfValues())
{
if (*first == false)
{
(*stream) << writer->GetFieldDelimiter();
}
*first = false;
stream << writer->GetString(iter->GetValue(index + cc));
}
}
}
......@@ -180,28 +138,20 @@ void vtkCSVWriterGetDataString(vtkArrayIteratorTemplate<vtkStdString>* iter, vtk
//-----------------------------------------------------------------------------
template <>
void vtkCSVWriterGetDataString(vtkArrayIteratorTemplate<char>* iter, vtkIdType tupleIndex,
ofstream* stream, vtkCSVWriter* writer, bool* first)
ofstream& stream, vtkCSVWriter* writer, bool* first)
{
int numComps = iter->GetNumberOfComponents();
vtkIdType index = tupleIndex * numComps;
for (int cc = 0; cc < numComps; cc++)
{
if ((index + cc) < iter->GetNumberOfValues())
if (!(*first))
{
if (*first == false)
{
(*stream) << writer->GetFieldDelimiter();
}
*first = false;
(*stream) << static_cast<int>(iter->GetValue(index + cc));
stream << writer->GetFieldDelimiter();
}
else
(*first) = false;
if ((index + cc) < iter->GetNumberOfValues())
{
if (*first == false)
{
(*stream) << writer->GetFieldDelimiter();
}
*first = false;
stream << static_cast<int>(iter->GetValue(index + cc));
}
}
}
......@@ -209,7 +159,7 @@ void vtkCSVWriterGetDataString(vtkArrayIteratorTemplate<char>* iter, vtkIdType t
//-----------------------------------------------------------------------------
template <>
void vtkCSVWriterGetDataString(vtkArrayIteratorTemplate<unsigned char>* iter, vtkIdType tupleIndex,
ofstream* stream, vtkCSVWriter* writer, bool* first)
ofstream& stream, vtkCSVWriter* writer, bool* first)
{
int numComps = iter->GetNumberOfComponents();
vtkIdType index = tupleIndex * numComps;
......@@ -219,92 +169,124 @@ void vtkCSVWriterGetDataString(vtkArrayIteratorTemplate<unsigned char>* iter, vt
{
if (*first == false)
{
(*stream) << writer->GetFieldDelimiter();
stream << writer->GetFieldDelimiter();
}
*first = false;
(*stream) << static_cast<int>(iter->GetValue(index + cc));
stream << static_cast<int>(iter->GetValue(index + cc));
}
else
{
if (*first == false)
{
(*stream) << writer->GetFieldDelimiter();
stream << writer->GetFieldDelimiter();
}
*first = false;
}
}
}
//-----------------------------------------------------------------------------
bool SomethingForMeToDo(int myRank, const std::vector<vtkIdType>& numRowsGlobal)
} // end anonymous namespace
class vtkCSVWriter::CSVFile
{
// there's something for me to do if either I have rows or I'm process 0
// and no process rows in which case I have to write the header
if (numRowsGlobal[myRank] > 0)
{
return true;
}
else if (myRank == 0)
ofstream Stream;
std::vector<std::pair<std::string, int> > ColumnInfo;
public:
int Open(const char* filename)
{
for (size_t i = 1; i < numRowsGlobal.size(); i++)
if (!filename)
{
if (numRowsGlobal[i] > 0)
{
return false; // someone else will write the header info
}
return vtkErrorCode::NoFileNameError;
}
return true; // I have to write the header info even though I have no rows
}
return false;
}
//-----------------------------------------------------------------------------
bool DoIWriteTheHeader(int myRank, const std::vector<vtkIdType>& numRowsGlobal)
{
for (int i = 0; i < myRank; i++)
{
if (numRowsGlobal[i])
this->Stream = ofstream(filename, ios::out);
if (this->Stream.fail())
{
return false; // someone before me has data and will write the header
return vtkErrorCode::CannotOpenFileError;
}
return vtkErrorCode::NoError;
}
return true; // note if process 0 is here it will write the header
}
//-----------------------------------------------------------------------------
void StartProcessWrite(
int myRank, const std::vector<vtkIdType>& numRowsGlobal, vtkMultiProcessController* controller)
{
if (DoIWriteTheHeader(myRank, numRowsGlobal) == false)
void WriteHeader(vtkTable* table, vtkCSVWriter* self)
{
this->WriteHeader(table->GetRowData(), self);
}
void WriteHeader(vtkDataSetAttributes* dsa, vtkCSVWriter* self)
{
int prevProc = myRank - 1;
while (static_cast<size_t>(prevProc) > 0 && numRowsGlobal[prevProc] == 0)
for (int cc = 0, numArrays = dsa->GetNumberOfArrays(); cc < numArrays; ++cc)
{
auto array = dsa->GetAbstractArray(cc);
const int num_comps = array->GetNumberOfComponents();
// save order of arrays written out in header
this->ColumnInfo.push_back(std::make_pair(std::string(array->GetName()), num_comps));
for (int comp = 0; comp < num_comps; ++comp)
{
if (cc > 0 || comp > 0)
{
// add separator for all but the very first column
this->Stream << self->GetFieldDelimiter();
}
std::ostringstream array_name;
array_name << array->GetName();
if (array->GetNumberOfComponents() > 1)
{
array_name << ":" << comp;
}
this->Stream << self->GetString(array_name.str());
}
}
this->Stream << "\n";
// push the floating point precision/notation type.
if (self->GetUseScientificNotation())
{
prevProc--;
this->Stream << std::scientific;
}
int tmp = 0; // just used for the blocking send/receive
controller->Receive(&tmp, 1, prevProc, 11419);
this->Stream << std::setprecision(self->GetPrecision());
}
}
//-----------------------------------------------------------------------------
void EndProcessWrite(
int myRank, const std::vector<vtkIdType>& numRowsGlobal, vtkMultiProcessController* controller)
{
int nextProc = myRank + 1;
while (static_cast<size_t>(nextProc) < numRowsGlobal.size() && numRowsGlobal[nextProc] == 0)
void WriteData(vtkTable* table, vtkCSVWriter* self)
{
nextProc++;
this->WriteData(table->GetRowData(), self);
}
if (static_cast<size_t>(nextProc) < numRowsGlobal.size())
void WriteData(vtkDataSetAttributes* dsa, vtkCSVWriter* self)
{
int tmp = 0; // just used for the blocking send/receive
controller->Send(&tmp, 1, nextProc, 11419);
}
}
std::vector<vtkSmartPointer<vtkArrayIterator> > columnsIters;
for (const auto& cinfo : this->ColumnInfo)
{
auto array = dsa->GetAbstractArray(cinfo.first.c_str());
if (array->GetNumberOfComponents() != cinfo.second)
{
vtkErrorWithObjectMacro(self, "Mismatched components for '" << array->GetName() << "'!");