Commit 4c64e089 authored by Utkarsh Ayachit's avatar Utkarsh Ayachit Committed by Kitware Robot

Merge topic 'read-selected-halos-both-readers'

0d0a51a1 Remove space from property name
9337ed45 Add load only this halo option to multiblock genericio reader
78d892ef Fixed loading data arrays when reading selected halos
parents abeac213 0d0a51a1
......@@ -122,7 +122,7 @@
</Documentation>
</IntVectorProperty>
<StringVectorProperty name="Halo Id"
<StringVectorProperty name="HaloId"
command="SetHaloIdVariableName"
number_of_elements="1"
panel_visibility="advanced">
......@@ -254,6 +254,40 @@
</Documentation>
</IntVectorProperty>
<StringVectorProperty name="HaloId"
command="SetHaloIdVariableName"
number_of_elements="1"
panel_visibility="advanced">
<StringListDomain name="array_list">
<RequiredProperties>
<Property function="ArrayList" name="PointArrayListInfo" />
</RequiredProperties>
</StringListDomain>
<Documentation>
Which data array in the file contains the halo ids. This is used with
the halo ids to load to determine which points to load.
</Documentation>
</StringVectorProperty>
<IntVectorProperty
name="HalosToLoad"
command="SetRequestedHaloId"
label="Halos To Load"
number_of_elements="0"
number_of_elements_per_command="1"
repeat_command="1"
set_number_command="SetNumberOfRequestedHaloIds"
use_index="1"
panel_visibility="advanced" >
<Documentation>
Only load in points with these halo ids from the file.
If no halo ids are specified, load all the points.
</Documentation>
<IntRangeDomain min="0" name="range" />
</IntVectorProperty>
</SourceProxy>
</ProxyGroup>
......@@ -284,7 +318,7 @@
<Property name="zAxis" />
<Property name="QueryRankNeighbors" />
<Property name="RankInQuery" />
<Property name="Halo Id" />
<Property name="HaloId" />
<Property name="HalosToLoad" />
</ExposedProperties>
</SubProxy>
......@@ -342,6 +376,8 @@
<Property name="zAxis" />
<Property name="ReadMethod"/>
<Property name="BlockAssignmentStrategy"/>
<Property name="HaloId" />
<Property name="HalosToLoad" />
</ExposedProperties>
</SubProxy>
<StringVectorProperty command="GetCurrentFileName"
......
/*=========================================================================
Program: Visualization Toolkit
Module: vtkPGenericIOReader.cxx
Module: vtkPGenericIOMultiBlockReader.cxx
Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
All rights reserved.
......@@ -20,6 +20,7 @@
#include "vtkCompositeDataPipeline.h"
#include "vtkDataArraySelection.h"
#include "vtkGenericIOUtilities.h"
#include "vtkIdList.h"
#include "vtkInformation.h"
#include "vtkInformationKey.h"
#include "vtkInformationDoubleKey.h"
......@@ -181,6 +182,7 @@ vtkPGenericIOMultiBlockReader::vtkPGenericIOMultiBlockReader()
this->XAxisVariableName = NULL;
this->YAxisVariableName = NULL;
this->ZAxisVariableName = NULL;
this->HaloIdVariableName = NULL;
this->GenericIOType = IOTYPEMPI;
this->BlockAssignment = ROUND_ROBIN;
this->BuildMetaData = false;
......@@ -194,6 +196,7 @@ vtkPGenericIOMultiBlockReader::vtkPGenericIOMultiBlockReader()
this->ArrayList = vtkStringArray::New();
this->PointDataArraySelection = vtkDataArraySelection::New();
this->HaloList = vtkIdList::New();
this->SelectionObserver = vtkCallbackCommand::New();
this->SelectionObserver->SetCallback(
&vtkPGenericIOMultiBlockReader::SelectionModifiedCallback);
......@@ -215,6 +218,7 @@ vtkPGenericIOMultiBlockReader::~vtkPGenericIOMultiBlockReader()
vtkGenericIOUtilities::SafeDeleteString(this->XAxisVariableName);
vtkGenericIOUtilities::SafeDeleteString(this->YAxisVariableName);
vtkGenericIOUtilities::SafeDeleteString(this->ZAxisVariableName);
vtkGenericIOUtilities::SafeDeleteString(this->HaloIdVariableName);
if (this->MetaData != NULL)
{
......@@ -223,6 +227,7 @@ vtkPGenericIOMultiBlockReader::~vtkPGenericIOMultiBlockReader()
this->ArrayList->Delete();
this->PointDataArraySelection->RemoveObserver(this->SelectionObserver);
this->HaloList->Delete();
this->SelectionObserver->Delete();
this->PointDataArraySelection->Delete();
......@@ -289,6 +294,47 @@ void vtkPGenericIOMultiBlockReader::SetPointArrayStatus(
}
}
//------------------------------------------------------------------------------
vtkIdType vtkPGenericIOMultiBlockReader::GetRequestedHaloId(vtkIdType i)
{
assert("pre: array index out of bounds" &&
(i >= 0 && this->HaloList->GetNumberOfIds() > i));
return this->HaloList->GetId(i);
}
//------------------------------------------------------------------------------
vtkIdType vtkPGenericIOMultiBlockReader::GetNumberOfRequestedHaloIds()
{
return this->HaloList->GetNumberOfIds();
}
//------------------------------------------------------------------------------
void vtkPGenericIOMultiBlockReader::SetNumberOfRequestedHaloIds(vtkIdType numIds)
{
this->HaloList->SetNumberOfIds(numIds);
this->Modified();
}
//------------------------------------------------------------------------------
void vtkPGenericIOMultiBlockReader::AddRequestedHaloId(vtkIdType haloId)
{
this->SetRequestedHaloId(this->GetNumberOfRequestedHaloIds(),haloId);
}
//------------------------------------------------------------------------------
void vtkPGenericIOMultiBlockReader::ClearRequestedHaloIds()
{
this->HaloList->Reset();
this->Modified();
}
//------------------------------------------------------------------------------
void vtkPGenericIOMultiBlockReader::SetRequestedHaloId(vtkIdType i, vtkIdType haloId)
{
*this->HaloList->WritePointer(i,1) = haloId;
this->Modified();
}
//------------------------------------------------------------------------------
bool vtkPGenericIOMultiBlockReader::ReaderParametersChanged()
{
......@@ -596,6 +642,13 @@ void vtkPGenericIOMultiBlockReader::LoadRawDataForBlock(int blockId)
this->LoadRawVariableDataForBlock( yaxis, blockId );
this->LoadRawVariableDataForBlock( zaxis, blockId );
if (this->HaloList->GetNumberOfIds() > 0)
{
std::string haloIds = std::string(this->HaloIdVariableName);
haloIds = vtkGenericIOUtilities::trim(haloIds);
this->LoadRawVariableDataForBlock(haloIds,blockId);
}
#ifdef DEBUG
std::cout << "\t==========\n";
std::cout << "\tNUMBER OF ARRAYS: "
......@@ -660,8 +713,9 @@ void vtkPGenericIOMultiBlockReader::GetPointFromRawData(
}
//------------------------------------------------------------------------------
void vtkPGenericIOMultiBlockReader::LoadCoordinatesForBlock(vtkUnstructuredGrid *grid,
int blockId)
void vtkPGenericIOMultiBlockReader::LoadCoordinatesForBlock(
vtkUnstructuredGrid *grid, std::set< vtkIdType >& pointsInSelectedHalos,
int blockId)
{
assert("pre: metadata is NULL!" && (this->MetaData != NULL));
assert("pre: grid is NULL!" && (grid != NULL) );
......@@ -708,13 +762,45 @@ void vtkPGenericIOMultiBlockReader::LoadCoordinatesForBlock(vtkUnstructuredGrid
double pnt[3];
vtkIdType idx = 0;
for( ;idx < nparticles; ++idx)
if (this->HaloList->GetNumberOfIds() == 0)
{
this->GetPointFromRawData(xType, xBuffer, yType, yBuffer, zType, zBuffer,
idx, pnt);
pnts->SetPoint(idx,pnt);
cells->InsertNextCell(1,&idx);
} // END for all points
for( ;idx < nparticles; ++idx)
{
this->GetPointFromRawData(xType, xBuffer, yType, yBuffer, zType, zBuffer,
idx, pnt);
pnts->SetPoint(idx,pnt);
cells->InsertNextCell(1,&idx);
} // END for all points
}
else
{
std::string haloVarName = std::string(this->HaloIdVariableName);
haloVarName = vtkGenericIOUtilities::trim(haloVarName);
int haloType = this->MetaData->VariableGenericIOType[haloVarName];
void* haloBuffer = dataBlock.RawCache[haloVarName];
for (vtkIdType i = 0; idx < nparticles; ++idx)
{
vtkIdType haloId = vtkGenericIOUtilities::GetIdFromRawBuffer(haloType,haloBuffer,idx);
bool isInRequestedHalo = false;
for (vtkIdType j = 0; j < this->GetNumberOfRequestedHaloIds(); ++j)
{
if (haloId == this->HaloList->GetId(j))
{
isInRequestedHalo = true;
pointsInSelectedHalos.insert(idx);
break;
}
}
if (isInRequestedHalo)
{
this->GetPointFromRawData(xType,xBuffer,yType,yBuffer,zType,zBuffer,idx,pnt);
pnts->SetPoint(i,pnt);
cells->InsertNextCell(1,&i);
++i;
}
}
pnts->SetNumberOfPoints(pointsInSelectedHalos.size());
}
grid->SetPoints(pnts);
......@@ -723,9 +809,25 @@ void vtkPGenericIOMultiBlockReader::LoadCoordinatesForBlock(vtkUnstructuredGrid
grid->Squeeze();
}
namespace {
template< typename T >
void GetOnlyDataInHalo(vtkDataArray* allData, vtkDataArray* haloData, std::set< vtkIdType > pointsInHalo)
{
T* data = (T*) allData->GetVoidPointer(0);
T* filteredData = (T*) haloData->GetVoidPointer(0);
vtkIdType i = 0;
for (std::set< vtkIdType >::iterator itr = pointsInHalo.begin();
itr != pointsInHalo.end(); ++itr)
{
filteredData[i++] = data[*itr];
}
}
}
//------------------------------------------------------------------------------
void vtkPGenericIOMultiBlockReader::LoadDataArraysForBlock(vtkUnstructuredGrid *grid,
int blockId)
void vtkPGenericIOMultiBlockReader::LoadDataArraysForBlock(
vtkUnstructuredGrid *grid, const std::set< vtkIdType >& pointsInSelectedHalos,
int blockId)
{
assert("pre: metadata is NULL!" && (this->MetaData != NULL));
assert("pre: grid is NULL!" && (grid != NULL) );
......@@ -734,8 +836,8 @@ void vtkPGenericIOMultiBlockReader::LoadDataArraysForBlock(vtkUnstructuredGrid *
block_t& dataBlock = this->MetaData->Blocks[blockId];
assert("pre: # points in dataset different from points in block" &&
(static_cast<uint64_t>(grid->GetNumberOfPoints()) == dataBlock.NumberOfElements));
// assert("pre: # points in dataset different from points in block" &&
// (static_cast<uint64_t>(grid->GetNumberOfPoints()) == dataBlock.NumberOfElements));
vtkPointData* PD = grid->GetPointData();
int arrayIdx = 0;
......@@ -745,16 +847,29 @@ void vtkPGenericIOMultiBlockReader::LoadDataArraysForBlock(vtkUnstructuredGrid *
if( this->PointDataArraySelection->ArrayIsEnabled(name) )
{
std::string varName( name );
vtkDataArray *dataArray =
vtkSmartPointer< vtkDataArray > dataArray;
dataArray.TakeReference(
vtkGenericIOUtilities::GetVtkDataArray(
varName,
this->MetaData->VariableGenericIOType[ varName ],
dataBlock.RawCache[ varName ],
dataBlock.NumberOfElements
);
));
if (this->HaloList->GetNumberOfIds() != 0)
{
vtkSmartPointer< vtkDataArray > onlyDataInHalo;
onlyDataInHalo.TakeReference(dataArray->NewInstance());
onlyDataInHalo->SetNumberOfTuples(grid->GetNumberOfPoints());
onlyDataInHalo->SetName(dataArray->GetName());
switch (dataArray->GetDataType()) {
vtkTemplateMacro(GetOnlyDataInHalo<VTK_TT>(
dataArray,onlyDataInHalo,pointsInSelectedHalos));
}
dataArray = onlyDataInHalo;
}
PD->AddArray( dataArray );
dataArray->Delete();
} // END if the array is enabled
} // END for all arrays
......@@ -771,12 +886,13 @@ vtkUnstructuredGrid* vtkPGenericIOMultiBlockReader::LoadBlock(int blockId)
this->LoadRawDataForBlock(blockId);
vtkUnstructuredGrid* grid = vtkUnstructuredGrid::New();
std::set< vtkIdType > pointsInSelectedHalos;
// STEP 2: Load coordinates
this->LoadCoordinatesForBlock(grid,blockId);
this->LoadCoordinatesForBlock(grid,pointsInSelectedHalos,blockId);
// STEP 3: Load data
this->LoadDataArraysForBlock(grid,blockId);
this->LoadDataArraysForBlock(grid,pointsInSelectedHalos,blockId);
if (this->Reader->IsSpatiallyDecomposed())
{
......
......@@ -20,10 +20,13 @@
#include "vtkPVVTKExtensionsCosmoToolsModule.h"
#include "vtkMultiBlockDataSetAlgorithm.h" // parent class
#include <set>
class vtkCallbackCommand;
class vtkDataArraySelection;
class vtkMultiProcessController;
class vtkStringArray;
class vtkIdList;
class vtkUnstructuredGrid;
class vtkInformationDoubleKey;
......@@ -114,6 +117,58 @@ public:
// Description:
// Sets the status of the array corresponding to the given name.
void SetPointArrayStatus(const char* name, int status);
// Description:
// Gets/Sets the variable name for the halo id of the particle.
// This is used by the requested halo selector to select only the
// points in the desired halos.
vtkSetStringMacro(HaloIdVariableName)
vtkGetStringMacro(HaloIdVariableName)
// Description:
// Gets the ith requested halo id.
// If the number of requested halo ids is
// greater than 0, only points with those halo ids will be read in.
// Otherwise all points will be read in.
vtkIdType GetRequestedHaloId(vtkIdType i);
// Description:
// Gets the number of requested halo ids.
// If the number of requested halo ids is
// greater than 0, only points with those halo ids will be read in.
// Otherwise all points will be read in.
vtkIdType GetNumberOfRequestedHaloIds();
// Description:
// Sets the number of requested halo ids.
// Use SetRequestedHaloId() to se the ids after this is called
// If the number of requested halo ids is
// greater than 0, only points with those halo ids will be read in.
// Otherwise all points will be read in.
void SetNumberOfRequestedHaloIds(vtkIdType numIds);
// Description:
// Adds the given halo id to the list of halo ids to request.
// If the number of requested halo ids is
// greater than 0, only points with those halo ids will be read in.
// Otherwise all points will be read in.
void AddRequestedHaloId(vtkIdType haloId);
// Description:
// Clears the list of requested halo ids.
// If the number of requested halo ids is
// greater than 0, only points with those halo ids will be read in.
// Otherwise all points will be read in.
void ClearRequestedHaloIds();
// Description:
// Sets the ith requested halo id to the given haloId.
// If the number of requested halo ids is
// greater than 0, only points with those halo ids will be read in.
// Otherwise all points will be read in.
void SetRequestedHaloId(vtkIdType i, vtkIdType haloId);
protected:
vtkPGenericIOMultiBlockReader();
~vtkPGenericIOMultiBlockReader();
......@@ -121,6 +176,7 @@ protected:
char* XAxisVariableName;
char* YAxisVariableName;
char* ZAxisVariableName;
char* HaloIdVariableName;
char* FileName;
int GenericIOType;
......@@ -132,6 +188,7 @@ protected:
vtkStringArray* ArrayList;
vtkDataArraySelection* PointDataArraySelection;
vtkIdList* HaloList;
vtkCallbackCommand* SelectionObserver;
gio::GenericIOReader* Reader;
......@@ -150,9 +207,13 @@ protected:
int xType, void* xBuffer, int yType, void* yBuffer, int zType,
void* zBuffer, vtkIdType id, double point[3]);
void LoadCoordinatesForBlock(vtkUnstructuredGrid* grid, int blockId);
void LoadCoordinatesForBlock(vtkUnstructuredGrid* grid,
std::set< vtkIdType >& pointsInSelectedHalos,
int blockId);
void LoadDataArraysForBlock(vtkUnstructuredGrid* grid, int blockId);
void LoadDataArraysForBlock(vtkUnstructuredGrid* grid,
const std::set<vtkIdType>& pointsInSelectedHalos,
int blockId);
vtkUnstructuredGrid* LoadBlock(int blockId);
......
......@@ -33,6 +33,7 @@
#include "vtkObjectFactory.h"
#include "vtkPointData.h"
#include "vtkPoints.h"
#include "vtkSmartPointer.h"
#include "vtkStdString.h"
#include "vtkStreamingDemandDrivenPipeline.h"
#include "vtkStringArray.h"
......@@ -628,7 +629,8 @@ void vtkPGenericIOReader::GetPointFromRawData(
}
//------------------------------------------------------------------------------
void vtkPGenericIOReader::LoadCoordinates(vtkUnstructuredGrid *grid)
void vtkPGenericIOReader::LoadCoordinates(vtkUnstructuredGrid *grid,
std::set< vtkIdType >& pointsInSelectedHalos)
{
assert("pre: grid is NULL!" && (grid != NULL) );
......@@ -688,7 +690,8 @@ void vtkPGenericIOReader::LoadCoordinates(vtkUnstructuredGrid *grid)
haloVarName = vtkGenericIOUtilities::trim(haloVarName);
int haloType = this->MetaData->VariableGenericIOType[haloVarName];
void* haloBuffer = this->MetaData->RawCache[haloVarName];
for (vtkIdType i = 0; idx < nparticles; ++idx)
vtkIdType numPointsSoFar = 0;
for (; idx < nparticles; ++idx)
{
vtkIdType haloId = vtkGenericIOUtilities::GetIdFromRawBuffer(haloType,haloBuffer,idx);
bool isInRequestedHalo = false;
......@@ -697,17 +700,20 @@ void vtkPGenericIOReader::LoadCoordinates(vtkUnstructuredGrid *grid)
if (haloId == this->HaloList->GetId(j))
{
isInRequestedHalo = true;
pointsInSelectedHalos.insert(idx);
break;
}
}
if (isInRequestedHalo)
{
this->GetPointFromRawData(xType,xBuffer,yType,yBuffer,zType,zBuffer,idx,pnt);
pnts->SetPoint(i,pnt);
cells->InsertNextCell(1,&i);
++i;
pnts->SetPoint(numPointsSoFar,pnt);
cells->InsertNextCell(1,&numPointsSoFar);
++numPointsSoFar;
}
}
// set the number of points correctly
pnts->SetNumberOfPoints(numPointsSoFar);
}
grid->SetPoints(pnts);
......@@ -719,8 +725,24 @@ void vtkPGenericIOReader::LoadCoordinates(vtkUnstructuredGrid *grid)
grid->Squeeze();
}
namespace {
template< typename T >
void GetOnlyDataInHalo(vtkDataArray* allData, vtkDataArray* haloData, std::set< vtkIdType > pointsInHalo)
{
T* data = (T*) allData->GetVoidPointer(0);
T* filteredData = (T*) haloData->GetVoidPointer(0);
vtkIdType i = 0;
for (std::set< vtkIdType >::iterator itr = pointsInHalo.begin();
itr != pointsInHalo.end(); ++itr)
{
filteredData[i++] = data[*itr];
}
}
}
//------------------------------------------------------------------------------
void vtkPGenericIOReader::LoadData(vtkUnstructuredGrid *grid)
void vtkPGenericIOReader::LoadData(vtkUnstructuredGrid *grid,
const std::set< vtkIdType >& pointsInSelectedHalos)
{
assert("pre: grid is NULL!" && (grid != NULL) );
......@@ -738,21 +760,33 @@ void vtkPGenericIOReader::LoadData(vtkUnstructuredGrid *grid)
if( this->PointDataArraySelection->ArrayIsEnabled(name) )
{
std::string varName = std::string( name );
vtkDataArray *dataArray =
vtkSmartPointer< vtkDataArray > dataArray;
dataArray.TakeReference(
vtkGenericIOUtilities::GetVtkDataArray(
varName,
this->MetaData->VariableGenericIOType[ varName ],
this->MetaData->RawCache[ varName ],
this->MetaData->NumberOfElements
);
));
if (this->HaloList->GetNumberOfIds() != 0)
{
vtkSmartPointer< vtkDataArray > onlyDataInHalo;
onlyDataInHalo.TakeReference(dataArray->NewInstance());
onlyDataInHalo->SetNumberOfTuples(grid->GetNumberOfPoints());
onlyDataInHalo->SetName(dataArray->GetName());
switch (dataArray->GetDataType()) {
vtkTemplateMacro(GetOnlyDataInHalo<VTK_TT>(
dataArray,onlyDataInHalo,pointsInSelectedHalos));
}
dataArray = onlyDataInHalo;
}
PD->AddArray( dataArray );
dataArray->Delete();
} // END if the array is enabled
} // END for all arrays
if (this->AppendBlockCoordinates && this->Reader->IsSpatiallyDecomposed())
{
vtkTypeUInt64Array* dataArray = vtkTypeUInt64Array::New();
vtkSmartPointer< vtkTypeUInt64Array > dataArray = vtkSmartPointer< vtkTypeUInt64Array >::New();
dataArray->SetNumberOfComponents(3);
dataArray->SetNumberOfTuples(this->MetaData->NumberOfElements);
dataArray->SetName("gio_block_indices");
......@@ -771,8 +805,25 @@ void vtkPGenericIOReader::LoadData(vtkUnstructuredGrid *grid)
}
dataArray->SetTupleValue(i,coords);
}
if (this->HaloList->GetNumberOfIds() != 0)
{
vtkSmartPointer< vtkTypeUInt64Array > onlyDataInHalo;
onlyDataInHalo.TakeReference(dataArray->NewInstance());
onlyDataInHalo->SetNumberOfComponents(3);
onlyDataInHalo->SetNumberOfTuples(grid->GetNumberOfPoints());
onlyDataInHalo->SetName(dataArray->GetName());
vtkIdType i = 0;
for (std::set< vtkIdType >::iterator itr = pointsInSelectedHalos.begin();
itr != pointsInSelectedHalos.end(); ++itr, ++i)
{
vtkTypeUInt64 data[3];
dataArray->GetTupleValue(*itr,data);
onlyDataInHalo->SetTupleValue(i,data);
}
dataArray = onlyDataInHalo;
}
PD->AddArray( dataArray );
dataArray->Delete();
}
}
......@@ -905,16 +956,17 @@ int vtkPGenericIOReader::RequestData(
vtkUnstructuredGrid::SafeDownCast(
outInfo->Get(vtkDataObject::DATA_OBJECT()));
assert("pre: output grid is NULL!" && (output != NULL) );
std::set< vtkIdType > pointsInSelectedHalos;
// STEP 1: Load raw data
this->LoadRawData();
// STEP 2: Load coordinates
this->LoadCoordinates(output);
this->LoadCoordinates(output,pointsInSelectedHalos);
MPI_Barrier(this->MetaData->MPICommunicator);
// STEP 3: Load data
this->LoadData(output);
this->LoadData(output,pointsInSelectedHalos);
MPI_Barrier(this->MetaData->MPICommunicator);
// STEP 4: Clear variables
......
......@@ -24,6 +24,8 @@
#include "vtkUnstructuredGridAlgorithm.h" // Base class
#include "vtkPVVTKExtensionsCosmoToolsModule.h" // For export macro
#include <set> // for std::set in protected methods