Commit e64416c6 authored by Ben Boeckel's avatar Ben Boeckel
Browse files

vtkAMRUtilities: split out vtkParallelAMRUtilities

This is split so that vtkIOXML loses its dependency on vtkFiltersAMR
which drags in vtkParallelCore. The parallel-related code can instead be
split out and the core utilities be moved into Common/DataModel.

Change-Id: I621cc40f2afb1831d25e0701a0075ad88dc3e149
parent ecf76248
......@@ -3,6 +3,7 @@ set(Module_SRCS
vtkAbstractPointLocator.cxx
vtkAdjacentVertexIterator.cxx
vtkAMRBox.cxx
vtkAMRUtilities.cxx
vtkAnimationScene.cxx
vtkAnnotation.cxx
vtkAnnotationLayers.cxx
......
......@@ -16,19 +16,14 @@
#include "vtkAMRInformation.h"
#include "vtkAMRUtilities.h"
#include "vtkCellData.h"
#include "vtkCommunicator.h"
#include "vtkCompositeDataIterator.h"
#include "vtkDataArray.h"
#include "vtkFieldData.h"
#include "vtkMath.h"
#include "vtkMultiProcessController.h"
#include "vtkOverlappingAMR.h"
#include "vtkPointData.h"
#include "vtkStructuredData.h"
#include "vtkUniformGrid.h"
#include "vtkAMRInformation.h"
#include "vtkUnsignedCharArray.h"
#include "vtkSmartPointer.h"
#include "vtkCompositeDataIterator.h"
#include <cmath>
#include <limits>
#include <cassert>
......@@ -39,155 +34,13 @@
#define JMAX(ext) ext[3]
#define KMIN(ext) ext[4]
#define KMAX(ext) ext[5]
#define PRINT(x) cout<<"("<<myRank<<")"<<x<<endl;
namespace
{
void BlankGridsAtLevel(vtkOverlappingAMR* amr, int levelIdx, std::vector<std::vector<unsigned int> >& children,
const std::vector<int>& processMap)
{
unsigned int numDataSets = amr->GetNumberOfDataSets(levelIdx);
int N;
for( unsigned int dataSetIdx=0; dataSetIdx<numDataSets; dataSetIdx++)
{
const vtkAMRBox& box = amr->GetAMRBox(levelIdx, dataSetIdx);
vtkUniformGrid* grid = amr->GetDataSet(levelIdx, dataSetIdx);
if (grid == NULL )
{
continue;
}
N = grid->GetNumberOfCells();
vtkUnsignedCharArray* vis = vtkUnsignedCharArray::New();
vis->SetName("visibility");
vis->SetNumberOfTuples( N );
vis->FillComponent(0,static_cast<char>(1));
grid->SetCellVisibilityArray(vis);
vis->Delete();
if (children.size() <= dataSetIdx)
continue;
std::vector<unsigned int>& dsChildren = children[dataSetIdx];
std::vector<unsigned int>::iterator iter;
// For each higher res box fill in the cells that
// it covers
for (iter=dsChildren.begin(); iter!=dsChildren.end(); iter++)
{
vtkAMRBox ibox;;
int childGridIndex = amr->GetCompositeIndex(levelIdx+1, *iter);
if(processMap[childGridIndex]<0)
{
continue;
}
if (amr->GetAMRInfo()->GetCoarsenedAMRBox(levelIdx+1, *iter, ibox))
{
ibox.Intersect(box);
const int *loCorner=ibox.GetLoCorner();
int hi[3];
ibox.GetValidHiCorner(hi);
for( int iz=loCorner[2]; iz<=hi[2]; iz++ )
{
for( int iy=loCorner[1]; iy<=hi[1]; iy++ )
{
for( int ix=loCorner[0]; ix<=hi[0]; ix++ )
{
vtkIdType id = vtkAMRBox::GetCellLinearIndex(box,ix, iy, iz, grid->GetDimensions());
vis->SetValue(id, 0);
} // END for x
} // END for y
} // END for z
}
} // Processing all higher boxes for a specific coarse grid
}
}
};
//------------------------------------------------------------------------------
void vtkAMRUtilities::PrintSelf( std::ostream& os, vtkIndent indent )
{
this->Superclass::PrintSelf( os, indent );
}
//------------------------------------------------------------------------------
void vtkAMRUtilities::DistributeProcessInformation(vtkOverlappingAMR* amr,
vtkMultiProcessController *controller,
std::vector<int>& processMap)
{
processMap.resize(amr->GetTotalNumberOfBlocks(),-1);
vtkSmartPointer<vtkCompositeDataIterator> iter;
iter.TakeReference(amr->NewIterator());
iter->SkipEmptyNodesOn();
if(!controller || controller->GetNumberOfProcesses()==1)
{
for(iter->GoToFirstItem(); !iter->IsDoneWithTraversal(); iter->GoToNextItem())
{
unsigned int index = iter->GetCurrentFlatIndex();
processMap[index] = 0;
}
return;
}
vtkAMRInformation* amrInfo = amr->GetAMRInfo();
int myRank = controller->GetLocalProcessId();
int numProcs = controller->GetNumberOfProcesses();
//get the active process ids
std::vector<int> myBlocks;
for(iter->GoToFirstItem(); !iter->IsDoneWithTraversal(); iter->GoToNextItem())
{
myBlocks.push_back(iter->GetCurrentFlatIndex());
}
vtkIdType myNumBlocks = myBlocks.size();
std::vector<vtkIdType> numBlocks(numProcs,0);
numBlocks[myRank] = myNumBlocks;
//gather the active process counts
controller->AllGather( &myNumBlocks, &numBlocks[0], 1);
//gather the blocks each process owns into one array
std::vector<vtkIdType> offsets(numProcs,0);
vtkIdType currentOffset(0);
for(int i=0; i<numProcs;i++)
{
offsets[i] = currentOffset;
currentOffset+=numBlocks[i];
}
PRINT("total # of active blocks: "<<currentOffset<<" out of total "<<amrInfo->GetTotalNumberOfBlocks());
std::vector<int> allBlocks(currentOffset,-1);
controller->AllGatherV(&myBlocks[0], &allBlocks[0], (vtkIdType)myBlocks.size(), &numBlocks[0], &offsets[0] );
#ifdef DEBUG
if(myRank==0)
{
for(int i=0; i<numProcs; i++)
{
vtkIdType offset= offsets[i];
int n = numBlocks[i];
cout<<"Rank "<<i<<" has: ";
for(vtkIdType j=offset; j<offset+n; j++)
{
cout<<allBlocks[j]<<" ";
}
cout<<endl;
}
}
#endif
for(int rank=0; rank<numProcs; rank++)
{
int offset= offsets[rank];
int n = numBlocks[rank];
for(int j=offset; j<offset+n; j++)
{
int index = allBlocks[j];
assert(index>=0);
processMap[index] =rank;
}
}
}
//------------------------------------------------------------------------------
bool vtkAMRUtilities::HasPartiallyOverlappingGhostCells(
vtkOverlappingAMR *amr)
......@@ -438,8 +291,7 @@ vtkUniformGrid* vtkAMRUtilities::StripGhostLayersFromGrid(
//------------------------------------------------------------------------------
void vtkAMRUtilities::StripGhostLayers(
vtkOverlappingAMR *ghostedAMRData,
vtkOverlappingAMR *strippedAMRData,
vtkMultiProcessController *controller)
vtkOverlappingAMR *strippedAMRData)
{
assert("pre: input AMR data is NULL" && (ghostedAMRData != NULL) );
assert("pre: outputAMR data is NULL" && (strippedAMRData != NULL) );
......@@ -504,30 +356,99 @@ void vtkAMRUtilities::StripGhostLayers(
}
} // END for all data at the given level
} // END for all levels
if( controller != NULL )
{
controller->Barrier();
}
}
void vtkAMRUtilities::BlankCells(vtkOverlappingAMR* amr, vtkMultiProcessController *myController)
//------------------------------------------------------------------------------
void vtkAMRUtilities::BlankCells(vtkOverlappingAMR* amr)
{
vtkAMRInformation* info = amr->GetAMRInfo();
if(!info->HasRefinementRatio())
{
info->GenerateRefinementRatio();
}
unsigned int numLevels =info->GetNumberOfLevels();
if(!info->HasChildrenInformation())
{
info->GenerateParentChildInformation();
}
std::vector<int> processorMap;
vtkAMRUtilities::DistributeProcessInformation(amr,myController,processorMap);
processorMap.resize(amr->GetTotalNumberOfBlocks(),-1);
vtkSmartPointer<vtkCompositeDataIterator> iter;
iter.TakeReference(amr->NewIterator());
iter->SkipEmptyNodesOn();
for(iter->GoToFirstItem(); !iter->IsDoneWithTraversal(); iter->GoToNextItem())
{
unsigned int index = iter->GetCurrentFlatIndex();
processorMap[index] = 0;
}
unsigned int numLevels =info->GetNumberOfLevels();
for(unsigned int i=0; i<numLevels; i++)
{
BlankGridsAtLevel(amr, i, info->GetChildrenAtLevel(i),processorMap);
}
}
//------------------------------------------------------------------------------
void vtkAMRUtilities::BlankGridsAtLevel(vtkOverlappingAMR* amr, int levelIdx,
std::vector<std::vector<unsigned int> >& children,
const std::vector<int>& processMap)
{
unsigned int numDataSets = amr->GetNumberOfDataSets(levelIdx);
int N;
for( unsigned int dataSetIdx=0; dataSetIdx<numDataSets; dataSetIdx++)
{
const vtkAMRBox& box = amr->GetAMRBox(levelIdx, dataSetIdx);
vtkUniformGrid* grid = amr->GetDataSet(levelIdx, dataSetIdx);
if (grid == NULL )
{
continue;
}
N = grid->GetNumberOfCells();
vtkUnsignedCharArray* vis = vtkUnsignedCharArray::New();
vis->SetName("visibility");
vis->SetNumberOfTuples( N );
vis->FillComponent(0,static_cast<char>(1));
grid->SetCellVisibilityArray(vis);
vis->Delete();
if (children.size() <= dataSetIdx)
continue;
std::vector<unsigned int>& dsChildren = children[dataSetIdx];
std::vector<unsigned int>::iterator iter;
// For each higher res box fill in the cells that
// it covers
for (iter=dsChildren.begin(); iter!=dsChildren.end(); iter++)
{
vtkAMRBox ibox;;
int childGridIndex = amr->GetCompositeIndex(levelIdx+1, *iter);
if(processMap[childGridIndex]<0)
{
continue;
}
if (amr->GetAMRInfo()->GetCoarsenedAMRBox(levelIdx+1, *iter, ibox))
{
ibox.Intersect(box);
const int *loCorner=ibox.GetLoCorner();
int hi[3];
ibox.GetValidHiCorner(hi);
for( int iz=loCorner[2]; iz<=hi[2]; iz++ )
{
for( int iy=loCorner[1]; iy<=hi[1]; iy++ )
{
for( int ix=loCorner[0]; ix<=hi[0]; ix++ )
{
vtkIdType id = vtkAMRBox::GetCellLinearIndex(box,ix, iy, iz, grid->GetDimensions());
vis->SetValue(id, 0);
} // END for x
} // END for y
} // END for z
}
} // Processing all higher boxes for a specific coarse grid
}
}
......@@ -12,7 +12,7 @@
PURPOSE. See the above copyright notice for more information.
=========================================================================*/
// .NAME vtkAMRUtilities -- Support for distributed/serial AMR operations
// .NAME vtkAMRUtilities -- Support for serial AMR operations
//
// .SECTION Description
// A concrete instance of vtkObject that employs a singleton design
......@@ -24,20 +24,16 @@
#ifndef VTKAMRUTILITIES_H_
#define VTKAMRUTILITIES_H_
#include "vtkFiltersAMRModule.h" // For export macro
#include "vtkCommonDataModelModule.h" // For export macro
#include "vtkObject.h"
#include <vector> // For C++ vector
// Forward declarations
class vtkAMRBox;
class vtkFieldData;
class vtkMultiProcessController;
class vtkOverlappingAMR;
class vtkUniformGridAMR;
class vtkUniformGrid;
class vtkAMRInformation;
class VTKFILTERSAMR_EXPORT vtkAMRUtilities : public vtkObject
class VTKCOMMONDATAMODEL_EXPORT vtkAMRUtilities : public vtkObject
{
public:
// Standard Routines
......@@ -54,8 +50,7 @@ public:
// 1) The ghosted AMR data must have complete metadata information.
static void StripGhostLayers(
vtkOverlappingAMR *ghostedAMRData,
vtkOverlappingAMR *strippedAMRData,
vtkMultiProcessController *myController=NULL);
vtkOverlappingAMR *strippedAMRData);
// Description:
// A quick test of whether partially overlapping ghost cells exist. This test
......@@ -64,13 +59,9 @@ public:
// cells are detected. Otherwise, false is returned.
static bool HasPartiallyOverlappingGhostCells(vtkOverlappingAMR *amr);
// Description:
// Compute map from block indices to process ids
static void DistributeProcessInformation(vtkOverlappingAMR* amr, vtkMultiProcessController *myController, std::vector<int>& ProcessMap);
// Description:
// Blank cells in overlapping AMR
static void BlankCells(vtkOverlappingAMR* amr, vtkMultiProcessController *myController);
static void BlankCells(vtkOverlappingAMR* amr);
protected:
vtkAMRUtilities() {}
......@@ -99,6 +90,9 @@ protected:
static vtkUniformGrid* StripGhostLayersFromGrid(
vtkUniformGrid* grid, int ghost[6]);
static void BlankGridsAtLevel(vtkOverlappingAMR* amr, int levelIdx,
std::vector<std::vector<unsigned int> >& children,
const std::vector<int>& processMap);
private:
vtkAMRUtilities(const vtkAMRUtilities&); // Not implemented
void operator=(const vtkAMRUtilities&); // Not implemented
......
......@@ -189,7 +189,7 @@ vtkOverlappingAMR* GetAMRDataSet()
data->SetDataSet( level, blockId,grid3);
grid3->Delete();
vtkAMRUtilities::BlankCells(data,NULL);
vtkAMRUtilities::BlankCells(data);
data->Audit();
return( data );
}
......@@ -193,6 +193,6 @@ vtkOverlappingAMR* GetAMRDataSet()
data->SetDataSet( level, blockId,grid3);
grid3->Delete();
vtkAMRUtilities::BlankCells(data,NULL);
vtkAMRUtilities::BlankCells(data);
return( data );
}
......@@ -4,8 +4,8 @@ set(Module_SRCS
vtkAMRResampleFilter.cxx
vtkAMRSliceFilter.cxx
vtkAMRToMultiBlockFilter.cxx
vtkAMRUtilities.cxx
vtkImageToAMR.cxx
vtkParallelAMRUtilities.cxx
)
set_source_files_properties(
......
......@@ -156,7 +156,7 @@ vtkOverlappingAMR* GetAMRDataSet(const int description)
assert(
"ERROR: Unhandled data description! Code should not reach here!" && false);
}
vtkAMRUtilities::BlankCells(amrDataSet,NULL);
vtkAMRUtilities::BlankCells(amrDataSet);
return( amrDataSet );
}
......
......@@ -353,6 +353,6 @@ int vtkAMRGaussianPulseSource::RequestData(
vtkErrorMacro("Dimensions must be either 2 or 3!");
}
vtkAMRUtilities::BlankCells(output,NULL);
vtkAMRUtilities::BlankCells(output);
return 1;
}
......@@ -21,7 +21,7 @@
#include "vtkInformationVector.h"
#include "vtkOverlappingAMR.h"
#include "vtkCell.h"
#include "vtkAMRUtilities.h"
#include "vtkParallelAMRUtilities.h"
#include "vtkPlane.h"
#include "vtkUniformGrid.h"
#include "vtkStructuredData.h"
......@@ -434,7 +434,7 @@ void vtkAMRSliceFilter::GetAMRSliceInPlane(
vtkTimerLog::MarkEndEvent( "AMRSlice::GetAMRSliceInPlane" );
vtkTimerLog::MarkStartEvent( "AMRSlice::Generate Blanking" );
vtkAMRUtilities::BlankCells(out, this->Controller);
vtkParallelAMRUtilities::BlankCells(out, this->Controller);
vtkTimerLog::MarkEndEvent( "AMRSlice::Generate Blanking" );
}
......
......@@ -346,7 +346,7 @@ int vtkImageToAMR::RequestData(vtkInformation* vtkNotUsed(request),
}
}
vtkAMRUtilities::BlankCells(amr,NULL);
vtkAMRUtilities::BlankCells(amr);
return 1;
}
......
/*=========================================================================
Program: Visualization Toolkit
Module: vtkAMRUtilities.cxx
Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
All rights reserved.
See Copyright.txt or http://www.kitware.com/Copyright.htm for details.
This software is distributed WITHOUT ANY WARRANTY; without even
the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE. See the above copyright notice for more information.
=========================================================================*/
#include "vtkAMRBox.h"
#include "vtkAMRInformation.h"
#include "vtkParallelAMRUtilities.h"
#include "vtkMultiProcessController.h"
#include "vtkOverlappingAMR.h"
#include "vtkUniformGrid.h"
#include "vtkSmartPointer.h"
#include "vtkCompositeDataIterator.h"
#include <cmath>
#include <limits>
#include <cassert>
#define PRINT(x) cout<<"("<<myRank<<")"<<x<<endl;
//------------------------------------------------------------------------------
void vtkParallelAMRUtilities::PrintSelf( std::ostream& os, vtkIndent indent )
{
this->Superclass::PrintSelf( os, indent );
}
//------------------------------------------------------------------------------
void vtkParallelAMRUtilities::DistributeProcessInformation(vtkOverlappingAMR* amr,
vtkMultiProcessController *controller,
std::vector<int>& processMap)
{
processMap.resize(amr->GetTotalNumberOfBlocks(),-1);
vtkSmartPointer<vtkCompositeDataIterator> iter;
iter.TakeReference(amr->NewIterator());
iter->SkipEmptyNodesOn();
if(!controller || controller->GetNumberOfProcesses()==1)
{
for(iter->GoToFirstItem(); !iter->IsDoneWithTraversal(); iter->GoToNextItem())
{
unsigned int index = iter->GetCurrentFlatIndex();
processMap[index] = 0;
}
return;
}
vtkAMRInformation* amrInfo = amr->GetAMRInfo();
int myRank = controller->GetLocalProcessId();
int numProcs = controller->GetNumberOfProcesses();
//get the active process ids
std::vector<int> myBlocks;
for(iter->GoToFirstItem(); !iter->IsDoneWithTraversal(); iter->GoToNextItem())
{
myBlocks.push_back(iter->GetCurrentFlatIndex());
}
vtkIdType myNumBlocks = myBlocks.size();
std::vector<vtkIdType> numBlocks(numProcs,0);
numBlocks[myRank] = myNumBlocks;
//gather the active process counts
controller->AllGather( &myNumBlocks, &numBlocks[0], 1);
//gather the blocks each process owns into one array
std::vector<vtkIdType> offsets(numProcs,0);
vtkIdType currentOffset(0);
for(int i=0; i<numProcs;i++)
{
offsets[i] = currentOffset;
currentOffset+=numBlocks[i];
}
PRINT("total # of active blocks: "<<currentOffset<<" out of total "<<amrInfo->GetTotalNumberOfBlocks());
std::vector<int> allBlocks(currentOffset,-1);
controller->AllGatherV(&myBlocks[0], &allBlocks[0], (vtkIdType)myBlocks.size(), &numBlocks[0], &offsets[0] );
#ifdef DEBUG
if(myRank==0)
{
for(int i=0; i<numProcs; i++)
{
vtkIdType offset= offsets[i];
int n = numBlocks[i];
cout<<"Rank "<<i<<" has: ";
for(vtkIdType j=offset; j<offset+n; j++)
{
cout<<allBlocks[j]<<" ";
}
cout<<endl;
}
}
#endif
for(int rank=0; rank<numProcs; rank++)
{
int offset= offsets[rank];
int n = numBlocks[rank];
for(int j=offset; j<offset+n; j++)
{
int index = allBlocks[j];
assert(index>=0);
processMap[index] =rank;
}
}
}
//------------------------------------------------------------------------------
void vtkParallelAMRUtilities::StripGhostLayers(
vtkOverlappingAMR *ghostedAMRData,
vtkOverlappingAMR *strippedAMRData,
vtkMultiProcessController *controller)
{
vtkAMRUtilities::StripGhostLayers(ghostedAMRData, strippedAMRData);
if( controller != NULL )
{
controller->Barrier();
}
}
//------------------------------------------------------------------------------
void vtkParallelAMRUtilities::BlankCells(vtkOverlappingAMR* amr, vtkMultiProcessController *myController)
{
vtkAMRInformation* info = amr->GetAMRInfo();
if(!info->HasRefinementRatio())
{
info->GenerateRefinementRatio();
}
if(!info->HasChildrenInformation())
{
info->GenerateParentChildInformation();
}
std::vector<int> processorMap;
vtkParallelAMRUtilities::DistributeProcessInformation(amr,myController,processorMap);
unsigned int numLevels =info->GetNumberOfLevels();
for(unsigned int i=0; i<numLevels; i++)
{
vtkAMRUtilities::BlankGridsAtLevel(amr, i, info->GetChildrenAtLevel(i),processorMap);
}
}
/*=========================================================================
Program: Visualization Toolkit
Module: vtkAMRUtilities.h
Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
All rights reserved.