vtkParallelAMRUtilities.cxx 4.56 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43
/*=========================================================================

 Program:   Visualization Toolkit
 Module:    vtkAMRUtilities.cxx

 Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
 All rights reserved.
 See Copyright.txt or http://www.kitware.com/Copyright.htm for details.

 This software is distributed WITHOUT ANY WARRANTY; without even
 the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
 PURPOSE.  See the above copyright notice for more information.

 =========================================================================*/
#include "vtkAMRBox.h"
#include "vtkAMRInformation.h"
#include "vtkParallelAMRUtilities.h"
#include "vtkMultiProcessController.h"
#include "vtkOverlappingAMR.h"
#include "vtkUniformGrid.h"
#include "vtkSmartPointer.h"
#include "vtkCompositeDataIterator.h"
#include <cmath>
#include <limits>
#include <cassert>

//------------------------------------------------------------------------------
void vtkParallelAMRUtilities::PrintSelf( std::ostream& os, vtkIndent indent )
{
  this->Superclass::PrintSelf( os, indent );
}

//------------------------------------------------------------------------------
void vtkParallelAMRUtilities::DistributeProcessInformation(vtkOverlappingAMR* amr,
                                                   vtkMultiProcessController *controller,
                                                   std::vector<int>& processMap)
{
  processMap.resize(amr->GetTotalNumberOfBlocks(),-1);
  vtkSmartPointer<vtkCompositeDataIterator> iter;
  iter.TakeReference(amr->NewIterator());
  iter->SkipEmptyNodesOn();

  if(!controller || controller->GetNumberOfProcesses()==1)
44
  {
45
    for(iter->GoToFirstItem(); !iter->IsDoneWithTraversal(); iter->GoToNextItem())
46
    {
47 48 49
      unsigned int index = iter->GetCurrentFlatIndex();
      processMap[index] = 0;
    }
50 51
    return;
  }
52 53 54 55 56 57 58
  vtkAMRInformation* amrInfo = amr->GetAMRInfo();
  int myRank = controller->GetLocalProcessId();
  int numProcs   = controller->GetNumberOfProcesses();

  //get the active process ids
  std::vector<int> myBlocks;
  for(iter->GoToFirstItem(); !iter->IsDoneWithTraversal(); iter->GoToNextItem())
59
  {
60
    myBlocks.push_back(iter->GetCurrentFlatIndex());
61
  }
62

63
  vtkIdType myNumBlocks = static_cast<vtkIdType>(myBlocks.size());
64 65 66 67 68 69 70 71 72 73
  std::vector<vtkIdType> numBlocks(numProcs,0);
  numBlocks[myRank] = myNumBlocks;

  //gather the active process counts
  controller->AllGather( &myNumBlocks, &numBlocks[0], 1);

  //gather the blocks each process owns into one array
  std::vector<vtkIdType> offsets(numProcs,0);
  vtkIdType currentOffset(0);
  for(int i=0; i<numProcs;i++)
74
  {
75 76
    offsets[i] = currentOffset;
    currentOffset+=numBlocks[i];
77
  }
78
  cout<<"("<<myRank<<")"<<"total # of active blocks: "<<currentOffset<<" out of total "<<amrInfo->GetTotalNumberOfBlocks()<<endl;
79 80 81 82 83
  std::vector<int> allBlocks(currentOffset,-1);
  controller->AllGatherV(&myBlocks[0], &allBlocks[0], (vtkIdType)myBlocks.size(), &numBlocks[0], &offsets[0] );

#ifdef DEBUG
  if(myRank==0)
84
  {
85
    for(int i=0; i<numProcs; i++)
86
    {
87 88 89 90
      vtkIdType offset= offsets[i];
      int n = numBlocks[i];
      cout<<"Rank "<<i<<" has: ";
      for(vtkIdType j=offset; j<offset+n; j++)
91
      {
92 93
        cout<<allBlocks[j]<<" ";
      }
94
      cout<<endl;
95
    }
96
  }
97 98
#endif
  for(int rank=0; rank<numProcs; rank++)
99
  {
100 101 102
    int offset= offsets[rank];
    int n = numBlocks[rank];
    for(int j=offset; j<offset+n; j++)
103
    {
104 105 106 107
      int index = allBlocks[j];
      assert(index>=0);
      processMap[index] =rank;
    }
108
  }
109 110 111 112 113 114 115 116 117 118
}

//------------------------------------------------------------------------------
void vtkParallelAMRUtilities::StripGhostLayers(
        vtkOverlappingAMR *ghostedAMRData,
        vtkOverlappingAMR *strippedAMRData,
        vtkMultiProcessController *controller)
{
  vtkAMRUtilities::StripGhostLayers(ghostedAMRData, strippedAMRData);

119
  if( controller != nullptr )
120
  {
121
    controller->Barrier();
122
  }
123 124 125 126 127 128 129
}

//------------------------------------------------------------------------------
void vtkParallelAMRUtilities::BlankCells(vtkOverlappingAMR* amr,  vtkMultiProcessController *myController)
{
  vtkAMRInformation* info = amr->GetAMRInfo();
  if(!info->HasRefinementRatio())
130
  {
131
    info->GenerateRefinementRatio();
132
  }
133
  if(!info->HasChildrenInformation())
134
  {
135
    info->GenerateParentChildInformation();
136
  }
137 138 139 140 141

  std::vector<int> processorMap;
  vtkParallelAMRUtilities::DistributeProcessInformation(amr,myController,processorMap);
  unsigned int numLevels =info->GetNumberOfLevels();
  for(unsigned int i=0; i<numLevels; i++)
142
  {
143
    vtkAMRUtilities::BlankGridsAtLevel(amr, i, info->GetChildrenAtLevel(i),processorMap);
144
  }
145
}