WIP: In transit data adaptor

mr for collab on sensei 3 in transit api @srizzi @wolf.m7 @lot @jgu @ewbethel

Example 1, an analysis that wants the data to land a certain way

The FFT needs data to land a certain way. In this case column based.

In this case row based

ConfigurableEndPoint

int main(int argc, char **argv)
{
  string xml_config = argv[1];
  string stream_id = argv[2];

  // create instances of adaptors
  ConfigurableAnalysis *aa = nullptr;
  DataAdaptor *da = nullptr;
  InTransitAdaptorFactory::Initialize(xml_config, aa, da);
  
  // open the stream
  da->OpenStream(stream_id);
  
  // process time steps
  int ok = adaptors.size();
  while (da->StreamGood())
  {
    // run the analyses
    aa->Execute(adaptor.first);

    // move to the next time step
    da->AdvanceStream();
  }

  // close the stream
  da->CloseStream();

  return 0;
}

FFTAnalysis

class FFTAnalysis : public sensei::AnalysisAdaptor
{
  int Initialize(const pugi::xml_node &node);

  bool Execute(sensei::DataAdaptor *da);

  string MeshName;
  string ArrayName;
  int RowBased;
};

bool FFTAnalysis::Execute(sensei::DataAdaptor *da)
{
  // if we are in transit set the desired view
  if (sensei::InTransitDataAdaptor *itda = dynamic_cast<sensei::InTransitDataAdaptor*>(da))
  {
    sensei::MeshMetadataPtr remote_mmd;
    itda->GetSenderMeshMetadata(this->MeshName, remote_mmd);

    // set up the desired block to process mapping
    // for how the data lands
    sensei::MeshMetadataPtr local_mmd;
    this->ComputeBlockLayout(remote_mmd, local_mmd);

    // tell the data adaptor what we need
    itda->SetReceiverMeshMetadata(this->MeshName, local_mmd);
  }

  // from here on, in transit and in situ are identical
  vtkDataObject *mesh;
  da->GetMesh(this->MeshName, false, mesh);
  da->AddArray(this->MeshName, this->ArrayName, this->ArrayCen);
  
  // process data
  // ...

  return 0;
}

int FFTAnalysis::ComputeBlockLayout(sensei::MeshMetadataPtr remote, sensei::MeshMetadataPtr &local)
{
   // analyze the remote BlockOwner & BlockIds
   // set the local BlockOwner & BlockIds such that desired blocks land on desired ranks

   // in this example...
   // remote->BlockOwner is {0, 1, ..., 8}
   // remote->BlockIds is {0, 1, ... , 8}
   
   if (this->RowBased)
   {
     // in this example...
     // local->BlockOwner = {0, 0, 0, 1, 1, 1, 2, 2, 2};
     // local->BlockIds = {0, 3, 6, 1, 4, 7, 2, 5, 8};
   }
   else
   {
     // in this example...
     // local->BlockOwner = {0, 0, 0, 1, 1, 1, 2, 2, 2};
     // local->BlockIds = {0, 1, 2, 3, 4, 5, 6, 7, 8};
   }

  return 0;
}

Histogram

This histogram does not care how blocks are distributed, but the user may want to say how they do.

All sensei::InTransitDataAdaptors support XML attribute 'partitioner' and methods 'block', 'cyclic', 'plane', 'mapped'.

Here the user has specified "block" in the XML file passed into the ConfigurableEndPoint.

Here the user has specified "cyclic" in the XML file

InTransitDataAdaptor partitioner defaults

int InTransitDataAdaptor::GetCyclicPartition(sensei::MeshMetadataPtr &remote, sensei::MeshMetadataPtr &local)
{

   // implement the "cyclic" partitioner decomposition

   // in this example...
   // remote->BlockOwner is {0, 1, ..., 8}
   // remote->BlockIds is {0, 1, ... , 8}
   
   // in this example...
   // local->BlockOwner = {0, 1, 0, 1, 0, 1, 0, 1, 0};
   // local->BlockIds = {0, 1, ... , 8}
}

int InTransitDataAdaptor::GetBlockPartition(sensei::MeshMetadataPtr &remote, sensei::MeshMetadataPtr &local)
{

   // implement the "block" partitioner decomposition

   // in this example...
   // remote->BlockOwner is {0, 1, ..., 8}
   // remote->BlockIds is {0, 1, ... , 8}
   
   // in this example...
   // local->BlockOwner = {0, 0, 0, 0, 0, 1, 1, 1, 1};
   // local->BlockIds = {0, 1, ... , 8}
}

int InTransitDataAdaptor::GetPlanePartition(sensei::MeshMetadataPtr &remote, sensei::MeshMetadataPtr &local)
{
    // details omitted
}

int InTransitDataAdaptor::GetMappedPartition(sensei::MeshMetadataPtr &remote, sensei::MeshMetadataPtr &local)
{
    // user passes the block to process map via XML
}