Commit c6c9fee5 authored by Utkarsh Ayachit's avatar Utkarsh Ayachit
Browse files

Parallel3D app is now functional again.

parent 062ffbe8
......@@ -7,12 +7,16 @@ set (sources
if(ENABLE_SENSEI)
list(APPEND sources
vtk_histogram.cxx
vtk_histogram.h)
vtk_histogram.h
HistogramAnalysisAdaptor.cxx
HistogramAnalysisAdaptor.h
)
endif()
#-----------------------------------------------------------------------
add_library(histogram STATIC ${sources})
target_compile_definitions(histogram INTERFACE ENABLE_HISTOGRAM)
target_include_directories(histogram INTERFACE ${CMAKE_CURRENT_SOURCE_DIR})
include(mpi)
mpi_link(histogram)
......
#include "HistogramAnalysisAdaptor.h"
#include "vtkDataArray.h"
#include "vtkDataObject.h"
#include "vtkInsituDataAdaptor.h"
#include "vtkObjectFactory.h"
#include "vtk_histogram.h"
vtkStandardNewMacro(HistogramAnalysisAdaptor);
//-----------------------------------------------------------------------------
HistogramAnalysisAdaptor::HistogramAnalysisAdaptor() :
Communicator(MPI_COMM_WORLD),
Bins(0),
Association(vtkDataObject::FIELD_ASSOCIATION_POINTS)
{
}
//-----------------------------------------------------------------------------
HistogramAnalysisAdaptor::~HistogramAnalysisAdaptor()
{
}
//-----------------------------------------------------------------------------
void HistogramAnalysisAdaptor::Initialize(
MPI_Comm comm, int bins, int association, const std::string& arrayname)
{
this->Communicator = comm;
this->Bins = bins;
this->ArrayName = arrayname;
this->Association = association;
}
//-----------------------------------------------------------------------------
bool HistogramAnalysisAdaptor::Execute(vtkInsituDataAdaptor* data)
{
vtk_histogram(this->Communicator, vtkDataArray::SafeDownCast(
data->GetArray(this->Association, this->ArrayName.c_str())), this->Bins);
return true;
}
#ifndef HISTOGRAMANALYSISADAPTOR_H
#define HISTOGRAMANALYSISADAPTOR_H
#include "vtkInsituAnalysisAdaptor.h"
#include <mpi.h>
/// HistogramAnalysisAdaptor is a vtkInsituAnalysisAdaptor specialization for
/// histogram analysis.
class HistogramAnalysisAdaptor : public vtkInsituAnalysisAdaptor
{
public:
static HistogramAnalysisAdaptor* New();
vtkTypeMacro(HistogramAnalysisAdaptor, vtkInsituAnalysisAdaptor);
void Initialize(MPI_Comm comm, int bins,
int association, const std::string& arrayname);
virtual bool Execute(vtkInsituDataAdaptor* data);
protected:
HistogramAnalysisAdaptor();
virtual ~HistogramAnalysisAdaptor();
MPI_Comm Communicator;
int Bins;
std::string ArrayName;
int Association;
private:
HistogramAnalysisAdaptor(const HistogramAnalysisAdaptor&);
void operator=(const HistogramAnalysisAdaptor&);
};
#endif
......@@ -51,7 +51,7 @@ struct HistogramWorker
} // end anon namespace
void histogram(MPI_Comm comm, vtkDataArray* array, int bins)
void vtk_histogram(MPI_Comm comm, vtkDataArray* array, int bins)
{
int rank;
MPI_Comm_rank(comm, &rank);
......@@ -82,7 +82,7 @@ void histogram(MPI_Comm comm, vtkDataArray* array, int bins)
MPI_Reduce(&worker.Histogram[0], &gHist[0], bins, MPI_UNSIGNED, MPI_SUM, 0, comm);
if (rank == 0)
{
std::cout << "Histogram:\n";
std::cout << "Histogram (VTK):\n";
double width = (g_range[1] - g_range[0]) / bins;
for (int i = 0; i < bins; ++i)
{
......
#include "Bridge.h"
#include "DataAdaptor.h"
#include "vtkInsituAnalysisAdaptor.h"
#include "vtkNew.h"
#include "vtkDataObject.h"
#ifdef ENABLE_HISTOGRAM
#include "HistogramAnalysisAdaptor.h"
#endif
#include <vector>
namespace BridgeInternals
{
static vtkSmartPointer<parallel3d::DataAdaptor> GlobalDataAdaptor;
typedef std::vector<vtkSmartPointer<vtkInsituAnalysisAdaptor> > GlobalAnalysesType;
static GlobalAnalysesType GlobalAnalyses;
bool ExecuteGlobalAnalyses()
{
for (GlobalAnalysesType::const_iterator iter = GlobalAnalyses.begin();
iter != GlobalAnalyses.end(); ++iter)
{
if (!iter->GetPointer()->Execute(GlobalDataAdaptor.GetPointer()))
{
return false;
}
}
return true;
}
}
//-----------------------------------------------------------------------------
void bridge_initialize(MPI_Comm comm,
int g_x, int g_y, int g_z,
int l_x, int l_y, int l_z,
uint64_t start_extents_x, uint64_t start_extents_y, uint64_t start_extents_z,
int tot_blocks_x, int tot_blocks_y, int tot_blocks_z,
int block_id_x, int block_id_y, int block_id_z,
int bins)
{
if (!BridgeInternals::GlobalDataAdaptor)
{
BridgeInternals::GlobalDataAdaptor = vtkSmartPointer<parallel3d::DataAdaptor>::New();
}
BridgeInternals::GlobalDataAdaptor->Initialize(
g_x, g_y, g_z,
l_x, l_y, l_z,
start_extents_x, start_extents_y, start_extents_z,
tot_blocks_x, tot_blocks_y, tot_blocks_z,
block_id_x, block_id_y, block_id_z);
#ifdef ENABLE_HISTOGRAM
vtkNew<HistogramAnalysisAdaptor> histogram[3];
histogram[0]->Initialize(comm, bins, vtkDataObject::FIELD_ASSOCIATION_POINTS, "pressure");
histogram[1]->Initialize(comm, bins, vtkDataObject::FIELD_ASSOCIATION_POINTS, "temperature");
histogram[2]->Initialize(comm, bins, vtkDataObject::FIELD_ASSOCIATION_POINTS, "density");
BridgeInternals::GlobalAnalyses.push_back(histogram[0].GetPointer());
BridgeInternals::GlobalAnalyses.push_back(histogram[1].GetPointer());
BridgeInternals::GlobalAnalyses.push_back(histogram[2].GetPointer());
#endif
}
//-----------------------------------------------------------------------------
void bridge_update(double *pressure, double* temperature, double* density)
{
BridgeInternals::GlobalDataAdaptor->AddArray("pressure", pressure);
BridgeInternals::GlobalDataAdaptor->AddArray("temperature", temperature);
BridgeInternals::GlobalDataAdaptor->AddArray("density", density);
BridgeInternals::ExecuteGlobalAnalyses();
BridgeInternals::GlobalDataAdaptor->ReleaseData();
}
//-----------------------------------------------------------------------------
void bridge_finalize()
{
BridgeInternals::GlobalAnalyses.clear();
BridgeInternals::GlobalDataAdaptor = NULL;
}
#ifndef INSITU_BRIDGE_H
#define INSITU_BRIDGE_H
#ifndef PARALLEL3D_BRIDGE_H
#define PARALLEL3D_BRIDGE_H
#include <mpi.h>
#include <stdint.h>
......@@ -7,8 +7,10 @@
#ifdef __cplusplus
extern "C" {
#endif
/// This defines the analysis bridge for parallel_3d miniapp.
void insitu_bridge_initialize(MPI_Comm comm,
/// Called before simulation loop
void bridge_initialize(MPI_Comm comm,
int g_x, int g_y, int g_z,
int l_x, int l_y, int l_z,
uint64_t start_extents_x, uint64_t start_extents_y, uint64_t start_extents_z,
......@@ -16,12 +18,15 @@ extern "C" {
int block_id_x, int block_id_y, int block_id_z,
int bins);
void insitu_bridge_update(double *pressure, double* temperature, double* density);
/// Called per timestep in the simulation loop
void bridge_update(double *pressure, double* temperature, double* density);
void insitu_bridge_finalize();
/// Called just before simulation terminates.
void bridge_finalize();
#ifdef __cplusplus
} // extern "C"
#endif
#endif
......@@ -5,7 +5,9 @@ set(sources
if(ENABLE_SENSEI)
list(APPEND sources
DataAdaptor.h
DataAdaptor.cxx)
DataAdaptor.cxx
Bridge.h
Bridge.cxx)
endif()
#------------------------------------------------------------------------------
......@@ -19,3 +21,6 @@ mpi_link(3D_Grid)
if(ENABLE_SENSEI)
target_link_libraries(3D_Grid PRIVATE core)
endif()
if(ENABLE_HISTOGRAM)
target_link_libraries(3D_Grid PRIVATE histogram)
endif()
#include "insitu_bridge.h"
#include "infrastructure/vtkInsituAnalysisAdaptor.h"
#include "infrastructure/vtkInsituDataAdaptor.h"
#include "vtkDataObject.h"
#include "vtkDoubleArray.h"
#include "vtkObjectFactory.h"
#include "vtkSmartPointer.h"
#ifndef USE_GENERIC_ARRAYS_API
#include "histogram.h"
#else
#include "histogram_generic_arrays.h"
#endif
// Each simulation code in its bridge, implements a subclass of
// vtkInsituDataAdaptor to map the simulation datastructures to VTK data model.
namespace BridgeInternals
{
class vtk3DGridInsituDataAdaptor : public vtkInsituDataAdaptor
{
public:
static vtk3DGridInsituDataAdaptor* New();
vtkTypeMacro(vtk3DGridInsituDataAdaptor, vtkInsituDataAdaptor);
bool Initialize(
int g_x, int g_y, int g_z,
int l_x, int l_y, int l_z,
uint64_t start_extents_x, uint64_t start_extents_y, uint64_t start_extents_z,
int tot_blocks_x, int tot_blocks_y, int tot_blocks_z,
int block_id_x, int block_id_y, int block_id_z)
{
// we only really need to save the local extents for our current example. So
// we'll just save that.
this->Extent[0] = start_extents_x;
this->Extent[1] = start_extents_x + l_x - 1;
this->Extent[2] = start_extents_y;
this->Extent[3] = start_extents_y + l_y - 1;
this->Extent[4] = start_extents_z;
this->Extent[5] = start_extents_z + l_z - 1;
}
void SetArrays(double* pressure, double *temperature, double* density)
{
this->PressurePtr = pressure;
this->TemperaturePtr = temperature;
this->DensityPtr = density;
}
virtual vtkDataObject* GetMesh()
{
// our analysis doesn't need a mesh so we punt on it for now.
// In theory, we'll create a new vtkImageData and return that.
vtkGenericWarningMacro("TODO: Not implemented currently.");
return NULL;
}
// Description:
// Subclasses should override this method to provide field arrays.
virtual vtkAbstractArray* GetArray(int association, const char* name)
{
if (association != vtkDataObject::FIELD_ASSOCIATION_POINTS)
{
return NULL;
}
vtkIdType size = (this->Extent[1] - this->Extent[0] + 1) *
(this->Extent[3] - this->Extent[2] + 1) *
(this->Extent[5] - this->Extent[4] + 1);
if (strcmp(name, "pressure") == 0)
{
if (this->Pressure == NULL)
{
this->Pressure = vtkSmartPointer<vtkDoubleArray>::New();
this->Pressure->SetName("pressure");
this->Pressure->SetArray(this->PressurePtr, size, 1);
}
return this->Pressure;
}
else if (strcmp(name, "temperature") == 0)
{
if (this->Temperature == NULL)
{
this->Temperature = vtkSmartPointer<vtkDoubleArray>::New();
this->Temperature->SetName("temperature");
this->Temperature->SetArray(this->TemperaturePtr, size, 1);
}
return this->Temperature;
}
else if (strcmp(name, "density") == 0)
{
if (this->Density == NULL)
{
this->Density = vtkSmartPointer<vtkDoubleArray>::New();
this->Density->SetName("density");
this->Density->SetArray(this->DensityPtr, size, 1);
}
return this->Density;
}
return NULL;
}
virtual unsigned int GetNumberOfArrays(int association)
{
if (association != vtkDataObject::FIELD_ASSOCIATION_POINTS)
{
return 0;
}
return 3;
}
virtual const char* GetArrayName(int association, unsigned int index)
{
if (association != vtkDataObject::FIELD_ASSOCIATION_POINTS)
{
return NULL;
}
switch (index)
{
case 0: return "pressure";
case 1: return "temperature";
case 2: return "density";
default: return NULL;
}
}
// Description:
// Method called to release data and end of each execution iteration.
virtual void ReleaseData()
{
this->PressurePtr = NULL;
this->TemperaturePtr = NULL;
this->DensityPtr = NULL;
this->Pressure = NULL;
this->Temperature = NULL;
this->Density = NULL;
}
protected:
vtk3DGridInsituDataAdaptor()
{
this->Extent[0] = this->Extent[2] = this->Extent[4] = -1;
this->Extent[1] = this->Extent[3] = this->Extent[5] = 0;
this->PressurePtr = NULL;
this->TemperaturePtr = NULL;
this->DensityPtr = NULL;
}
~vtk3DGridInsituDataAdaptor()
{
}
int Extent[6];
double* PressurePtr;
double* TemperaturePtr;
double* DensityPtr;
vtkSmartPointer<vtkDoubleArray> Pressure;
vtkSmartPointer<vtkDoubleArray> Temperature;
vtkSmartPointer<vtkDoubleArray> Density;
private:
vtk3DGridInsituDataAdaptor(const vtk3DGridInsituDataAdaptor&);
void operator=(const vtk3DGridInsituDataAdaptor);
};
vtkStandardNewMacro(vtk3DGridInsituDataAdaptor);
class vtkHistogramAnalysisAdaptor : public vtkInsituAnalysisAdaptor
{
public:
static vtkHistogramAnalysisAdaptor* New();
vtkTypeMacro(vtkHistogramAnalysisAdaptor, vtkInsituAnalysisAdaptor);
void Initialize(MPI_Comm comm, int bins)
{
this->Communicator = comm;
this->Bins = bins;
}
virtual bool Execute(vtkInsituDataAdaptor* data)
{
#ifndef USE_GENERIC_ARRAYS_API
// downcase to vtkDoubleArray and call GetPointer() on it.
if (vtkDoubleArray* array = vtkDoubleArray::SafeDownCast(
data->GetArray(vtkDataObject::FIELD_ASSOCIATION_POINTS, "pressure")))
{
histogram(this->Communicator,
array->GetPointer(0), array->GetNumberOfTuples(), this->Bins);
}
if (vtkDoubleArray* array = vtkDoubleArray::SafeDownCast(
data->GetArray(vtkDataObject::FIELD_ASSOCIATION_POINTS, "temperature")))
{
histogram(this->Communicator,
array->GetPointer(0), array->GetNumberOfTuples(), this->Bins);
}
if (vtkDoubleArray* array = vtkDoubleArray::SafeDownCast(
data->GetArray(vtkDataObject::FIELD_ASSOCIATION_POINTS, "density")))
{
histogram(this->Communicator,
array->GetPointer(0), array->GetNumberOfTuples(), this->Bins);
}
#else
histogram(this->Communicator, vtkDataArray::SafeDownCast(
data->GetArray(vtkDataObject::FIELD_ASSOCIATION_POINTS, "pressure")), this->Bins);
histogram(this->Communicator, vtkDataArray::SafeDownCast(
data->GetArray(vtkDataObject::FIELD_ASSOCIATION_POINTS, "temperature")), this->Bins);
histogram(this->Communicator, vtkDataArray::SafeDownCast(
data->GetArray(vtkDataObject::FIELD_ASSOCIATION_POINTS, "density")), this->Bins);
#endif
return true;
}
protected:
vtkHistogramAnalysisAdaptor()
: Communicator(MPI_COMM_WORLD), Bins(0)
{
}
~vtkHistogramAnalysisAdaptor()
{
}
MPI_Comm Communicator;
int Bins;
private:
vtkHistogramAnalysisAdaptor(const vtkHistogramAnalysisAdaptor&);
void operator=(const vtkHistogramAnalysisAdaptor&);
};
vtkStandardNewMacro(vtkHistogramAnalysisAdaptor);
static vtkSmartPointer<vtk3DGridInsituDataAdaptor> DataAdaptor;
static vtkSmartPointer<vtkHistogramAnalysisAdaptor> Analysis;
}
//-----------------------------------------------------------------------------
void insitu_bridge_initialize(MPI_Comm comm,
int g_x, int g_y, int g_z,
int l_x, int l_y, int l_z,
uint64_t start_extents_x, uint64_t start_extents_y, uint64_t start_extents_z,
int tot_blocks_x, int tot_blocks_y, int tot_blocks_z,
int block_id_x, int block_id_y, int block_id_z,
int bins)
{
if (!BridgeInternals::DataAdaptor)
{
BridgeInternals::DataAdaptor =
vtkSmartPointer<BridgeInternals::vtk3DGridInsituDataAdaptor>::New();
}
BridgeInternals::DataAdaptor->Initialize(
g_x, g_y, g_z,
l_x, l_y, l_z,
start_extents_x, start_extents_y, start_extents_z,
tot_blocks_x, tot_blocks_y, tot_blocks_z,
block_id_x, block_id_y, block_id_z);
if (!BridgeInternals::Analysis)
{
// TODO: based on which insitu infrastructure we want to use for the
// analysis, we will instantiate the appropriate vtkHistogramAnalysisAdaptor
// subclass.
BridgeInternals::Analysis =
vtkSmartPointer<BridgeInternals::vtkHistogramAnalysisAdaptor>::New();
}
BridgeInternals::Analysis->Initialize(comm, bins);
}
//-----------------------------------------------------------------------------
void insitu_bridge_update(double *pressure, double* temperature, double* density)
{
BridgeInternals::DataAdaptor->SetArrays(pressure, temperature, density);
BridgeInternals::Analysis->Execute(BridgeInternals::DataAdaptor);
BridgeInternals::DataAdaptor->ReleaseData();
}
//-----------------------------------------------------------------------------
void insitu_bridge_finalize()
{
BridgeInternals::Analysis = NULL;
BridgeInternals::DataAdaptor = NULL;
}
......@@ -14,6 +14,9 @@
#include <errno.h>
#include <math.h>
#include <mpi.h>
#ifdef ENABLE_SENSEI
# include "Bridge.h"
#endif
int dim = 3;
......@@ -133,6 +136,16 @@ int main(int argc, char **argv)
printf("Total Blocks are %dX%dX%d \n", tot_blocks_z, tot_blocks_y, tot_blocks_x);
}
#ifdef ENABLE_SENSEI
bridge_initialize(MPI_COMM_WORLD,
g_x, g_y, g_z,
l_x, l_y, l_z,
start_extents_x, start_extents_y, start_extents_z,
tot_blocks_x, tot_blocks_y, tot_blocks_z,
block_id_x, block_id_y, block_id_z,
bins);
#endif
//////////////////////////////////
// allocate the variables and
// intialize to a pattern
......@@ -197,17 +210,31 @@ int main(int argc, char **argv)
// Compute several analyses?
/////////////////////////////
#ifdef ENABLE_SENSEI
bridge_update(pressure, temperature, density);
#else
// "Analysis" routine
histogram(MPI_COMM_WORLD, pressure, l_z*l_y*l_x, bins);
histogram(MPI_COMM_WORLD, temperature, l_z*l_y*l_x, bins);
histogram(MPI_COMM_WORLD, density, l_z*l_y*l_x, bins);
#endif
MPI_Barrier(MPI_COMM_WORLD);
/////////////////////////////
// Clean up heap variables
/////////////////////////////
#ifdef ENABLE_SENSEI
bridge_finalize();
#endif
if (pressure){
free(pressure);
pressure = 0;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment