Commit 136c54ea authored by Iulian Grindeanu's avatar Iulian Grindeanu Committed by vijaysm
Browse files

Implement more API methods related to parallel queries. A simple version

of the driver has also been added.

- more global variables, that store file ids, pcomm, mapping between pid and STRING
entities are ghosted, but the material sets/ neumann sets are not filled yet
with ghost entities

- ghost addl entities too. Will pass 3 for addl entities when ghosting
it means that edges and faces will be ghosted too. It still does not
resolve the issue of populating the classic sets. So the Neumann sets
and material sets, after ghosting, will not contain any ghost entities.
Will need an extra step / call to solve this problem.
- Add support for deregistering the application

should delete the relevant entities
it is failing now to delete entities from the file set
1) addl_ents in ghosting are not added to the file set
2) not sure if skin entities are added either
parent dbce44c5
......@@ -18,14 +18,42 @@ copy it in this folder (imoab/src/mhdf) temporarily; after imoab is part of moab
#include <H5Tpublic.h>
#include <iostream>
#include "moab/ParallelComm.hpp"
#include "MBTagConventions.hpp"
#include <sstream>
// global variables (static?)
// global variables ; should they be organized in a structure, for easier references?
// or how do we keep them global?
Interface * MBI = 0;
// we should also have the default tags stored, initialized
Tag gtags[5]; // material, neumann, dirichlet, partition tag, globalID
// should this be part of init moab?
struct appData {
EntityHandle file_set;
Range all_verts;
Range primary_elems;
Range mat_sets;
Range neu_sets;
Range diri_sets;
};
// are there reasons to have multiple moab inits? Is ref count needed?
int refCountMB( 0) ;
int iArgc;
iMOAB_String * iArgv;
/*
list of moab entity sets corresponding to each application and pid
*/
int unused_pid =0;
// std::vector<EntityHandle> app_FileSets; // in order of creation
std::map<std::string, int> appIdMap; // from app string (uppercase) to app id
std::vector<ParallelComm*> pcomms; // created in order of applications, one moab::ParallelComm for each
std::vector<appData> appDatas; // the same order as pcomms
/**
\fn ErrorCode iMOABInitialize( int argc, iMOAB_String* argv )
\brief Initialize the iMOAB interface implementation and create the MOAB instance, if not created already (reference counted).
......@@ -40,19 +68,34 @@ ErrCode iMOABInitialize( int argc, iMOAB_String* argv )
iArgc = argc;
iArgv = argv; // shalow copy
if (0==refCountMB)
{
MBI = new Core();
// retrieve the default tags
const char* const shared_set_tag_names[] = {MATERIAL_SET_TAG_NAME,
NEUMANN_SET_TAG_NAME,
DIRICHLET_SET_TAG_NAME,
GLOBAL_ID_TAG_NAME};
// blocks, visible surfaceBC(neumann), vertexBC (Dirichlet), global id, parallel partition
for (int i = 0; i < 4; i++) {
ErrorCode rval = MBI->tag_get_handle(shared_set_tag_names[i], 1, MB_TYPE_INTEGER,
gtags[i], MB_TAG_ANY);
if (MB_SUCCESS!=rval)
return 1;
}
}
refCountMB++;
return MB_SUCCESS;
}
#if 0
/**
\fn ErrorCode iMOABInitializeFortran( )
\brief Initialize the iMOAB interface implementation from Fortran driver and create the MOAB instance, if not created already (reference counted).
<B>Operations:</B> Collective
*/
#if 0
ErrorCode iMOABInitializeFortran( );
#endif
......@@ -85,9 +128,39 @@ ErrCode iMOABFinalize()
\param[out] pid (iMOAB_AppID) The unique pointer to the application ID
*/
#if 0
ErrorCode RegisterApplication( iMOAB_String app_name, MPI_Comm* comm, iMOAB_AppID pid );
ErrCode RegisterApplication( iMOAB_String app_name, MPI_Comm* comm, iMOAB_AppID pid )
{
// will create a parallel comm for this application too, so there will be a
// mapping from *pid to file set and to parallel comm instances
std::string name(app_name);
if (appIdMap.find(name)!=appIdMap.end())
{
std::cout << " application already registered \n";
return 1;
}
*pid = unused_pid++;
appIdMap[name] = *pid;
// now create ParallelComm and a file set for this application
ParallelComm * pco = new ParallelComm(MBI, *comm);
#if 1
int index = pco->get_id(); // t could be useful to get app id from pcomm instance ...
assert(index==*pid);
#endif
pcomms.push_back(pco);
// create now the file set that will be used for loading the model in
EntityHandle file_set;
ErrorCode rval = MBI->create_meshset(MESHSET_SET, file_set);
if (MB_SUCCESS != rval )
return 1;
appData app_data;
app_data.file_set=file_set;
appDatas.push_back(app_data); // it will correspond to app_FileSets[*pid] will be the file set of interest
return 0;
}
#if 0
/**
\fn ErrorCode RegisterFortranApplication( iMOAB_String app_name, int* comm, iMOAB_AppID pid, int app_name_length )
\brief Register a Fortran-basedapplication - Create a unique application ID and bootstrap interfaces for further queries.
......@@ -103,8 +176,10 @@ ErrorCode RegisterApplication( iMOAB_String app_name, MPI_Comm* comm, iMOAB_AppI
\param[out] pid (iMOAB_AppID) The unique pointer to the application ID
\param[in] app_name_length (int) Length of application name string
*/
ErrorCode RegisterFortranApplication( iMOAB_String app_name, int* comm, iMOAB_AppID pid, int app_name_length );
ErrorCode RegisterFortranApplication( iMOAB_String app_name, int* comm, iMOAB_AppID pid, int app_name_length );
#endif
/**
\fn ErrorCode DeregisterApplication( iMOAB_AppID pid )
\brief De-Register application: delete mesh (set) associated with the application ID
......@@ -113,8 +188,35 @@ ErrorCode RegisterFortranApplication( iMOAB_String app_name, int* comm, iMOAB_Ap
\param[in] pid (iMOAB_AppID) The unique pointer to the application ID
*/
ErrorCode DeregisterApplication( iMOAB_AppID pid );
#endif
ErrCode DeregisterApplication( iMOAB_AppID pid )
{
// the file set , parallel comm are all in vectors indexed by *pid
// assume we did not delete anything yet
// *pid will not be reused if we register another application
ParallelComm * pco = pcomms[*pid];
// we could get the pco also with
// ParallelComm * pcomm = ParallelComm::get_pcomm(MBI, *pid);
EntityHandle fileSet = appDatas[*pid].file_set;
// get all entities part of the file set
Range fileents;
ErrorCode rval = MBI->get_entities_by_handle(fileSet, fileents, /*recursive */true);
if (MB_SUCCESS != rval )
return 1;
fileents.insert(fileSet);
rval = MBI->get_entities_by_type(fileSet, MBENTITYSET, fileents); // append all mesh sets
if (MB_SUCCESS != rval )
return 1;
delete pco;
rval = MBI->delete_entities(fileents);
if (MB_SUCCESS != rval )
return 1;
return 0;
}
/**
\fn ErrorCode ReadHeaderInfo ( iMOAB_String filename, int* num_global_vertices, int* num_global_elements, int* num_dimension, int* num_parts, int filename_length )
\brief Get global information from the file
......@@ -232,7 +334,7 @@ ErrCode ReadHeaderInfo ( iMOAB_String filename, int* num_global_vertices, int* n
}
#if 0
/**
\fn ErrorCode LoadMesh( iMOAB_AppID pid, iMOAB_String filename, iMOAB_String read_options, int num_ghost_layers, int filename_length, int read_options_length )
\brief Load a MOAB mesh file in parallel and exchange ghost layers as requested
......@@ -253,7 +355,39 @@ ErrCode ReadHeaderInfo ( iMOAB_String filename, int* num_global_vertices, int* n
*/
ErrorCode LoadMesh( iMOAB_AppID pid, iMOAB_String filename, iMOAB_String read_options, int num_ghost_layers, int filename_length, int read_options_length );
ErrCode LoadMesh( iMOAB_AppID pid, iMOAB_String filename, iMOAB_String read_options, int * num_ghost_layers, int filename_length, int read_options_length )
{
// make sure we use the file set and pcomm associated with the *pid
std::ostringstream newopts;
newopts << read_options;
newopts << ";PARALLEL_COMM="<<*pid;
if (*num_ghost_layers>=1)
{
// if we want ghosts, we will want additional entities, the last .1
// because the addl ents can be edges, faces that are part of the neumann sets
newopts << ";PARALLEL_GHOSTS=3.0."<<*num_ghost_layers<<".3";
}
ErrorCode rval = MBI->load_file(filename, &appDatas[*pid].file_set, newopts.str().c_str());
if (MB_SUCCESS!=rval)
return 1;
int rank = pcomms[*pid]->rank();
int nprocs=pcomms[*pid]->size();
#if 1
// some debugging stuff
std::ostringstream outfile;
outfile <<"TaskMesh_n" <<nprocs<<"."<< rank<<".h5m";
// the mesh contains ghosts too, but they are not part of mat/neumann set
// write in serial the file, to see what tags are missing
rval = MBI->write_file(outfile.str().c_str()); // everything on root
if (MB_SUCCESS!=rval)
return 1;
#endif
return 0;
}
/**
\fn ErrorCode WriteMesh( iMOAB_AppID pid, iMOAB_String filename, iMOAB_String write_options, int filename_length, int write_options_length )
......@@ -270,7 +404,17 @@ ErrorCode LoadMesh( iMOAB_AppID pid, iMOAB_String filename, iMOAB_String read_op
\param[in] filename_length (int*) Length of the filename string
\param[in] write_options_length (int*) Length of the write options string
*/
ErrorCode WriteMesh( iMOAB_AppID pid, iMOAB_String filename, iMOAB_String write_options, int filename_length, int write_options_length );
ErrCode WriteMesh( iMOAB_AppID pid, iMOAB_String filename, iMOAB_String write_options, int filename_length, int write_options_length )
{
// maybe do some processing of strings and lengths
// maybe do some options processing?
ErrorCode rval = MBI->write_file(filename,0, write_options, &appDatas[*pid].file_set, 1);
if (MB_SUCCESS!=rval)
return 1;
return 0;
}
/**
\fn ErrorCode GetMeshInfo( iMOAB_AppID pid, int* num_visible_vertices, int* num_visible_elements, int *num_visible_blocks, int* num_visible_surfaceBC, int* num_visible_vertexBC )
......@@ -285,8 +429,42 @@ ErrorCode WriteMesh( iMOAB_AppID pid, iMOAB_String filename, iMOAB_String write_
\param[out] num_visible_surfaceBC (int*) The number of surfaces that have a NEUMANN_SET B.C defined in local mesh in current partition/process arranged as: owned only, ghosted/shared, total_visible (array allocated by client, <TT>size := 3</TT>)
\param[out] num_visible_vertexBC (int*) The number of vertices that have a DIRICHLET_SET B.C defined in local mesh in current partition/process arranged as: owned only, ghosted/shared, total_visible (array allocated by client, <TT>size := 3</TT>)
*/
ErrorCode GetMeshInfo( iMOAB_AppID pid, int* num_visible_vertices, int* num_visible_elements, int *num_visible_blocks, int* num_visible_surfaceBC, int* num_visible_vertexBC );
ErrCode GetMeshInfo( iMOAB_AppID pid, int* num_visible_vertices, int* num_visible_elements, int *num_visible_blocks, int* num_visible_surfaceBC, int* num_visible_vertexBC )
{
// this will include ghost elements
// we should keep a data structure with mesh, sets, etc, for each pid
//
EntityHandle fileSet=appDatas[*pid].file_set;
ErrorCode rval = MBI->get_entities_by_type(fileSet, MBVERTEX, appDatas[*pid].all_verts, true); // recursive
if (MB_SUCCESS!=rval)
return 1;
*num_visible_vertices = (int) appDatas[*pid].all_verts.size();
// is dimension 3?
rval = MBI->get_entities_by_dimension(fileSet, 3, appDatas[*pid].primary_elems, true); // recursive
if (MB_SUCCESS!=rval)
return 1;
*num_visible_elements = (int) appDatas[*pid].primary_elems.size();
// get all blocks, BCs, etc
rval = MBI->get_entities_by_type_and_tag(fileSet, MBENTITYSET, &(gtags[0]), 0, 1, appDatas[*pid].mat_sets , Interface::UNION);
if (MB_SUCCESS!=rval)
return 1;
*num_visible_blocks = (int)appDatas[*pid].mat_sets.size();
rval = MBI->get_entities_by_type_and_tag(fileSet, MBENTITYSET, &(gtags[1]), 0, 1, appDatas[*pid].neu_sets , Interface::UNION);
if (MB_SUCCESS!=rval)
return 1;
*num_visible_surfaceBC = (int)appDatas[*pid].neu_sets.size();
rval = MBI->get_entities_by_type_and_tag(fileSet, MBENTITYSET, &(gtags[2]), 0, 1, appDatas[*pid].diri_sets , Interface::UNION);
if (MB_SUCCESS!=rval)
return 1;
*num_visible_vertexBC= (int)appDatas[*pid].diri_sets.size();
return 0;
}
#if 0
/**
\fn ErrorCode GetVertexID( iMOAB_AppID pid, int vertices_length, iMOAB_GlobalID* global_vertex_ID, iMOAB_LocalID* local_vertex_ID )
\brief Get the global vertex ID for all locally visible (owned and shared/ghosted) vertices
......
......@@ -141,7 +141,7 @@ ErrCode ReadHeaderInfo ( iMOAB_String filename, int* num_global_vertices, int* n
\param[in] filename_length (int) Length of the filename string
\param[in] read_options_length (int) Length of the read options string
*/
ErrCode LoadMesh( iMOAB_AppID pid, iMOAB_String filename, iMOAB_String read_options, int num_ghost_layers, int filename_length, int read_options_length );
ErrCode LoadMesh( iMOAB_AppID pid, iMOAB_String filename, iMOAB_String read_options, int * num_ghost_layers, int filename_length, int read_options_length );
/**
\fn ErrCode WriteMesh( iMOAB_AppID pid, iMOAB_String filename, iMOAB_String write_options, int filename_length, int write_options_length )
......
......@@ -8,3 +8,14 @@ Steps to test driver.c interface:
make driver.o
make driver
// test driver with ./driver p8ex1.h5m
Notes about this example file:
does not have overlapping neumann sets! (material sets are of course, non overlapping)
does not have dirichlett sets at all
it contains 4 material sets, and 8 parts; after ghosting, more material sets will be visible
on each task (the same it is maybe true for neumann sets)
this file is copied from https://svn.mcs.anl.gov/repos/SHARP/trunk, modules/unic/short_term_MOAB
It was checked in by Vijay, to test changes to moab parallel reader in 2012!!
......@@ -7,6 +7,14 @@
int main(int argc, char * argv[])
{
MPI_Init(&argc, &argv);
int nprocs, rank;
MPI_Comm comm=MPI_COMM_WORLD;
MPI_Comm_size(comm, &nprocs);
MPI_Comm_rank(comm, &rank);
char * filen = "p8ex1.h5m";
if (argc>1)
filen = argv[1];
......@@ -20,10 +28,45 @@ int main(int argc, char * argv[])
CHECKRC(rc, "failed to read header info");
printf("file %s has %d vertices, %d elements, %d parts in partition\n", filen,
num_global_vertices, num_global_elements, num_parts);
if (0==rank)
{
printf("file %s has %d vertices, %d elements, %d parts in partition\n", filen,
num_global_vertices, num_global_elements, num_parts);
}
int appID;
iMOAB_AppID pid=&appID;
rc = RegisterApplication( "PROTEUS", &comm, pid);
CHECKRC(rc, "failed to register application");
char *read_opts="PARALLEL=READ_PART;PARTITION=PARALLEL_PARTITION;PARALLEL_RESOLVE_SHARED_ENTS";
int num_ghost_layers=1;
rc = LoadMesh( pid, filen, read_opts, &num_ghost_layers, strlen(filen), strlen(read_opts) );
CHECKRC(rc, "failed to load mesh");
int nverts, nelem, nblocks, nsbc, ndbc;
rc = GetMeshInfo( pid, &nverts, &nelem, &nblocks, &nsbc, &ndbc);
CHECKRC(rc, "failed to get mesh info");
if (0==rank)
{
printf("on rank %d, there are \n"
" %d visible vertices\n"
" %d visible elements\n"
" %d visible blocks\n"
" %d visible neumann sets\n"
" %d visible dirichlet sets\n", rank, nverts, nelem, nblocks, nsbc, ndbc);
}
char outputFile[] = "fnew.h5m";
char writeOptions[] ="PARALLEL=WRITE_PART";
rc = WriteMesh(pid, outputFile, writeOptions,
strlen(outputFile), strlen(writeOptions) );
rc = DeregisterApplication(pid);
CHECKRC(rc, "failed to de-register application");
rc = iMOABFinalize();
CHECKRC(rc, "failed to finalize MOAB");
MPI_Finalize();
return 0;
}
......@@ -13,7 +13,7 @@ endif
.SUFFIXES: .o .cpp .F90
driver: driver.o ${MOAB_LIBDIR}/libMOAB.la
driver: driver.o ../src/imoab.o ${MOAB_LIBDIR}/libMOAB.la
${MOAB_CXX} -o $@ $< ../src/imoab.o ${MOAB_LIBS_LINK}
clean:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment