/* WriteDamsel.cpp
 * The Damsel library provides mesh-aware parallel I/O; see http://trac.mcs.anl.gov/projects/damsel/wiki for details,
 * though for now that site is restricted to project participants.  Damsel uses a data model that's very similar to
 * that used in MOAB and ITAPS.  It uses the same basic data model concepts of entities, sets, tags, and interface.
 * In theory, we should be able to completely save/restore to/from Damsel any data that can be saved/restored to/from
 * our native HDF5-based reader/writer.
 *
 * Mapping between MOAB-Damsel data models
 * =======================================
 * Basic data model entities, MOAB <--> Damsel:
 *        Entity <--> Entity
 *     EntitySet <--> Collection
 *           Tag <--> Tag
 * API/data strutures:
 *          Range (n1) <--> Sequence container
 *         std::vector <--> Vector container
 *          Range (n2) <--> Tree container
 *
 * n1: single contiguous subrange
 * n2: multiple subranges
 *
 * Conventions
 * ===========
 * There are parts of MOAB data structures that need to be stored to Damsel that aren't represented in the Damsel data model,
 * e.g. dense vs. sparse storage type, set tracking flags.  
 * - We need to store these as tags in Damsel.  
 * - Since Damsel tags need to have a MOAB counterpart, we have to create those as tag data in MOAB too 
 *   (duplicating the data in the data structures, bummer).
 * - Because we may want to use these tags for multiple Damsel writes/models, we create the MOAB-side tags in the WriteDamsel
 *   constructor, not in the init_tags function that's called for every write
 * - Conventional tags have names prefixed with mbdmsl_ to avoid name conflicts with other MOAB tags.
 * Here we list the conventional tags used by MOAB's Damsel reader/writer.
 * 
 * Tag name                   Tag char's (storage type, data type, length, def val)       Values, used for what
 * --------                   -----------------------------------------------------       --------------------
 * mbdmsl_XCOORDS             dense; double[1]; 0.0                                       MOAB vertex x coordinate
 * mbdmsl_YCOORDS             dense; double[1]; 0.0                                       MOAB vertex y coordinate
 * mbdmsl_ZCOORDS             dense; double[1]; 0.0                                       MOAB vertex z coordinate
 * mbdmsl_COLL_FLAGS          sparse; char; 1; 0x0                                        bit 0: 0=set-type, 1=vector-type
 *                                                                                            1: 1=tracking, 0=not tracking
 * mbdmsl_PARENTS      sparse; handle; var;                                        (list of parent sets)
 * mbdmsl_CHILDS       sparse; handle; var;                                        (list of child sets)
 *
 * 
 * 
 */

#include "WriteDamsel.hpp"

#include "DamselUtil.hpp"
#include "damsel.h"
#include "assert.h"
#include "moab/Interface.hpp"
#include "moab/Core.hpp"
#include "moab/Range.hpp"
#include "moab/Error.hpp"
#include "moab/WriteUtilIface.hpp"
#include "MBTagConventions.hpp"
#include "EntitySequence.hpp"
#include "Internals.hpp"
#include "DenseTag.hpp"
#include "SparseTag.hpp"

namespace moab {

WriterIface* WriteDamsel::factory(Interface* iface)
{
  return new WriteDamsel(iface);
}

WriteDamsel::WriteDamsel(Interface *impl)
  : mbImpl(impl), mWriteIface(NULL), sequenceManager(NULL), dU(),
    DAMSEL_FLAGS(DAMSEL_IS_TRACKING)
{
  assert(impl != NULL);

  impl->query_interface(mWriteIface);
  assert(mWriteIface);

  sequenceManager = dynamic_cast<Core*>(impl)->sequence_manager();
  assert(sequenceManager);

  ErrorCode rval = mbImpl->tag_get_handle("mbdmsl_XCOORDS", 1, MB_TYPE_DOUBLE,
                                         dU.xcoordsTag.mTagh, MB_TAG_DENSE | MB_TAG_CREAT);MB_CHK_SET_ERR_CONT(rval, "Failed to create_tag mbdmsl_XCOORDS");
  dU.xcoordsTag.tagType = MB_TAG_ANY;
  dU.tagMap.push_back(dU.xcoordsTag);
  rval = mbImpl->tag_get_handle("mbdmsl_YCOORDS", 1, MB_TYPE_DOUBLE,
                               dU.ycoordsTag.mTagh, MB_TAG_DENSE | MB_TAG_CREAT);MB_CHK_SET_ERR_CONT(rval, "Failed to create_tag mbdmsl_YCOORDS");
  dU.ycoordsTag.tagType = MB_TAG_ANY;
  dU.tagMap.push_back(dU.ycoordsTag);

  rval = mbImpl->tag_get_handle("mbdmsl_ZCOORDS", 1, MB_TYPE_DOUBLE,
                               dU.zcoordsTag.mTagh, MB_TAG_DENSE | MB_TAG_CREAT);MB_CHK_SET_ERR_CONT(rval, "Failed to create_tag mbdmsl_ZCOORDS");
  dU.zcoordsTag.tagType = MB_TAG_ANY;
  dU.tagMap.push_back(dU.zcoordsTag);

  rval = mbImpl->tag_get_handle("mbdmsl_COLL_FLAGS", 1, MB_TYPE_INTEGER,
                               dU.collFlagsTag.mTagh, MB_TAG_DENSE | MB_TAG_CREAT);MB_CHK_SET_ERR_CONT(rval, "Failed to create_tag mbdmsl_COLL_FLAGS");
  dU.collFlagsTag.tagType = MB_TAG_ANY;
  dU.tagMap.push_back(dU.collFlagsTag);

/*
  rval = mbImpl->tag_get_handle("mbdmsl_PARENTS", 1, MB_TYPE_HANDLE,
                               dU.parentsTag.mTagh, MB_TAG_DENSE | MB_TAG_CREAT | MB_TAG_VARLEN);MB_CHK_SET_ERR_CONT(rval, "Failed to create_tag mbdmsl_PARENTS");
  dU.parentsTag.tagType = MB_TAG_DENSE;
  dU.tagMap.push_back(dU.parentsTag);

  rval = mbImpl->tag_get_handle("mbdmsl_CHILDREN", 1, MB_TYPE_HANDLE,
                               dU.childrenTag.mTagh, MB_TAG_DENSE | MB_TAG_CREAT | MB_TAG_VARLEN);MB_CHK_SET_ERR_CONT(rval, "Failed to create_tag mbdmsl_CHILDREN");
  dU.childrenTag.tagType = MB_TAG_DENSE;
  dU.tagMap.push_back(dU.childrenTag);
*/

  dU.moabHandleType = (sizeof(EntityHandle) == 64 ? DAMSEL_HANDLE_TYPE_HANDLE64 :
                       DAMSEL_HANDLE_TYPE_HANDLE32);
}

WriteDamsel::~WriteDamsel()
{
  if (mWriteIface)
    mbImpl->release_interface(mWriteIface);
}

ErrorCode WriteDamsel::write_file(const char *file_name,
                                  const bool /* overwrite */,
                                  const FileOptions& opts,
                                  const EntityHandle *meshset_list,
                                  const int num_sets,
                                  const std::vector<std::string>& /* qa_records */,
                                  const Tag* /* tag_list */,
                                  int /* num_tags */,
                                  int /* requested_output_dimension */)
{
  // Gather all entities into one big range
  Range all_ents;
  ErrorCode rval;
  damsel_err_t err;

  dU.dmslLib = DMSLlib_init();

  // Create a damsel model
  dU.dmslModel = DMSLmodel_create(sizeof(EntityHandle) == 8 ? DAMSEL_HANDLE_TYPE_HANDLE64 :
                               DAMSEL_HANDLE_TYPE_HANDLE32);

  // Attach to a file, since we need it for creating containers
  MPI_Comm comm = MPI_COMM_WORLD;
  unlink(file_name);
  err = DMSLmodel_attach(dU.dmslModel, file_name, comm, NULL);CHK_DMSL_ERR(err, "DMSLmodel_attach failed");

  rval = mWriteIface->gather_entities(all_ents, meshset_list, num_sets);MB_CHK_SET_ERR(rval, "Gather entities failed in WriteDamsel");

  if (all_ents.empty())
    return MB_SUCCESS;

  // Create damsel tags for MOAB dense, sparse, and conventional tags
  rval = init_tag_info();MB_CHK_ERR(rval);

  // Iterate through the groups of contiguous sequences of handles
  RangeSeqIntersectIter rsi(sequenceManager);
  rval = rsi.init(all_ents.begin(), all_ents.end());

  while (MB_SUCCESS == rval) {
    // Write subrange of things to damsel: map handles, map entity definition data (connectivity/coords/set contents),
    // map dense tags
    rval = write_subrange(rsi);MB_CHK_SET_ERR(rval, "Failed to write subrange");

    rval = rsi.step();
    while (MB_ENTITY_NOT_FOUND == rval)
      rval = rsi.step();
  }

  // Write sparse tags
  rval = map_sparse_tags();MB_CHK_SET_ERR(rval, "Failed to write sparse tags");

  //damsel_request_t request;
  //err = DMSLmodel_transfer_async(dU.dmslModel, DAMSEL_TRANSFER_TYPE_WRITE, &request);
  err = DMSLmodel_transfer_sync(dU.dmslModel, DAMSEL_TRANSFER_TYPE_WRITE);CHK_DMSL_ERR(err, "DMSLmodel_transfer_asynch failed");

  //damsel_status_t status;
  //err = DMSLmodel_wait(request, &status);CHK_DMSL_ERR(err, "DMSLmodel_wait failed");

  DMSLmodel_close(dU.dmslModel);

  DMSLlib_finalize(dU.dmslLib);

  // We should be done
  return MB_SUCCESS;
}

ErrorCode WriteDamsel::init_tag_info()
{
  // Initialize allTags and tagIndices
  std::vector<Tag> tmp_mtags;
  ErrorCode rval = mbImpl->tag_get_tags(tmp_mtags);MB_CHK_SET_ERR(rval, "Failed to get all tag handles.");
  int dum_size;
  damsel_err_t err;

  // Define damsel tag handles for all dense/sparse tags
  for (std::vector<Tag>::iterator vit = tmp_mtags.begin(); vit != tmp_mtags.end(); ++vit) {
    if (((*vit)->get_storage_type() != MB_TAG_DENSE && (*vit)->get_storage_type() != MB_TAG_SPARSE) ||
        mbImpl->tag_get_length(*vit, dum_size) == MB_VARIABLE_DATA_LENGTH ||
        dum_size != 1) {
      std::cerr << "Warning: tag " << (*vit)->get_name()
                << "is not of type dense or sparse, and is not currently supported by the damsel writer." 
                << std::endl;
      continue;
    }

    std::vector<DamselUtil::tinfo>::iterator vit2 =
        std::find_if(dU.tagMap.begin(), dU.tagMap.end(), DamselUtil::MtagP<DamselUtil::tinfo>(*vit));

    if (vit2 != dU.tagMap.end() && (*vit2).tagType == MB_TAG_ANY) 
      // Conventional tag - skip
      continue;

    else if (vit2 == dU.tagMap.end()) {
      // Create a damsel counterpart for this tag
      Tag thandle = *vit;
      err = DMSLtag_define(dU.dmslModel, (damsel_handle_ptr)&thandle,
                                        DamselUtil::mtod_data_type[(*vit)->get_data_type()],
                            (*vit)->get_name().c_str());CHK_DMSL_ERR(err, "Failure to get Damsel tag for MOAB tag");
      dU.tagMap.push_back(DamselUtil::tinfo(thandle, 0, (*vit)->get_storage_type()));
    }
    else {
      // Assert there's a corresponding moab tag handle
      assert((*vit2).mTagh);
    }
  }

  // Do the same for conventional tags:
  // XCOORDS
  err = DMSLtag_define(dU.dmslModel, (damsel_handle_ptr)&(dU.xcoordsTag.mTagh),
                                       DamselUtil::mtod_data_type[MB_TYPE_DOUBLE],
                                       dU.xcoordsTag.mTagh->get_name().c_str());
  dU.tagMap.push_back(dU.xcoordsTag);CHK_DMSL_ERR(err, "Failure to get Damsel tag for MOAB tag");

  // YCOORDS
  err = DMSLtag_define(dU.dmslModel, (damsel_handle_ptr)&(dU.ycoordsTag.mTagh),
                                       DamselUtil::mtod_data_type[MB_TYPE_DOUBLE],
                                       dU.ycoordsTag.mTagh->get_name().c_str());
  dU.tagMap.push_back(dU.ycoordsTag);CHK_DMSL_ERR(err, "Failure to get Damsel tag for MOAB tag");

  // ZCOORDS
  err = DMSLtag_define(dU.dmslModel, (damsel_handle_ptr)&(dU.zcoordsTag.mTagh),
                                       DamselUtil::mtod_data_type[MB_TYPE_DOUBLE],
                                       dU.zcoordsTag.mTagh->get_name().c_str());
  dU.tagMap.push_back(dU.zcoordsTag);CHK_DMSL_ERR(err, "Failure to get Damsel tag for MOAB tag");

  // COLL_FLAGS
  err = DMSLtag_define(dU.dmslModel, (damsel_handle_ptr)&(dU.collFlagsTag.mTagh),
                                       DamselUtil::mtod_data_type[MB_TYPE_INTEGER],
                                       dU.collFlagsTag.mTagh->get_name().c_str());
  dU.tagMap.push_back(dU.collFlagsTag);CHK_DMSL_ERR(err, "Failure to get Damsel tag for MOAB tag");

/*
  SKIP PARENTS/CHILDREN FOR NOW, UNTIL WE HAVE VAR LENGTH TAGS IN DAMSEL

  // PARENTS
  dU.parentsTagPair.second = DMSLtag_define(dU.dmslModel, (damsel_handle_ptr)&(dU.collFlagsTagPair.first), 
                                           DamselUtil::mtod_data_type[(dU.collFlagsTagPair.first)->get_data_type()],
                                           (dU.parentsTagPair.first)->get_name().c_str());
  if (DAMSEL_TAG_INVALID == dtag)
    MB_SET_ERR(MB_FAILURE, "Failure to get Damsel tag for MOAB tag " << (dU.parentsTagPair.first)->get_name());

  // CHILDREN
  dU.childrenTagPair.second = DMSLtag_define(dU.dmslModel, (damsel_handle_ptr)&(dU.collFlagsTagPair.first), 
                                           DamselUtil::mtod_data_type[(dU.collFlagsTagPair.first)->get_data_type()],
                                           (dU.childrenTagPair.first)->get_name().c_str());
  if (DAMSEL_TAG_INVALID == dtag)
    MB_SET_ERR(MB_FAILURE, "Failure to get Damsel tag for MOAB tag " << (dU.childrenTagPair.first)->get_name());
*/

  // Map the tag handles in one big call
  int num_tags = dU.tagMap.size();
  std::vector<Tag> moab_taghs;
  moab_taghs.reserve(num_tags);
  for (std::vector<DamselUtil::tinfo>::iterator vit = dU.tagMap.begin(); vit != dU.tagMap.end(); ++vit) {
    moab_taghs.push_back((*vit).mTagh);
  }

  damsel_container mtags = DMSLcontainer_create_vector(dU.dmslModel, (damsel_handle_ptr)&moab_taghs[0], moab_taghs.size());
  std::cerr << "MOAB: created model container: mtags = " << mtags <<std::endl;

  err = DMSLmodel_map_handles_inventing_file_handles(mtags);CHK_DMSL_ERR(err, "Failed to map tag handles");

  err = DMSLcontainer_release(mtags);CHK_DMSL_ERR(err, "Problem releasing tag handle container");

  return MB_SUCCESS;
}

ErrorCode WriteDamsel::write_vertices(RangeSeqIntersectIter &rsi)
{
  // Write the vertices; these vertices will be in the same sequence and will be contiguous, guaranteed
  EntityHandle start_vert = rsi.get_start_handle(), end_vert = rsi.get_end_handle();

  // Create a damsel container for these vertex handles
  damsel_container vertex_cont = DMSLcontainer_create_sequence(dU.dmslModel, start_vert, (int)(end_vert-start_vert+1), 1);
  std::cerr << "MOAB: created model container: vertex_cont = " << vertex_cont <<std::endl;
  if (DAMSEL_CONTAINER_INVALID == vertex_cont)
    MB_SET_ERR(MB_FAILURE, "Failed to create vertex sequence for vertices starting with handle " << rsi.get_start_handle());

  damsel_err_t err = DMSLmodel_map_handles_inventing_file_handles(vertex_cont);CHK_DMSL_ERR(err, "Failed to map handles");

  // Define the entities to damsel
  err = DMSLentity_define(vertex_cont, DAMSEL_ENTITY_TYPE_VERTEX, 1, vertex_cont);CHK_DMSL_ERR(err, "Failure in DMSLentity_define for vertices starting with handle " << rsi.get_start_handle());

  // Get the vertex coordinates storage locations and pass to damsel
  Range vert_range(start_vert, end_vert);
  double *xcoords = NULL, *ycoords = NULL, *zcoords = NULL;
  int count;
  ErrorCode rval = mbImpl->coords_iterate(vert_range.begin(), vert_range.end(),
                                          xcoords, ycoords, zcoords, count);MB_CHK_SET_ERR(rval, "Failed to get coordinate iterator for vertices starting with handle " << rsi.get_start_handle());
  if (count != (int)vert_range.size()) {
    MB_SET_ERR(MB_FAILURE, "Vertex subrange not in the same sequence for vertices starting with handle " << rsi.get_start_handle());
  }

  if (xcoords && !ycoords && !zcoords) {
    // Interleaved

    // Map the data to damsel
    err = DMSLmodel_map_tag(xcoords, vertex_cont, (damsel_handle_ptr)&dU.xcoordsTag.mTagh);CHK_DMSL_ERR(err, "Failed to assign vertex coordinates tag for vertices starting with handle " << rsi.get_start_handle());
  }
  else {
    // Map the data to damsel
    err = DMSLmodel_map_tag(xcoords, vertex_cont, (damsel_handle_ptr)&dU.xcoordsTag.mTagh);CHK_DMSL_ERR(err, "Failed to assign vertex x coordinates tag for vertices starting with handle " << rsi.get_start_handle());
    err = DMSLmodel_map_tag(ycoords, vertex_cont, (damsel_handle_ptr)&dU.ycoordsTag.mTagh);CHK_DMSL_ERR(err, "Failed to assign vertex y coordinates tag for vertices starting with handle " << rsi.get_start_handle());
    err = DMSLmodel_map_tag(zcoords, vertex_cont, (damsel_handle_ptr)&dU.zcoordsTag.mTagh);CHK_DMSL_ERR(err, "Failed to assign vertex z coordinates tag for vertices starting with handle " << rsi.get_start_handle());
  }

  // Write/map dense tags
  rval = map_dense_tags(rsi, vertex_cont);MB_CHK_ERR(rval);

  err = DMSLcontainer_release(vertex_cont);CHK_DMSL_ERR(err, "Problem releasing vertex handle container");

  return MB_SUCCESS;
}

ErrorCode WriteDamsel::write_entities(RangeSeqIntersectIter &rsi)
{
  // Write the entities; these entities will be in the same sequence and will be contiguous, guaranteed
  EntityHandle start_ent = rsi.get_start_handle(), end_ent = rsi.get_end_handle();

  // Create a damsel container for these entity handles
  damsel_container ent_cont;
  ent_cont = DMSLcontainer_create_sequence(dU.dmslModel, start_ent, (int)(end_ent-start_ent+1), 1);
  std::cerr << "MOAB: created model container: ent_cont = " << ent_cont <<std::endl;
  if (DAMSEL_CONTAINER_INVALID == ent_cont)
    MB_SET_ERR(MB_FAILURE, "Bad sequence returned by Damsel");

  damsel_err_t err = DMSLmodel_map_handles_inventing_file_handles(ent_cont);CHK_DMSL_ERR(err, "Failed to map handles");

  // Get # verts per entity and entity type
  EntityType etype = mbImpl->type_from_handle(start_ent);
  assert(MBMAXTYPE != etype);
  int num_connect = rsi.get_sequence()->values_per_entity();
  assert(0 < num_connect);

  // Get the connectivity storage location and pass to damsel
  Range ent_range(start_ent, end_ent);
  int count;
  EntityHandle *connect;
  int verts_per_ent;
  ErrorCode rval = mbImpl->connect_iterate(ent_range.begin(), ent_range.end(), connect, verts_per_ent, count);MB_CHK_SET_ERR(rval, "Failed to get connect iterator for entities starting with handle " << rsi.get_start_handle());
  if (count != (int)ent_range.size())
    MB_SET_ERR(MB_FAILURE, "Entity subrange not in the same sequence for entities starting with handle " << rsi.get_start_handle());

  // Define the entities to damsel
  err = DMSLentity_define_fast(ent_cont, DamselUtil::mtod_entity_type[etype], num_connect, (damsel_handle*)connect);CHK_DMSL_ERR(err, "DMSLentity_define failed for entities starting with handle " << rsi.get_start_handle());

  // Write dense tags
  rval = map_dense_tags(rsi, ent_cont);MB_CHK_ERR(rval);

  err = DMSLcontainer_release(ent_cont);CHK_DMSL_ERR(err, "Problem releasing entity handle container");

  return MB_SUCCESS;
}

ErrorCode WriteDamsel::map_dense_tags(RangeSeqIntersectIter &rsi, damsel_container &ent_cont)
{
  // All dense_tags have been initialized before this, so here we just go through
  // them and map data if there is any
  const unsigned char *val_ptr;
  ErrorCode rval = MB_SUCCESS;
  std::vector<DamselUtil::tinfo>::iterator tagit;
  damsel_err_t err;
  for (tagit = dU.tagMap.begin(); tagit != dU.tagMap.end(); ++tagit) {
    if ((*tagit).tagType != MB_TAG_DENSE)
      continue;

    // Get a ptr to memory for this tag/sequence
    DenseTag *dtag = dynamic_cast<DenseTag*>((*tagit).mTagh);
    assert(dtag);
    rval = dtag->get_array(rsi.get_sequence(), val_ptr);
    MB_CHK_SET_ERR(rval, "Failed to get tag coordinates pointer for vertices starting with handle " << rsi.get_start_handle());

    // If ptr is NULL, no data for this tag in this sequence
    if (!val_ptr)
      continue;

    // Else, register with damsel
    err = DMSLmodel_map_tag((void*)val_ptr, ent_cont, (damsel_handle_ptr)&dtag);CHK_DMSL_ERR(err, "Failed to write coordinates tag for vertices starting with handle " << rsi.get_start_handle());
  }

  return rval;
}

ErrorCode WriteDamsel::map_sparse_tags()
{
  // All sparse_tags have been initialized before this, so here we just go through
  // them and map data if there is any
  ErrorCode rval = MB_SUCCESS;
  damsel_err_t err;
  std::vector<DamselUtil::tinfo>::iterator tagit;
  std::vector<unsigned char> tag_values;
  std::vector<EntityHandle> tagged_ents;
  damsel_container ent_cont;
  for (tagit = dU.tagMap.begin(); tagit != dU.tagMap.end(); ++tagit) {
    if ((*tagit).tagType != MB_TAG_SPARSE)
      continue;
    // Get a ptr to memory for this tag/sequence
    SparseTag *stag = dynamic_cast<SparseTag*>((*tagit).mTagh);
    assert(stag);
    Range output_ents;
    rval = stag->get_tagged_entities(sequenceManager, output_ents);MB_CHK_SET_ERR(rval, "Trouble getting tagged entities for tag " << stag->get_name());

    // If no entities have this tag set, don't map it
    if (output_ents.empty())
      continue;

    // Else, register with damsel
    // Allocate space for and get values
    tag_values.resize(stag->get_size() * output_ents.size());
    rval = mbImpl->tag_get_data(stag, output_ents, &tag_values[0]);MB_CHK_SET_ERR(rval, "Trouble getting tag values for tag " << stag->get_name());

    // Build a vector of entity handles from the range, and a container from that
    tagged_ents.resize(output_ents.size());
    std::copy(output_ents.begin(), output_ents.end(), tagged_ents.begin());
    ent_cont = DMSLcontainer_create_vector(dU.dmslModel, (damsel_handle_ptr)&tagged_ents[0], tagged_ents.size());
    std::cerr << "MOAB: created model container: sparse_tag_ent_cont = " << ent_cont <<std::endl;
    if (DAMSEL_CONTAINER_INVALID == ent_cont)
      MB_SET_ERR(MB_FAILURE, "Trouble creating entity handle container for tag " << stag->get_name());

    // Now map it
    err = DMSLmodel_map_tag((void*)&tag_values[0], ent_cont, (damsel_handle_ptr)&stag);CHK_DMSL_ERR(err, "Failed to write tag " << stag->get_name());

    err = DMSLcontainer_release(ent_cont);CHK_DMSL_ERR(err, "Problem releasing entity handle container");
  }

  return rval;
}

ErrorCode WriteDamsel::write_sets(RangeSeqIntersectIter &rsi)
{
  // Write the sets
  ErrorCode rval = MB_SUCCESS;
  std::vector<EntityHandle> ents;
  damsel_container mcont;
  damsel_err_t err;
  unsigned int i, num_sets = rsi.get_end_handle() - rsi.get_start_handle() + 1;
  std::vector<unsigned int> set_flags(num_sets, 0);
  EntityHandle seth;
  for (seth = rsi.get_start_handle(), i = 0; seth <= rsi.get_end_handle(); seth++, i++) {
    // Get all the entities in the set
    ents.clear();
    rval = mbImpl->get_entities_by_handle(seth, ents);MB_CHK_SET_ERR(rval, "get_entities_by_handle failed for set " << seth);
    if (!ents.empty()) {
      mcont = DMSLcontainer_create_vector(dU.dmslModel, (damsel_handle*)&ents[0], ents.size());
    }
    else {
      mcont = DMSLcontainer_create_vector(dU.dmslModel, (damsel_handle*)NULL, 0);
    }
    std::cerr << "MOAB: created model container: sets_cont = " << mcont <<std::endl;

    // Get the set type (range or set)
    unsigned int opts;
    rval = mbImpl->get_meshset_options(seth, opts);MB_CHK_SET_ERR(rval, "Failed to get options for meshset " << seth);
    damsel_collection_type coll_type = (opts&MESHSET_SET ? DAMSEL_HANDLE_COLLECTION_TYPE_SET :
                     DAMSEL_HANDLE_COLLECTION_TYPE_VECTOR);

    // Parents/children...

    // Set flags
    if (opts & MESHSET_TRACK_OWNER)
      set_flags[i] |= MESHSET_TRACK_OWNER;
    else
      set_flags[i] &= !MESHSET_TRACK_OWNER;

    // Create the collection
    DMSLcoll_create(dU.dmslModel, (damsel_handle_ptr)&seth, mcont, coll_type);

    // Release the container
    err = DMSLcontainer_release(mcont);CHK_DMSL_ERR(err, "Problem releasing set entity handle container");
  }

  // Set the COLL_FLAGS tag, using assign (direct)
  // Make a container of set handles...
  mcont = DMSLcontainer_create_sequence(dU.dmslModel, rsi.get_start_handle(), num_sets, 1);
  std::cerr << "MOAB: created model container: sets_cont = " << mcont <<std::endl;

  // Assign the tags on them
  err = DMSLmodel_map_tag(&set_flags[0], mcont, (damsel_handle_ptr)&(dU.collFlagsTag.mTagh));CHK_DMSL_ERR(err, "Failed to assign COLL_FLAGS tag for sets");

  // Map set handles
  err = DMSLmodel_map_handles_inventing_file_handles(mcont);CHK_DMSL_ERR(err, "Failed to map set handles");

  // Map other dense tags
  rval = map_dense_tags(rsi, mcont);MB_CHK_SET_ERR(rval, "Failed to map dense tags for sets");

  return rval;
}

} // namespace moab
