fides 2020-09-03 (f507c538)

Code extracted from:

    https://gitlab.kitware.com/third-party/fides.git

at commit f507c53825ce50738b52ce07605feea2531bcfac (for/vtk-20200903-master).

.gitattributes

+data/* filter=lfs diff=lfs merge=lfs -text
+/**/data/* filter=lfs diff=lfs merge=lfs -text
+/**/data/*/* filter=lfs diff=lfs merge=lfs -text
+
+# XXX(kitware): Ignore whitespace issues.
+* -whitespace

CMakeLists.txt

+if (FALSE) # XXX(kitware): VTK handles CMake minimum versions
+cmake_minimum_required(VERSION 3.9)
+endif()
+project(FIDES LANGUAGES C CXX VERSION 0.1)
+
+list(INSERT CMAKE_MODULE_PATH 0 "${PROJECT_SOURCE_DIR}/cmake")
+
+if (FALSE) # XXX(kitware): not needed for VTK
+# Add options for performing sanitization
+include(cmake/FidesSanitize.cmake)
+
+# Add options for performing code coverage tests
+include(cmake/FidesCoverage.cmake)
+endif()
+
+include(FidesModule)
+
+if (FALSE) # XXX(kitware): VTK handles VTK-m
+find_package(ADIOS2 REQUIRED)
+find_package(VTKm REQUIRED)
+else()
+  vtk_module_find_package(PACKAGE ADIOS2)
+endif()
+
+if (FALSE) # XXX(kitware): VTK handles building shared libs/testing and MPI
+option(BUILD_SHARED_LIBS "Build fides with shared libraries" ON)
+set(FIDES_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS})
+option(FIDES_ENABLE_TESTING "Enable Fides Testing" ON)
+option(FIDES_ENABLE_EXAMPLES "Build Fides Examples" ON)
+
+if(ADIOS2_HAVE_MPI)
+  set(MPI_DEFS FIDES_USE_MPI)
+  set(MPI_LIBS MPI::MPI_CXX)
+endif()
+
+else()
+  set(FIDES_ENABLE_TESTING OFF)
+endif()
+
+option(FIDES_USE_EXTERNAL_RAPIDJSON "Use external RapidJSON" OFF)
+mark_as_advanced(FIDES_USE_EXTERNAL_RAPIDJSON)
+if (FIDES_USE_EXTERNAL_RAPIDJSON)
+  find_package(RapidJSON REQUIRED)
+endif()
+
+if (NOT DEFINED FIDES_INSTALL_INCLUDE_DIR)
+  set(FIDES_INSTALL_INCLUDE_DIR "include/fides")
+endif()
+
+if (NOT DEFINED FIDES_EXPORT_NAME)
+  set(FIDES_EXPORT_NAME "FidesTargets")
+endif()
+
+add_subdirectory(fides)
+
+if(VTK_USE_MPI)
+  set(MPI_DEFS FIDES_USE_MPI)
+endif()
+
+# add third party
+add_subdirectory(thirdparty/rapidjson)
+
+if (FALSE) # XXX(kitware): don't need testing related stuff
+if (FIDES_ENABLE_TESTING)
+  enable_testing()
+  include(CTest)
+  add_subdirectory(tests)
+endif()
+
+if (FIDES_ENABLE_SANITIZER)
+  set(preload_libraries)
+  if (FIDES_SANITIZER MATCHES "address")
+    find_library(FIDES_ASAN_LIBRARY NAMES libasan.so.5 DOC "ASan library")
+    mark_as_advanced(FIDES_ASAN_LIBRARY)
+
+    list(APPEND preload_libraries
+      "${FIDES_ASAN_LIBRARY}")
+  endif ()
+endif()
+
+set(FIDES_COVERAGE_ENABLED OFF)
+if (FIDES_ENABLE_COVERAGE)
+  set(FIDES_COVERAGE_ENABLED ON)
+endif()
+
+configure_file(
+  "${CMAKE_CURRENT_SOURCE_DIR}/cmake/CTestCustom.cmake.in"
+  "${CMAKE_CURRENT_BINARY_DIR}/CTestCustom.cmake"
+  )
+
+if (FIDES_ENABLE_EXAMPLES)
+  add_subdirectory(examples)
+endif()
+
+option(FIDES_BUILD_DOCS "Build Fides User Guide locally" OFF)
+if (FIDES_BUILD_DOCS)
+  add_subdirectory(docs)
+endif()
+endif()

LICENSE.txt

+========================================================================
+Copyright (c) 2019 Kitware Inc.
+
+Kitware, Inc.
+1712 Route 9, Suite 300
+Clifton Park, NY
+12065 USA
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the
+   distribution.
+
+ * Neither the name of Kitware nor the names of any contributors may
+   be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+========================================================================

README.kitware.md

+# Fides fork for VTK
+
+This branch contains changes required to embed Fides into VTK. This includes
+changes made primarily to the build system to allow it to be embedded into
+another source tree as well as a header to facilitate mangling of the symbols
+to avoid conflicts with other copies of the library within a single process.
+
+
+* add to .gitattributes to pass commit checks within VTK
+* update CMake files to use vtk_module to get into VTK's install tree

README.md

+[![Documentation Status](https://readthedocs.org/projects/fides/badge/?version=latest)](https://fides.readthedocs.io/en/latest/?badge=latest)
+
+## Fides: Adaptable Data Interfaces and Services
+
+Fides is a library for adding metadata to ADIOS2 files so they can be more easily read and understood by VTK-m.
+
+The metadata required to visualize a dataset is often different than the metadata required in other contexts, so despite the fact we want our simulation data to be "self-describing", this statement has different meanings in different contexts, and hence it is almost impossible to realize in practice.
+
+To use Fides, you must first create a `.json` file which has information relevant for processing the file in VTK-m.
+
+## Documentation
+
+Documentation is hosted at [Read The Docs](https://fides.readthedocs.io/en/latest/?badge=latest).
+
+## Dependencies
+
+Fides depends of VTK-m and ADIOS2.
+
+A commit of VTK-m which is known to work is 4d010fee (from Aug 6, 2020).
+
+Due to recent fixes in ADIOS, use commit 3324c77f95 (from Apr 13, 2020) or later.
+Run the unit tests if doubt arises.
+
+
+## Testing
+
+`Fides` uses `git lfs` to manage its datasets.
+This must be initialized after cloning.
+
+```
+fides$ git lfs install
+fides$ git lfs pull
+```
+
+The tests are managed by `ctest`, so in the build directory just run
+
+```
+build_fides$ ctest -V
+```

cmake/FidesExportHeaderTemplate.h.in

+//=============================================================================
+//
+//  Copyright (c) Kitware, Inc.
+//  All rights reserved.
+//  See LICENSE.txt for details.
+//
+//  This software is distributed WITHOUT ANY WARRANTY; without even
+//  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+//  PURPOSE.  See the above copyright notice for more information.
+//=============================================================================
+
+#ifndef @EXPORT_MACRO_NAME@_EXPORT_H
+#define @EXPORT_MACRO_NAME@_EXPORT_H
+
+#if defined(_MSC_VER)
+# if @EXPORT_IS_BUILT_STATIC@
+    /* This is a static component and has no need for exports
+       elf based static libraries are able to have hidden/default visibility
+       controls on symbols so we should propagate this information in that
+       use case
+    */
+#   define @EXPORT_MACRO_NAME@_EXPORT_DEFINE
+#   define @EXPORT_MACRO_NAME@_IMPORT_DEFINE
+#   define @EXPORT_MACRO_NAME@_NO_EXPORT_DEFINE
+# else
+#   define @EXPORT_MACRO_NAME@_EXPORT_DEFINE __declspec(dllexport)
+#   define @EXPORT_MACRO_NAME@_IMPORT_DEFINE __declspec(dllimport)
+#   define @EXPORT_MACRO_NAME@_NO_EXPORT_DEFINE
+# endif
+#else
+#   define @EXPORT_MACRO_NAME@_EXPORT_DEFINE __attribute__((visibility("default")))
+#   define @EXPORT_MACRO_NAME@_IMPORT_DEFINE __attribute__((visibility("default")))
+#   define @EXPORT_MACRO_NAME@_NO_EXPORT_DEFINE __attribute__((visibility("hidden")))
+#endif
+
+#ifndef @EXPORT_MACRO_NAME@_EXPORT
+# if defined(@EXPORT_IMPORT_CONDITION@)
+    /* We are building this library */
+#   define @EXPORT_MACRO_NAME@_EXPORT @EXPORT_MACRO_NAME@_EXPORT_DEFINE
+# else
+    /* We are using this library */
+#   define @EXPORT_MACRO_NAME@_EXPORT @EXPORT_MACRO_NAME@_IMPORT_DEFINE
+# endif
+#endif
+
+#ifndef @EXPORT_MACRO_NAME@_TEMPLATE_EXPORT
+# if defined(@EXPORT_IMPORT_CONDITION@) && defined(_MSC_VER)
+  /* Warning C4910 on windows state that extern explicit template can't be
+     labeled with __declspec(dllexport). So that is why we use a new custom
+     define. But when other modules ( e.g. rendering ) include this header
+     we need them to see that the extern template is actually being imported.
+  */
+    /* We are building this library with MSVC */
+#   define @EXPORT_MACRO_NAME@_TEMPLATE_EXPORT
+# elif defined(@EXPORT_IMPORT_CONDITION@)
+    /* We are building this library */
+#   define @EXPORT_MACRO_NAME@_TEMPLATE_EXPORT @EXPORT_MACRO_NAME@_EXPORT_DEFINE
+# else
+    /* We are using this library */
+#   define @EXPORT_MACRO_NAME@_TEMPLATE_EXPORT @EXPORT_MACRO_NAME@_IMPORT_DEFINE
+# endif
+#endif
+
+#ifndef @EXPORT_MACRO_NAME@_NO_EXPORT
+  #define @EXPORT_MACRO_NAME@_NO_EXPORT @EXPORT_MACRO_NAME@_NO_EXPORT_DEFINE
+#endif
+
+#endif
+

cmake/FidesModule.cmake

+##============================================================================
+##  Copyright (c) Kitware, Inc.
+##  All rights reserved.
+##  See LICENSE.txt for details.
+##
+##  This software is distributed WITHOUT ANY WARRANTY; without even
+##  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+##  PURPOSE.  See the above copyright notice for more information.
+##============================================================================
+
+#-----------------------------------------------------------------------------
+# Utility to build a kit name from the current directory.
+function(fides_get_kit_name kitvar)
+  string(REPLACE "${FIDES_SOURCE_DIR}/" "" dir_prefix ${CMAKE_CURRENT_SOURCE_DIR})
+  string(REPLACE "/" "_" kit "${dir_prefix}")
+  set(${kitvar} "${kit}" PARENT_SCOPE)
+  # Optional second argument to get dir_prefix.
+  if (${ARGC} GREATER 1)
+    set(${ARGV1} "${dir_prefix}" PARENT_SCOPE)
+  endif (${ARGC} GREATER 1)
+endfunction(fides_get_kit_name)

cmake/fides_generate_export_header.cmake

+#-----------------------------------------------------------------------------
+function(fides_generate_export_header lib_name)
+  # Now generate a header that holds the macros needed to easily export
+  # template classes. This
+  string(TOUPPER ${lib_name} BASE_NAME_UPPER)
+  set(EXPORT_MACRO_NAME "${BASE_NAME_UPPER}")
+
+  set(EXPORT_IS_BUILT_STATIC 0)
+  get_target_property(is_static ${lib_name} TYPE)
+  if(${is_static} STREQUAL "STATIC_LIBRARY")
+    #If we are building statically set the define symbol
+    set(EXPORT_IS_BUILT_STATIC 1)
+  endif()
+  unset(is_static)
+
+  get_target_property(EXPORT_IMPORT_CONDITION ${lib_name} DEFINE_SYMBOL)
+  if(NOT EXPORT_IMPORT_CONDITION)
+    #set EXPORT_IMPORT_CONDITION to what the DEFINE_SYMBOL would be when
+    #building shared
+    set(EXPORT_IMPORT_CONDITION ${lib_name}_EXPORTS)
+  endif()
+
+  configure_file(
+      ${FIDES_SOURCE_DIR}/cmake/FidesExportHeaderTemplate.h.in
+      ${CMAKE_CURRENT_BINARY_DIR}/${lib_name}_export.h
+    @ONLY)
+
+endfunction()

fides/Array.h

+//============================================================================
+//  Copyright (c) Kitware, Inc.
+//  All rights reserved.
+//  See LICENSE.txt for details.
+//  This software is distributed WITHOUT ANY WARRANTY; without even
+//  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+//  PURPOSE.  See the above copyright notice for more information.
+//
+//============================================================================
+
+#ifndef fides_datamodel_Array_H_
+#define fides_datamodel_Array_H_
+
+#include <vtkm/cont/VariantArrayHandle.h>
+
+#include <fides/DataModel.h>
+#include <fides/Value.h>
+#include <fides/xgc/XGCCommon.h>
+
+namespace fides
+{
+namespace datamodel
+{
+
+/// \brief Superclass for all specialized array implementations.
+struct ArrayBase : public DataModelBase
+{
+  /// Reads and returns array handles. Has to be implemented
+  /// by subclasses.
+  virtual std::vector<vtkm::cont::VariantArrayHandle> Read(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources,
+    const fides::metadata::MetaData& selections) = 0;
+
+  /// Returns the number of blocks in the underlying variable.
+  /// Used by the reader to provide meta-data on blocks.
+  /// Has to be implemented by subclasses.
+  virtual size_t GetNumberOfBlocks(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources) = 0;
+
+  virtual ~ArrayBase() {};
+};
+
+/// \brief A placeholder for setting up Wildcard Fields.
+///
+/// When a wildcard field is specified in a data model,
+/// Fides first creates \c ArrayPlacholder object for its
+/// data. When reading metadata, Fides is able to read ADIOS
+/// files (or streams) and expand the wildcard fields.
+/// At this point the actual Arrays to be used for the fields
+/// can be created. ArrayPlaceholders should no longer exist
+/// after metadata is read and thus Read and GetNumberOfBlocks
+/// will throw errors if they are used.
+struct ArrayPlaceholder : public ArrayBase
+{
+  /// Overridden to process the JSON for an array belonging
+  /// to a wildcard field
+  void ProcessJSON(const rapidjson::Value& json,
+                   DataSourcesType& sources) override;
+
+  /// Throws error because this Array class is a placeholder for
+  /// arrays belonging to wildcard fields that will eventually
+  /// be expanded
+  std::vector<vtkm::cont::VariantArrayHandle> Read(
+    const std::unordered_map<std::string, std::string>&,
+    DataSourcesType&,
+    const fides::metadata::MetaData&) override;
+
+  /// Throws error because this Array class is a placeholder for
+  /// arrays belonging to wildcard fields that will eventually
+  /// be expanded
+  size_t GetNumberOfBlocks(
+    const std::unordered_map<std::string, std::string>&,
+    DataSourcesType&) override;
+
+private:
+  std::string ArrayType;
+};
+
+/// \brief Data model object for VTK-m array handles.
+///
+/// \c fides::datamodel::Array is responsible of creating
+/// VTK-m \c ArrayHandles by loading data defined by the Fides
+/// data model. This class delegates its responsibilities to
+/// one of the specialized \c ArrayBase subclasses it creates
+/// during JSON parsing.
+struct Array : public DataModelBase
+{
+  /// Overridden to handle Array specific items.
+  /// This will create an internal ArrayBase subclass
+  /// depending on the array_type value.
+  void ProcessJSON(const rapidjson::Value& json,
+                   DataSourcesType& sources) override;
+
+  /// Sets up an ArrayPlaceholder for a wildcard field.
+  /// Should be used by wildcard fields instead of
+  /// ProcessJSON.
+  void CreatePlaceholder(const rapidjson::Value& json,
+      DataSourcesType& sources);
+
+  /// Reads and returns array handles. Handled by the
+  /// internal ArrayBase subclass.
+  std::vector<vtkm::cont::VariantArrayHandle> Read(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources,
+    const fides::metadata::MetaData& selections);
+
+  /// Returns the number of blocks in the underlying variable.
+  /// Used by the reader to provide meta-data on blocks.
+  /// Handled by the internal ArrayBase subclass.
+  size_t GetNumberOfBlocks(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources);
+
+private:
+  std::unique_ptr<ArrayBase> ArrayImpl = nullptr;
+  // only used when this array belongs to a wildcard field
+  std::unique_ptr<ArrayPlaceholder> Placeholder = nullptr;
+};
+
+/// \brief Class to read \c ArrayHandle objects.
+///
+/// \c ArrayBasic reads ArrayHandle objects with basic storage.
+struct ArrayBasic : public ArrayBase
+{
+  void ProcessJSON(const rapidjson::Value& json,
+                   DataSourcesType& sources) override;
+
+  /// Reads and returns array handles. The heavy-lifting is
+  /// handled by the \c DataModelBase \c ReadSelf() method.
+  /// The paths are passed to the \c DataSources to create
+  /// file paths. \c selections restrict the data that is loaded.
+  std::vector<vtkm::cont::VariantArrayHandle> Read(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources,
+    const fides::metadata::MetaData& selections) override;
+
+  /// Returns the number of blocks in the underlying variable.
+  /// Used by the reader to provide meta-data on blocks.
+  size_t GetNumberOfBlocks(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources) override;
+
+private:
+  fides::io::IsVector IsVector = fides::io::IsVector::Auto;
+};
+
+/// \brief Class to read \c ArrayHandleUniformPointCoordinates objects.
+///
+/// \c ArrayUniformPointCoordinates creates ArrayHandleUniformPointCoordinates
+/// objects. It depends on a number of Value objects to obtains dimensions
+/// (per block), origin and spacing. The origin of each block is computed based
+/// on spacing and additional values provided by the Dimensions
+/// Value object (start indices for each block).
+struct ArrayUniformPointCoordinates : public ArrayBase
+{
+  /// Overridden to handle ArrayUniformPointCoordinates specific items.
+  void ProcessJSON(const rapidjson::Value& json,
+                   DataSourcesType& sources) override;
+
+  /// Reads and returns array handles. This class depends on
+  /// a number of Value objects to obtains dimensions (per block),
+  /// origin and spacing. The origin of each block is computed based
+  /// on spacing and additional values provided by the Dimensions
+  /// Value object (start indices for each block).
+  std::vector<vtkm::cont::VariantArrayHandle> Read(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources,
+    const fides::metadata::MetaData& selections) override;
+
+  /// Returns the number of blocks in the underlying variable.
+  /// Used by the reader to provide meta-data on blocks.
+  size_t GetNumberOfBlocks(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources) override;
+
+private:
+  std::unique_ptr<Value> Dimensions = nullptr;
+  std::unique_ptr<Value> Origin = nullptr;
+  std::unique_ptr<Value> Spacing = nullptr;
+};
+
+struct ArrayCartesianProduct : public ArrayBase
+{
+  /// Overridden to handle ArrayCartesianProduct specific items.
+  void ProcessJSON(const rapidjson::Value& json,
+                   DataSourcesType& sources) override;
+
+  /// Reads and returns array handles. This class depends on
+  /// three separate (basic) array  objects that form the
+  /// cartesian product.
+  std::vector<vtkm::cont::VariantArrayHandle> Read(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources,
+    const fides::metadata::MetaData& selections) override;
+
+  /// Returns the number of blocks in the underlying variable.
+  /// Used by the reader to provide meta-data on blocks.
+  /// Uses the number of blocks in the first (x) array.
+  size_t GetNumberOfBlocks(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources) override;
+
+private:
+  std::unique_ptr<Array> XArray = nullptr;
+  std::unique_ptr<Array> YArray = nullptr;
+  std::unique_ptr<Array> ZArray = nullptr;
+};
+
+/// \brief Class for handling XGC data and contains common functionality
+/// for handling both XGC coordinates and fields.
+struct ArrayXGC : public ArrayBase
+{
+  ArrayXGC();
+
+  /// Returns the number of blocks in the underlying variable.
+  /// Used by the reader to provide meta-data on blocks.
+  size_t GetNumberOfBlocks(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources) override;
+
+protected:
+  std::unique_ptr<XGCCommon> CommonImpl;
+  vtkm::Id NumberOfPlanes = -1;
+  bool EngineChecked = false;
+
+  /// Ensures that the inline engine isn't being used since it's not
+  /// supported for XGC
+  void CheckEngineType(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources, std::string& dataSourceName);
+
+  /// Gets the shape of the variable
+  std::vector<size_t> GetShape(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources);
+};
+
+/// \brief Class to read \c ArrayXGCCoordinates objects.
+struct ArrayXGCCoordinates : public ArrayXGC
+{
+  /// Overridden to handle ArrayXGCCoordinates specific items.
+  void ProcessJSON(const rapidjson::Value& json,
+                   DataSourcesType& sources) override;
+
+  /// Reads and returns array handles.
+  std::vector<vtkm::cont::VariantArrayHandle> Read(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources,
+    const fides::metadata::MetaData& selections) override;
+
+private:
+  bool IsCylindrical = true;
+  struct AddToVectorFunctor;
+};
+
+/// \brief Class to read \c ArrayXGCField objects.
+struct ArrayXGCField : public ArrayXGC
+{
+  /// Reads and returns array handles.
+  std::vector<vtkm::cont::VariantArrayHandle> Read(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources,
+    const fides::metadata::MetaData& selections) override;
+
+  /// Special handling for reading 3D variables. Use instead of the superclass's
+  /// ReadSelf()
+  std::unordered_map<size_t, vtkm::cont::VariantArrayHandle> Read3DVariable(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources,
+    const fides::metadata::MetaData& selections);
+
+private:
+  /// If Is2DField, then that means the variable is duplicated for each plane
+  /// If false, then each plane has its own set of values.
+  bool Is2DField = true;
+  bool FieldDimsChecked = false;
+  struct AddToVectorFunctor;
+};
+}
+}
+
+#endif

fides/CMakeLists.txt

+set(srcs
+  FidesTypes.cxx
+  Array.cxx
+  CellSet.cxx
+  CoordinateSystem.cxx
+  DataModel.cxx
+  DataSetReader.cxx
+  DataSource.cxx
+  Field.cxx
+  FieldData.cxx
+  FieldDataManager.cxx
+  Keys.cxx
+  MetaData.cxx
+  Value.cxx
+)
+
+set(headers
+  FidesTypes.h
+  Array.h
+  CellSet.h
+  CoordinateSystem.h
+  DataModel.h
+  DataSetReader.h
+  DataSource.h
+  Field.h
+  FieldData.h
+  FieldDataManager.h
+  Keys.h
+  MetaData.h
+  Value.h
+)
+
+include(${FIDES_SOURCE_DIR}/cmake/fides_generate_export_header.cmake)
+
+if (FALSE) # XXX(kitware): use vtk_module system
+add_library(fides ${srcs})
+else()
+  vtk_module_add_module(VTK::fides
+    SOURCES ${srcs}
+    HEADERS ${headers}
+    HEADERS_SUBDIR "vtkfides/fides")
+endif()
+
+fides_generate_export_header(fides)
+
+target_compile_definitions(fides PRIVATE ${MPI_DEFS})
+
+if (FALSE) # XXX(kitware): use vtk_module system
+target_link_libraries(fides PUBLIC vtkm_cont adios2::adios2 PRIVATE
+  fides_rapidjson)
+
+target_include_directories(fides PUBLIC
+  $<BUILD_INTERFACE:${FIDES_SOURCE_DIR}>
+  $<BUILD_INTERFACE:${FIDES_BINARY_DIR}/fides>
+  $<INSTALL_INTERFACE:include>
+  )
+else()
+target_link_libraries(fides PUBLIC VTK::vtkm PRIVATE
+  adios2::adios2 fides_rapidjson)
+target_include_directories(fides PUBLIC
+  $<BUILD_INTERFACE:${FIDES_SOURCE_DIR}>
+  $<BUILD_INTERFACE:${FIDES_BINARY_DIR}/fides>
+  $<INSTALL_INTERFACE:${_vtk_build_HEADERS_DESTINATION}/vtkfides/fides>
+  )
+
+include(GenerateExportHeader)
+generate_export_header(fides
+  EXPORT_MACRO_NAME VTKFIDES_EXPORT
+  EXPORT_FILE_NAME vtkfides_export.h)
+endif()
+
+add_subdirectory(xgc)
+add_subdirectory(predefined)
+
+#-------------------------------------------------------------------------------
+# generate the config file that includes the exports
+if (FALSE) # XXX(kitware): VTK handles installation
+include(CMakePackageConfigHelpers)
+configure_package_config_file(${FIDES_SOURCE_DIR}/cmake/Config.cmake.in
+  "${FIDES_BINARY_DIR}/FidesConfig.cmake"
+  INSTALL_DESTINATION lib/cmake/fides
+  NO_SET_AND_CHECK_MACRO
+  NO_CHECK_REQUIRED_COMPONENTS_MACRO
+  )
+
+install(FILES "${FIDES_BINARY_DIR}/FidesConfig.cmake" DESTINATION lib/cmake/fides)
+
+install(TARGETS fides DESTINATION lib
+  EXPORT ${FIDES_EXPORT_NAME})
+
+install(EXPORT ${FIDES_EXPORT_NAME} FILE FidesTargets.cmake DESTINATION lib/cmake/fides)
+
+export(EXPORT ${FIDES_EXPORT_NAME}
+  FILE FidesTargets.cmake
+  )
+
+install(FILES ${headers} ${CMAKE_CURRENT_BINARY_DIR}/fides_export.h
+  DESTINATION ${FIDES_INSTALL_INCLUDE_DIR})
+endif()

fides/CellSet.cxx

+//============================================================================
+//  Copyright (c) Kitware, Inc.
+//  All rights reserved.
+//  See LICENSE.txt for details.
+//  This software is distributed WITHOUT ANY WARRANTY; without even
+//  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+//  PURPOSE.  See the above copyright notice for more information.
+//
+//============================================================================
+
+#include <fides/CellSet.h>
+
+#include <vtkm/cont/Algorithm.h>
+#include <vtkm/CellShape.h>
+#include <vtkm/cont/CellSetExtrude.h>
+
+namespace fides
+{
+namespace datamodel
+{
+
+void CellSet::ProcessJSON(const rapidjson::Value& json,
+                          DataSourcesType& sources)
+{
+  if (!json.HasMember("cell_set_type") || !json["cell_set_type"].IsString())
+  {
+    throw std::runtime_error(
+      this->ObjectName  + " must provide a valid cell_set_type.");
+  }
+  const std::string& cellSetType = json["cell_set_type"].GetString();
+  if (cellSetType == "structured")
+  {
+    this->CellSetImpl.reset(new CellSetStructured());
+  }
+  else if (cellSetType == "single_type")
+  {
+    this->CellSetImpl.reset(new CellSetSingleType());
+  }
+  else if (cellSetType == "explicit")
+  {
+    this->CellSetImpl.reset(new CellSetExplicit());
+  }
+  else if (cellSetType == "xgc")
+  {
+    this->CellSetImpl.reset(new CellSetXGC());
+  }
+  else
+  {
+    throw std::runtime_error(cellSetType + " is not a valid cell_set type.");
+  }
+  this->CellSetImpl->ProcessJSON(json, sources);
+}
+
+std::vector<vtkm::cont::DynamicCellSet> CellSet::Read(
+  const std::unordered_map<std::string, std::string>& paths,
+  DataSourcesType& sources,
+  const fides::metadata::MetaData& selections)
+{
+  return this->CellSetImpl->Read(paths, sources, selections);
+}
+
+void CellSet::PostRead(
+  vtkm::cont::PartitionedDataSet& partitions,
+  const fides::metadata::MetaData& selections)
+{
+  this->CellSetImpl->PostRead(partitions, selections);
+}
+
+void CellSetSingleType::ProcessJSON(const rapidjson::Value& json,
+                                    DataSourcesType& sources)
+{
+  this->CellSetBase::ProcessJSON(json, sources);
+
+  if (!json.HasMember("cell_type"))
+  {
+    throw std::runtime_error(
+      this->ObjectName  + " must provide a cell_type.");
+  }
+  std::string cellType = json["cell_type"].GetString();
+
+  if (cellType == "vertex")
+  {
+    this->CellInformation = std::pair<unsigned char, int>(
+      vtkm::CELL_SHAPE_VERTEX, 1);
+  }
+  else if (cellType == "line")
+  {
+    this->CellInformation = std::pair<unsigned char, int>(
+      vtkm::CELL_SHAPE_LINE, 2);
+  }
+  else if (cellType == "triangle")
+  {
+    this->CellInformation = std::pair<unsigned char, int>(
+      vtkm::CELL_SHAPE_TRIANGLE, 3);
+  }
+  else if (cellType == "quad")
+  {
+    this->CellInformation = std::pair<unsigned char, int>(
+      vtkm::CELL_SHAPE_QUAD, 4);
+  }
+  else if (cellType == "tetrahedron")
+  {
+    this->CellInformation = std::pair<unsigned char, int>(
+      vtkm::CELL_SHAPE_TETRA, 4);
+  }
+  else if (cellType == "hexahedron")
+  {
+    this->CellInformation = std::pair<unsigned char, int>(
+      vtkm::CELL_SHAPE_HEXAHEDRON, 8);
+  }
+  else if (cellType == "wedge")
+  {
+    this->CellInformation = std::pair<unsigned char, int>(
+      vtkm::CELL_SHAPE_WEDGE, 6);
+  }
+  else if (cellType == "pyramid")
+  {
+    this->CellInformation = std::pair<unsigned char, int>(
+      vtkm::CELL_SHAPE_PYRAMID, 5);
+  }
+  else
+  {
+    throw std::runtime_error(
+      "Unrecognized cell type " + cellType);
+  }
+}
+
+std::vector<vtkm::cont::DynamicCellSet> CellSetSingleType::Read(
+  const std::unordered_map<std::string, std::string>& paths,
+  DataSourcesType& sources,
+  const fides::metadata::MetaData& selections)
+{
+  if(this->IsStatic && !this->CellSetCache.empty())
+  {
+    return this->CellSetCache;
+  }
+
+  // Temporarily setting IsStatic to false to avoid
+  // caching the array also.
+  bool isStatic = this->IsStatic;
+  this->IsStatic = false;
+  this->ConnectivityArrays = this->ReadSelf(paths, sources, selections);
+  this->IsStatic = isStatic;
+  std::vector<vtkm::cont::DynamicCellSet> cellSets;
+  size_t nArrays = this->ConnectivityArrays.size();
+  cellSets.reserve(nArrays);
+  for(size_t i=0; i<nArrays; i++)
+  {
+    vtkm::cont::CellSetSingleType<> cellSet;
+    cellSets.push_back(cellSet);
+  }
+  if (this->IsStatic)
+  {
+    this->CellSetCache = cellSets;
+  }
+  return cellSets;
+}
+
+void CellSetSingleType::PostRead(
+  vtkm::cont::PartitionedDataSet& partitions,
+  const fides::metadata::MetaData& fidesNotUsed(selections))
+{
+  size_t nParts = static_cast<size_t>(partitions.GetNumberOfPartitions());
+  for(size_t i=0; i<nParts; i++)
+  {
+    auto& pds = partitions.GetPartition(static_cast<vtkm::Id>(i));
+    vtkm::cont::ArrayHandle<vtkm::Id> connCasted =
+      this->ConnectivityArrays[i].Cast<vtkm::cont::ArrayHandle<vtkm::Id> >();
+    auto& cellSet = pds.GetCellSet().Cast<vtkm::cont::CellSetSingleType<>>();
+    cellSet.Fill(pds.GetNumberOfPoints(),
+                 this->CellInformation.first,
+                 this->CellInformation.second,
+                 connCasted);
+  }
+  if (!this->IsStatic)
+  {
+    this->ConnectivityArrays.clear();
+  }
+}
+
+void CellSetExplicit::ProcessJSON(const rapidjson::Value& json,
+                                    DataSourcesType& sources)
+{
+  if (!json.HasMember("cell_types") || !json["cell_types"].IsObject())
+  {
+    throw std::runtime_error(
+      this->ObjectName  + " must provide a cell_types object.");
+  }
+  this->CellTypes.reset(new Array());
+  const auto& cellTypes = json["cell_types"];
+  this->CellTypes->ProcessJSON(cellTypes, sources);
+
+  if (!json.HasMember("number_of_vertices") || !json["number_of_vertices"].IsObject())
+  {
+    throw std::runtime_error(
+      this->ObjectName  + " must provide a number_of_vertices object.");
+  }
+  this->NumberOfVertices.reset(new Array());
+  const auto& numVertices = json["number_of_vertices"];
+  this->NumberOfVertices->ProcessJSON(numVertices, sources);
+
+  if (!json.HasMember("connectivity") || !json["connectivity"].IsObject())
+  {
+    throw std::runtime_error(
+      this->ObjectName  + " must provide a connectivity object.");
+  }
+  this->Connectivity.reset(new Array());
+  const auto& conn = json["connectivity"];
+  this->Connectivity->ProcessJSON(conn, sources);
+}
+
+std::vector<vtkm::cont::DynamicCellSet> CellSetExplicit::Read(
+  const std::unordered_map<std::string, std::string>& paths,
+  DataSourcesType& sources,
+  const fides::metadata::MetaData& selections)
+{
+  if(this->IsStatic && !this->CellSetCache.empty())
+  {
+    return this->CellSetCache;
+  }
+
+  this->ConnectivityArrays =
+    this->Connectivity->Read(paths, sources, selections);
+  this->NumberOfVerticesArrays =
+    this->NumberOfVertices->Read(paths, sources, selections);
+  this->CellTypesArrays =
+    this->CellTypes->Read(paths, sources, selections);
+
+  std::vector<vtkm::cont::DynamicCellSet> cellSets;
+  size_t nArrays = this->ConnectivityArrays.size();
+  cellSets.reserve(nArrays);
+  for(size_t i=0; i<nArrays; i++)
+  {
+    vtkm::cont::CellSetExplicit<> cellSet;
+    cellSets.push_back(cellSet);
+  }
+  if (this->IsStatic)
+  {
+    this->CellSetCache = cellSets;
+  }
+  return cellSets;
+}
+
+void CellSetExplicit::PostRead(
+  vtkm::cont::PartitionedDataSet& partitions,
+  const fides::metadata::MetaData& fidesNotUsed(selections))
+{
+  size_t nParts = static_cast<size_t>(partitions.GetNumberOfPartitions());
+  for(size_t i=0; i<nParts; i++)
+  {
+    const auto& pds = partitions.GetPartition(static_cast<vtkm::Id>(i));
+    vtkm::cont::ArrayHandle<vtkm::IdComponent> nVertsCasted =
+      this->NumberOfVerticesArrays[i].Cast<vtkm::cont::ArrayHandle<vtkm::IdComponent> >();
+    vtkm::cont::ArrayHandle<vtkm::Id> offsets;
+    vtkm::cont::Algorithm::ScanExtended(
+      vtkm::cont::make_ArrayHandleCast<vtkm::Id, vtkm::cont::ArrayHandle<vtkm::IdComponent>>(nVertsCasted), offsets);
+    vtkm::cont::ArrayHandle<vtkm::Id> connCasted =
+      this->ConnectivityArrays[i].Cast<vtkm::cont::ArrayHandle<vtkm::Id> >();
+    vtkm::cont::ArrayHandle<vtkm::UInt8> typesCasted =
+      this->CellTypesArrays[i].Cast<vtkm::cont::ArrayHandle<vtkm::UInt8> >();
+    auto& cellSet = pds.GetCellSet().Cast<vtkm::cont::CellSetExplicit<>>();
+    cellSet.Fill(pds.GetNumberOfPoints(), typesCasted, connCasted, offsets);
+  }
+  if (!this->IsStatic)
+  {
+    this->ConnectivityArrays.clear();
+    this->NumberOfVerticesArrays.clear();
+    this->CellTypesArrays.clear();
+  }
+}
+
+void CellSetStructured::ProcessJSON(const rapidjson::Value& json,
+                                    DataSourcesType& sources)
+{
+  if (!json.HasMember("dimensions") || !json["dimensions"].IsObject())
+  {
+    throw std::runtime_error(
+      this->ObjectName  + " must provide a dimensions object.");
+  }
+  this->Dimensions.reset(new Value());
+  const auto& dimensions = json["dimensions"];
+  this->Dimensions->ProcessJSON(dimensions, sources);
+}
+
+std::vector<vtkm::cont::DynamicCellSet> CellSetStructured::Read(
+  const std::unordered_map<std::string, std::string>& paths,
+  DataSourcesType& sources,
+  const fides::metadata::MetaData& selections)
+{
+  std::vector<vtkm::cont::VariantArrayHandle> dims =
+    this->Dimensions->Read(paths, sources, selections);
+  std::vector<vtkm::cont::DynamicCellSet> ret;
+  ret.reserve(dims.size());
+  for(const auto& array : dims)
+  {
+    auto dimsB = array.Cast<vtkm::cont::ArrayHandle<size_t> >();
+    auto dimsPortal = dimsB.ReadPortal();
+    vtkm::Id3 dimValues(static_cast<vtkm::Id>(dimsPortal.Get(0)),
+                        static_cast<vtkm::Id>(dimsPortal.Get(1)),
+                        static_cast<vtkm::Id>(dimsPortal.Get(2)));
+    vtkm::cont::CellSetStructured<3> cellSet;
+    cellSet.SetPointDimensions(dimValues);
+    vtkm::Id3 start(static_cast<vtkm::Id>(dimsPortal.Get(3)),
+                    static_cast<vtkm::Id>(dimsPortal.Get(4)),
+                    static_cast<vtkm::Id>(dimsPortal.Get(5)));
+    cellSet.SetGlobalPointIndexStart(start);
+    ret.push_back(cellSet);
+  }
+  return ret;
+}
+
+
+void CellSetXGC::ProcessJSON(const rapidjson::Value& json,
+                                    DataSourcesType& sources)
+{
+    if (!json.HasMember("cells") || !json["cells"].IsObject())
+    {
+      throw std::runtime_error("must provide a cells object for XGC CellSet.");
+    }
+    this->CellConnectivity.reset(new Array());
+    this->CellConnectivity->ProcessJSON(json["cells"], sources);
+
+    if (!json.HasMember("plane_connectivity") || !json["plane_connectivity"].IsObject())
+    {
+      throw std::runtime_error("must provide a plane_connectivity object for XGC CellSet.");
+    }
+    this->PlaneConnectivity.reset(new Array());
+    this->PlaneConnectivity->ProcessJSON(json["plane_connectivity"], sources);
+
+    if (json.HasMember("periodic") && json["periodic"].IsBool())
+    {
+      this->IsPeriodic = json["periodic"].GetBool();
+    }
+}
+
+std::vector<vtkm::cont::DynamicCellSet> CellSetXGC::Read(
+  const std::unordered_map<std::string, std::string>& paths,
+  DataSourcesType& sources,
+  const fides::metadata::MetaData& selections)
+{
+  if(this->IsStatic && !this->CellSetCache.empty())
+  {
+    return this->CellSetCache;
+  }
+
+  if (this->NumberOfPlanes < 0)
+  {
+    this->NumberOfPlanes = this->CommonImpl->GetNumberOfPlanes(paths, sources);
+  }
+
+  fides::metadata::MetaData newSelections = selections;
+  newSelections.Remove(fides::keys::BLOCK_SELECTION());
+
+  std::vector<vtkm::cont::DynamicCellSet> cellSets;
+
+  //load the connect_list
+  std::vector<vtkm::cont::VariantArrayHandle> connectivityVec =
+    this->CellConnectivity->Read(paths, sources, newSelections);
+  if (connectivityVec.size() != 1)
+  {
+    throw std::runtime_error("XGC CellConnectivity should have one Array");
+  }
+
+  using intType = vtkm::cont::ArrayHandle<vtkm::Int32>;
+  intType connectivityAH;
+  if (connectivityVec[0].IsType<intType>())
+  {
+    connectivityVec[0].CopyTo(connectivityAH);
+  }
+  else
+  {
+    throw std::runtime_error(
+      "Only int arrays are supported for XGC cell connectivity.");
+  }
+
+  std::vector<vtkm::cont::VariantArrayHandle> planeConnectivityVec =
+      this->PlaneConnectivity->Read(paths, sources, newSelections);
+
+  if (planeConnectivityVec.size() > 1)
+  {
+    throw std::runtime_error("xgc nextNode is supposed to be included in one array.");
+  }
+  intType planeConnectivityAH;
+  if (planeConnectivityVec[0].IsType<intType>())
+  {
+    planeConnectivityVec[0].CopyTo(planeConnectivityAH);
+  }
+  else
+  {
+    throw std::runtime_error(
+      "Only int arrays are supported for XGC plane connectivity.");
+  }
+
+  auto numPointsPerPlane = planeConnectivityVec[0].GetNumberOfValues();
+  // blocks info doesn't need to be added to the selection for CellSet, since
+  // it's not needed for reading the data
+  std::vector<XGCBlockInfo> blocksInfo;
+  if (selections.Has(fides::keys::BLOCK_SELECTION()))
+  {
+    blocksInfo = this->CommonImpl->GetXGCBlockInfo(
+        selections.Get<fides::metadata::Vector<size_t> >(fides::keys::BLOCK_SELECTION()).Data);
+  }
+  else
+  {
+    blocksInfo = this->CommonImpl->GetXGCBlockInfo(std::vector<size_t>());
+  }
+  if (blocksInfo.empty())
+  {
+    throw std::runtime_error("No XGC block info returned. May want to double check block selection.");
+  }
+
+  for (size_t i = 0; i < blocksInfo.size(); ++i)
+  {
+    const auto& block = blocksInfo[i];
+    auto xgcCell = vtkm::cont::CellSetExtrude(connectivityAH,
+      static_cast<vtkm::Int32>(numPointsPerPlane),
+      static_cast<vtkm::Int32>(block.NumberOfPlanesOwned),
+      planeConnectivityAH, this->IsPeriodic);
+
+    cellSets.push_back(xgcCell);
+  }
+
+  if (this->IsStatic)
+  {
+    this->CellSetCache = cellSets;
+  }
+  return cellSets;
+}
+
+}
+}

fides/CellSet.h

+//============================================================================
+//  Copyright (c) Kitware, Inc.
+//  All rights reserved.
+//  See LICENSE.txt for details.
+//  This software is distributed WITHOUT ANY WARRANTY; without even
+//  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+//  PURPOSE.  See the above copyright notice for more information.
+//
+//============================================================================
+
+#ifndef fides_datamodel_CellSet_H_
+#define fides_datamodel_CellSet_H_
+
+#include <fides/DataModel.h>
+#include <fides/Value.h>
+#include <fides/Array.h>
+#include <fides/xgc/XGCCommon.h>
+
+#include <vtkm/cont/DynamicCellSet.h>
+#include <vtkm/cont/PartitionedDataSet.h>
+
+namespace fides
+{
+namespace datamodel
+{
+
+/// \brief Superclass for all specific cellset implementations.
+///
+/// \c CellSetBase and its subclasses are internal to the \c CellSet
+/// class. They handle specific cellset cases. \c CellSetBase is
+/// an abstract class that establishes the API.
+struct CellSetBase : public DataModelBase
+{
+  /// Reads and returns the cell sets.
+  /// The paths are passed to the \c DataSources to create
+  /// file paths. \c selections restrict the data that is loaded.
+  virtual std::vector<vtkm::cont::DynamicCellSet> Read(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources,
+    const fides::metadata::MetaData& selections) = 0;
+
+  /// This is called after all data is read from disk/buffers,
+  /// enabling any work that needs to access array values and other
+  /// dataset data.
+  virtual void PostRead(
+    vtkm::cont::PartitionedDataSet&,
+    const fides::metadata::MetaData&)
+  {}
+
+  virtual ~CellSetBase() {};
+};
+
+/// \brief Data model object for VTK-m cell sets.
+///
+/// \c fides::datamodel::CellSet is responsible of creating
+/// a VTK-m cell set for each block. Note that this class
+/// acts as a variant in that it will create a class of the
+/// appropriate type to handle the specific cell set type
+/// in use and delegate functionality to that class.
+struct CellSet : public DataModelBase
+{
+  /// Overridden to handle CellSet specific items.
+  void ProcessJSON(const rapidjson::Value& json,
+                   DataSourcesType& sources) override;
+
+  /// Reads and returns the cell sets.
+  /// The paths are passed to the \c DataSources to create
+  /// file paths. \c selections restrict the data that is loaded.
+  std::vector<vtkm::cont::DynamicCellSet> Read(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources,
+    const fides::metadata::MetaData& selections);
+
+  /// This is called after all data is read from disk/buffers,
+  /// enabling any work that needs to access array values and other
+  /// dataset data.
+  void PostRead(
+    vtkm::cont::PartitionedDataSet& partitions,
+    const fides::metadata::MetaData& selections);
+
+private:
+  std::unique_ptr<CellSetBase> CellSetImpl = nullptr;
+};
+
+/// \brief Class to read unstructured grids of single cell type.
+///
+/// This class implements the \c CellSetBase API for reading
+/// \c CellSetSingleType objects.
+struct CellSetSingleType : public CellSetBase
+{
+  /// Overridden to handle CellSetSingleType specific items.
+  void ProcessJSON(const rapidjson::Value& json,
+                   DataSourcesType& sources) override;
+
+  /// Reads and returns the cell sets.
+  /// The paths are passed to the \c DataSources to create
+  /// file paths. \c selections restrict the data that is loaded.
+  std::vector<vtkm::cont::DynamicCellSet> Read(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources,
+    const fides::metadata::MetaData& selections) override;
+
+  /// This is called after all data is read from disk/buffers,
+  /// enabling any work that needs to access array values and other
+  /// dataset data. In this case, it is necessary to know the number
+  /// of points to create the VTK-m cellset.
+  virtual void PostRead(
+    vtkm::cont::PartitionedDataSet& partitions,
+    const fides::metadata::MetaData& selections) override;
+
+protected:
+  std::pair<unsigned char, int> CellInformation;
+  std::vector<vtkm::cont::DynamicCellSet> CellSetCache;
+  std::vector<vtkm::cont::VariantArrayHandle> ConnectivityArrays;
+};
+
+/// \brief Class to read unstructured grids of mixed cell types.
+///
+/// This class implements the \c CellSetBase API for reading
+/// \c CellSetExplicit objects.
+struct CellSetExplicit : public CellSetBase
+{
+  /// Overridden to handle CellSetExplicit specific items.
+  void ProcessJSON(const rapidjson::Value& json,
+                   DataSourcesType& sources) override;
+
+  /// Reads and returns the cell sets.
+  /// The paths are passed to the \c DataSources to create
+  /// file paths. \c selections restrict the data that is loaded.
+  std::vector<vtkm::cont::DynamicCellSet> Read(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources,
+    const fides::metadata::MetaData& selections) override;
+
+  /// This is called after all data is read from disk/buffers,
+  /// enabling any work that needs to access array values and other
+  /// dataset data. In this case, it is necessary to know the number
+  /// of points to create the VTK-m cellset.
+  virtual void PostRead(
+    vtkm::cont::PartitionedDataSet& partitions,
+    const fides::metadata::MetaData& selections) override;
+
+protected:
+  std::vector<vtkm::cont::DynamicCellSet> CellSetCache;
+  std::unique_ptr<Array> CellTypes = nullptr;
+  std::unique_ptr<Array> NumberOfVertices = nullptr;
+  std::unique_ptr<Array> Connectivity = nullptr;
+  std::vector<vtkm::cont::VariantArrayHandle> CellTypesArrays;
+  std::vector<vtkm::cont::VariantArrayHandle> NumberOfVerticesArrays;
+  std::vector<vtkm::cont::VariantArrayHandle> ConnectivityArrays;
+};
+
+/// \brief Class to read structured grids.
+///
+/// This class implements the \c CellSetBase API for reading
+/// \c CellSetStructured objects.
+struct CellSetStructured : public CellSetBase
+{
+  /// Overridden to handle CellSetSingleType specific items.
+  void ProcessJSON(const rapidjson::Value& json,
+                   DataSourcesType& sources) override;
+
+  /// Reads and returns the cell sets.
+  /// The paths are passed to the \c DataSources to create
+  /// file paths. \c selections restrict the data that is loaded.
+  std::vector<vtkm::cont::DynamicCellSet> Read(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources,
+    const fides::metadata::MetaData& selections) override;
+
+private:
+  std::unique_ptr<Value> Dimensions = nullptr;
+};
+
+
+/// \brief Class to read XGC 2.5D cell set.
+///
+/// This class implements the \c CellSetBase API for reading
+/// \c CellSetXGC objects.
+struct CellSetXGC : public CellSetBase
+{
+  CellSetXGC() : CommonImpl(new XGCCommon()) {}
+
+  /// Overridden to handle CellSetXGC specific items.
+  void ProcessJSON(const rapidjson::Value& json,
+                   DataSourcesType& sources) override;
+
+  /// Reads and returns the cell sets.
+  /// The paths are passed to the \c DataSources to create
+  /// file paths. \c selections restrict the data that is loaded.
+  std::vector<vtkm::cont::DynamicCellSet> Read(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources,
+    const fides::metadata::MetaData& selections) override;
+
+private:
+  std::vector<vtkm::cont::DynamicCellSet> CellSetCache;
+  std::unique_ptr<Array> CellConnectivity = nullptr;
+  std::unique_ptr<Array> PlaneConnectivity = nullptr;
+  vtkm::Id NumberOfPlanes = -1;
+  bool IsPeriodic = true;
+  std::unique_ptr<XGCCommon> CommonImpl;
+
+};
+
+}
+}
+
+#endif

fides/CoordinateSystem.cxx

+//============================================================================
+//  Copyright (c) Kitware, Inc.
+//  All rights reserved.
+//  See LICENSE.txt for details.
+//  This software is distributed WITHOUT ANY WARRANTY; without even
+//  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+//  PURPOSE.  See the above copyright notice for more information.
+//
+//============================================================================
+
+#include <fides/CoordinateSystem.h>
+
+namespace fides
+{
+namespace datamodel
+{
+
+void CoordinateSystem::ProcessJSON(const rapidjson::Value& json,
+                                   DataSourcesType& sources)
+{
+  this->Array.reset();
+  if (!json.HasMember("array") || !json["array"].IsObject())
+  {
+    throw std::runtime_error(
+      this->ObjectName  + " must provide an array object.");
+  }
+  this->Array = std::make_shared<fides::datamodel::Array>();
+  this->Array->ObjectName = "array";
+  this->Array->ProcessJSON(json["array"], sources);
+}
+
+size_t CoordinateSystem::GetNumberOfBlocks(
+  const std::unordered_map<std::string, std::string>& paths,
+  DataSourcesType& sources)
+{
+  if (!this->NumberOfBlocksInitialized)
+  {
+    this->NumberOfBlocks =
+      this->Array->GetNumberOfBlocks(paths, sources);
+    this->NumberOfBlocksInitialized = true;
+  }
+  return this->NumberOfBlocks;
+}
+
+std::vector<vtkm::cont::CoordinateSystem> CoordinateSystem::Read(
+  const std::unordered_map<std::string, std::string>& paths,
+  DataSourcesType& sources,
+  const fides::metadata::MetaData& selections)
+{
+  //using CoordinateStorageTypes =
+  //  vtkm::List<vtkm::cont::ArrayHandleUniformPointCoordinates::StorageTag,
+  //                    vtkm::cont::StorageTagBasic>;
+  // using VecType = vtkm::Vec<float, 3>;
+  std::vector<vtkm::cont::VariantArrayHandle> arrays =
+    this->Array->Read(paths, sources, selections);
+  std::vector<vtkm::cont::CoordinateSystem> coordSystems;
+   coordSystems.reserve(arrays.size());
+  // using CoordsHandleType = vtkm::cont::ArrayHandle<VecType>;
+  for(auto& array : arrays)
+  {
+    // auto expandedTypeArray = array.ResetStorageList(CoordinateStorageTypes{});
+    // CoordsHandleType coordsHandle = array.Cast<CoordsHandleType>();
+    // coordSystems.push_back(
+    //   vtkm::cont::CoordinateSystem("coordinates", coordsHandle));
+    coordSystems.push_back(
+      vtkm::cont::CoordinateSystem("coordinates", array));
+  }
+  return coordSystems;
+}
+
+}
+}

fides/CoordinateSystem.h

+//============================================================================
+//  Copyright (c) Kitware, Inc.
+//  All rights reserved.
+//  See LICENSE.txt for details.
+//  This software is distributed WITHOUT ANY WARRANTY; without even
+//  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+//  PURPOSE.  See the above copyright notice for more information.
+//
+//============================================================================
+
+#ifndef fides_datamodel_CoordinateSystem_H_
+#define fides_datamodel_CoordinateSystem_H_
+
+#include <fides/Array.h>
+#include <fides/DataModel.h>
+#include <fides/MetaData.h>
+
+#include <vtkm/cont/CoordinateSystem.h>
+
+namespace fides
+{
+namespace datamodel
+{
+
+/// \brief Data model object for VTK-m coordinate systems.
+///
+/// \c fides::datamodel::CoordinateSystem is responsible of creating
+/// VTK-m coordinate systems by loading data defined by the Fides
+/// data model.
+struct CoordinateSystem : public DataModelBase
+{
+  /// Overridden to handle the undelying Array. The Array
+  /// object determines the actual type of the coordinate system.
+  void ProcessJSON(const rapidjson::Value& json,
+                   DataSourcesType& sources) override;
+
+  /// Reads and returns coordinate systems. The heavy-lifting is
+  /// handled by the underlying Array object.
+  /// The paths are passed to the \c DataSources to create
+  /// file paths. \c selections restrict the data that is loaded.
+  std::vector<vtkm::cont::CoordinateSystem> Read(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources,
+    const fides::metadata::MetaData& selections);
+
+  /// Returns the number of blocks in the underlying Array variable.
+  /// Used by the reader to provide meta-data on blocks.
+  size_t GetNumberOfBlocks(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources);
+
+private:
+  std::shared_ptr<fides::datamodel::Array> Array;
+  bool NumberOfBlocksInitialized = false;
+  size_t NumberOfBlocks = 0;
+};
+
+}
+}
+
+#endif

fides/DataModel.cxx

+//============================================================================
+//  Copyright (c) Kitware, Inc.
+//  All rights reserved.
+//  See LICENSE.txt for details.
+//  This software is distributed WITHOUT ANY WARRANTY; without even
+//  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+//  PURPOSE.  See the above copyright notice for more information.
+//
+//============================================================================
+
+#include <fides/DataModel.h>
+
+#include <fides/DataSource.h>
+
+namespace fides
+{
+namespace datamodel
+{
+
+std::vector<vtkm::cont::VariantArrayHandle> DataModelBase::ReadSelf(
+  const std::unordered_map<std::string, std::string>& paths,
+  DataSourcesType& sources,
+  const fides::metadata::MetaData& selections,
+  fides::io::IsVector isItVector)
+{
+  if(this->IsStatic && !this->Cache.empty())
+  {
+    return this->Cache;
+  }
+  auto itr = paths.find(this->DataSourceName);
+  if (itr == paths.end())
+  {
+    throw std::runtime_error("Could not find data_source with name "
+      + this->DataSourceName + " among the input paths.");
+  }
+  const auto& ds = sources[this->DataSourceName];
+  std::string path = itr->second + ds->FileName;
+  ds->OpenSource(path);
+  std::vector<vtkm::cont::VariantArrayHandle> var =
+    ds->ReadVariable(this->VariableName, selections, isItVector);
+  if (this->IsStatic)
+  {
+    this->Cache = var;
+  }
+  return var;
+}
+
+std::string DataModelBase::FindDataSource(
+  const rapidjson::Value& dataModel,
+  DataSourcesType& sources) const
+{
+  if (!dataModel.HasMember("data_source"))
+  {
+    throw std::runtime_error(this->ObjectName + " must provide a data_source.");
+  }
+  std::string dsname = dataModel["data_source"].GetString();
+  auto iter = sources.find(dsname);
+  if(iter == sources.end())
+  {
+    throw std::runtime_error("data_source." + dsname + " was not found.");
+  }
+  return dsname;
+}
+
+void DataModelBase::ProcessJSON(const rapidjson::Value& json,
+                                DataSourcesType& sources)
+{
+  if (!json.HasMember("variable"))
+  {
+    throw std::runtime_error(
+      this->ObjectName  + " must provide a variable.");
+  }
+  std::string varName = json["variable"].GetString();
+  this->VariableName = varName;
+  this->DataSourceName = this->FindDataSource(json, sources);
+
+  if (json.HasMember("static") && json["static"].IsBool())
+  {
+    if(json["static"].GetBool())
+    {
+      this->IsStatic = true;
+    }
+  }
+}
+
+}
+}

fides/DataModel.h

+//============================================================================
+//  Copyright (c) Kitware, Inc.
+//  All rights reserved.
+//  See LICENSE.txt for details.
+//  This software is distributed WITHOUT ANY WARRANTY; without even
+//  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+//  PURPOSE.  See the above copyright notice for more information.
+//
+//============================================================================
+
+#ifndef fides_datamodel_DataModel_H_
+#define fides_datamodel_DataModel_H_
+
+#include <fides/DataSource.h>
+
+#include <vtkm/cont/VariantArrayHandle.h>
+
+#include <fides_rapidjson.h>
+#include FIDES_RAPIDJSON(rapidjson/document.h)
+
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace fides
+{
+namespace datamodel
+{
+
+using DataSourceType = fides::io::DataSource;
+using DataSourcesType =
+  std::unordered_map<std::string, std::shared_ptr<DataSourceType> >;
+
+/// \brief Superclass for all data model classes.
+///
+/// Data model classes represent different structures that reside
+/// in a VTK-m dataset and that are mapped to different variables
+/// read by data sources. This class provides common basic functionality
+/// to all data model objects. These can be overriden by subclasses.
+struct DataModelBase
+{
+  DataModelBase() = default;
+  DataModelBase(const DataModelBase &other)
+  {
+    if(this != &other)
+    {
+      this->ObjectName = other.ObjectName;
+      this->DataSourceName = other.DataSourceName;
+      this->VariableName = other.VariableName;
+      this->IsStatic = other.IsStatic;
+    }
+  }
+
+  virtual ~DataModelBase() = default;
+
+  /// Perform basic parsing of the JSON object, filling in
+  /// common data members such as the data source and variable
+  /// name.
+  virtual void ProcessJSON(const rapidjson::Value& json,
+                           DataSourcesType& sources);
+
+  std::string ObjectName = "";
+  std::string DataSourceName = "";
+  std::string VariableName = "";
+  // Is the variable time dependent or static.
+  bool IsStatic = false;
+
+protected:
+
+  std::string FindDataSource(
+    const rapidjson::Value& dataModel, DataSourcesType& sources) const;
+
+  // Data reading usually happens through this method, which works
+  // with the data source. This also handles data caching for static
+  // variables.
+  std::vector<vtkm::cont::VariantArrayHandle> ReadSelf(
+    const std::unordered_map<std::string, std::string>& paths,
+    DataSourcesType& sources,
+    const fides::metadata::MetaData& selections,
+    fides::io::IsVector isItVector=fides::io::IsVector::Auto);
+
+  std::vector<vtkm::cont::VariantArrayHandle> Cache;
+};
+
+}
+}
+
+#endif

fides/DataSetReader.h

+//============================================================================
+//  Copyright (c) Kitware, Inc.
+//  All rights reserved.
+//  See LICENSE.txt for details.
+//  This software is distributed WITHOUT ANY WARRANTY; without even
+//  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+//  PURPOSE.  See the above copyright notice for more information.
+//
+//============================================================================
+
+#ifndef fides_io_DataSetReader_h
+#define fides_io_DataSetReader_h
+
+#include <fides/FidesTypes.h>
+#include <fides/MetaData.h>
+#include <fides/FieldDataManager.h>
+
+#include <vtkm/cont/DataSet.h>
+#include <vtkm/cont/PartitionedDataSet.h>
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+
+#include "fides_export.h"
+
+namespace fides
+{
+namespace io
+{
+
+/// \brief General purpose reader for data described by an Fides data model.
+///
+/// \c fides::io::DataSetReader reads data described by an Fides data model
+/// and creates VTK-m datasets. See the Fides schema definition for the
+/// supported data model. \c DataSetReader also supports reading meta-data.
+///
+class FIDES_EXPORT DataSetReader
+{
+public:
+  /*! Input types when setting up the DataSetReader */
+  enum class DataModelInput
+  {
+    JSONFile,    /*!< Brief input is path to a JSON file with the data model */
+    JSONString,  /*!< Brief input is JSON containing the data model stored in a string */
+    BPFile       /*!< Brief input is a BP file that contains attributes that provide details
+                      for the predefined data model to be generated by Fides */
+  };
+
+  DataSetReader();
+
+  /// Constructor to set up the Fides reader
+  /// \param dataModel the value should be 1) a path
+  /// to a JSON file describing the data model to be used by the reader,
+  /// 2) a string containing valid JSON, or 3) a path to a BP file containing attributes
+  /// that Fides can use to generate a data model.
+  /// \param inputType specifies what is stored in the dataModel arg. Optional
+  /// \param params a map of ADIOS engine parameters to be
+  /// used for each data source. Optional
+  /// \sa DataModelInput
+  DataSetReader(const std::string dataModel,
+    DataModelInput inputType = DataModelInput::JSONFile,
+    const Params& params = Params());
+
+  ~DataSetReader();
+
+  /// Checks a bp file for an attribute containing information that Fides can
+  /// use to generate the data model. Static so that it doesn't require setting
+  /// up the DataSetReader first. Useful for applications like ParaView, where
+  /// it wants to check if it can use Fides to read a file without needing to
+  /// configure Fides first.
+  /// \param filename Name of file to check
+  /// \param attrName Name of attribute to look for
+  static bool CheckForDataModelAttribute(
+    const std::string& filename,
+    const std::string& attrName = "Fides_Data_Model");
+
+  /// Sets the parameters for a given data source.
+  /// Currently, only the inline engine requires this to be called, which
+  /// must happen before attempting to read data.
+  /// \param source name of the \c DataSource, which should match a data_sources
+  /// name given in the data model JSON.
+  /// \param params a map of parameters and their values
+  void SetDataSourceParameters(const std::string source,
+    const DataSourceParams& params);
+
+  /// Set the IO for a given \c source. This call should only be used when
+  /// using the inline engine and must be called before attempting to read data or metadata.
+  /// \param source name of the \c DataSource, which should match a data_sources
+  /// name given in the data model JSON.
+  /// \param io pointer to the ADIOS IO object
+  void SetDataSourceIO(const std::string source, void* io);
+
+  /// Read and return meta-data. This includes information such as the
+  /// number of blocks, available fields etc.
+  /// \param paths a map that provides
+  /// the paths (filenames usually) corresponding to each data source.
+  fides::metadata::MetaData ReadMetaData(
+    const std::unordered_map<std::string,
+      std::string>& paths);
+
+  /// Read and return heavy-data.
+  /// \param paths a map that provides
+  /// the paths (filenames usually) corresponding to each data source.
+  /// \param selections provides support for reading a subset of
+  /// the data by providing choices for things such as time and blocks.
+  vtkm::cont::PartitionedDataSet ReadDataSet(
+    const std::unordered_map<std::string,
+      std::string>& paths,
+    const fides::metadata::MetaData& selections);
+
+  /// When reading in streaming mode, this method has to be called before
+  /// reading any meta-data or heavy data. It will also move the reader
+  /// to the next step. Fides will loop on a data source while ADIOS
+  /// reports that it is NotReady, but the user should also check the return
+  /// which could return fides::StepStatus::OK or fides::StepStatus::EndOfStream.
+  /// If EndOfStream, all steps have been read.
+  /// \param paths a map that provides
+  /// the paths (filenames usually) corresponding to each data source.
+  StepStatus PrepareNextStep(
+    const std::unordered_map<std::string, std::string>& paths);
+
+  /// Same as \c ReadDataSet except that it works in streaming mode and
+  /// needs to be preceeded by PrepareStep.
+  /// \param paths a map that provides
+  /// the paths (filenames usually) corresponding to each data source.
+  /// \param selections provides support for reading a subset of
+  /// the data by providing choices for things such as time and blocks.
+  vtkm::cont::PartitionedDataSet ReadStep(
+    const std::unordered_map<std::string,
+      std::string>& paths,
+    const fides::metadata::MetaData& selections);
+
+  /// Get a pointer to the field data manager
+  /// \sa FieldDataManager, FieldData
+  std::shared_ptr<fides::datamodel::FieldDataManager> GetFieldData();
+
+  /// Get std::vector of DataSource names.
+  std::vector<std::string> GetDataSourceNames();
+
+private:
+  class DataSetReaderImpl;
+  std::unique_ptr<DataSetReaderImpl> Impl;
+
+  vtkm::cont::PartitionedDataSet ReadDataSetInternal(
+    const std::unordered_map<std::string,
+      std::string>& paths,
+    const fides::metadata::MetaData& selections);
+};
+
+} // end namespace io
+} // end namespace fides
+
+#endif // fides_io_DataSetReader_h