IOSS 2.0

Iocgns_DecompositionData.h
Go to the documentation of this file.
1 /*
2 * Copyright(C) 1999-2017 National Technology & Engineering Solutions
3 * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
4 * NTESS, the U.S. Government retains certain rights in this software.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are
8 * met:
9 *
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 *
13 * * Redistributions in binary form must reproduce the above
14 * copyright notice, this list of conditions and the following
15 * disclaimer in the documentation and/or other materials provided
16 * with the distribution.
17 *
18 * * Neither the name of NTESS nor the names of its
19 * contributors may be used to endorse or promote products derived
20 * from this software without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 */
34 #ifndef IOCGNS_DECOMPOSITONDATA_H
35 #define IOCGNS_DECOMPOSITONDATA_H
36
37 #include <string>
38 #include <unordered_map>
39 #include <vector>
40
41 #define USE_ROBIN
42 #if defined USE_STD
43 #include <unordered_map>
44 #elif defined USE_HOPSCOTCH
45 #include <hash/bhopscotch_map.h>
46 #elif defined USE_ROBIN
47 #include <hash/robin_map.h>
48 #endif
49
50 #include <cstddef>
51 #include <cstdint>
52
53 #include <Ioss_CodeTypes.h>
54 #include <Ioss_Decomposition.h>
55 #include <Ioss_Field.h>
56 #include <Ioss_MeshType.h>
57 #include <Ioss_PropertyManager.h>
58 #include <Ioss_StructuredBlock.h>
59 #include <cgns/Iocgns_StructuredZoneData.h>
60
61 #include <cgnslib.h>
62
63 #if 0
64 #if !defined(NO_PARMETIS_SUPPORT)
65 #include <parmetis.h>
66 #endif
67 #endif
68
69 #undef MPICPP
70 #if !defined(NO_ZOLTAN_SUPPORT)
71 #include <zoltan_cpp.h>
72 #endif
73 namespace Ioss {
74 class Field;
75 template <typename INT> class Decomposition;
76 } // namespace Ioss
77
78 namespace Iocgns {
79
80 class ZoneData
81 {
82 public:
83 std::string m_name;
84 size_t m_nodeOffset;
85 size_t m_nodeCount;
86 size_t m_elementOffset;
87 };
88
89 class DecompositionDataBase
90 {
91 public:
92 DecompositionDataBase() {}
93
94 virtual ~DecompositionDataBase();
95 virtual void decompose_model(int filePtr, Ioss::MeshType mesh_type) = 0;
96 virtual size_t ioss_node_count() const = 0;
97 virtual size_t ioss_elem_count() const = 0;
98 virtual int int_size() const = 0;
99
100 virtual int spatial_dimension() const = 0;
101 virtual size_t global_node_count() const = 0;
102 virtual size_t global_elem_count() const = 0;
103
104 virtual size_t decomp_node_offset() const = 0;
105 virtual size_t decomp_node_count() const = 0;
106 virtual size_t decomp_elem_offset() const = 0;
107 virtual size_t decomp_elem_count() const = 0;
108
109 virtual std::vector<double> &centroids() = 0;
110
111 virtual size_t get_commset_node_size() const = 0;
112
113 virtual void get_node_coordinates(int filePtr, double *ioss_data,
114 const Ioss::Field &field) const = 0;
115
116 void get_block_connectivity(int filePtr, void *data, int blk_seq) const;
117
118 void get_element_field(int filePtr, int solution_index, int blk_seq, int field_index,
119 double *data) const;
120
121 void get_node_field(int filePtr, int solution_index, int field_index, double *data) const;
122
123 void get_node_entity_proc_data(void *entity_proc, const Ioss::MapContainer &node_map,
124 bool do_map) const;
125
126 template <typename T>
127 void communicate_element_data(T *file_data, T *ioss_data, size_t comp_count) const;
128
129 template <typename T>
130 void communicate_node_data(T *file_data, T *ioss_data, size_t comp_count) const;
131
132 void get_sideset_element_side(int filePtr, const Ioss::SetDecompositionData &sset,
133 void *data) const;
134
135 std::vector<ZoneData> m_zones;
136 std::vector<Ioss::BlockDecompositionData> m_elementBlocks;
137 std::vector<Ioss::SetDecompositionData> m_sideSets;
138 std::vector<Iocgns::StructuredZoneData *> m_structuredZones;
139
140 // Maps nodes shared between zones.
141 // TODO: Currently each processor has same map; need to figure out how to reduce size
142 #if defined USE_STD
143 using ZoneSharedMap = std::unordered_map<cgsize_t, cgsize_t>;
144 #elif defined USE_HOPSCOTCH
145 // using ZoneSharedMap = tsl::hopscotch_map<cgsize_t, cgsize_t>;
146 using ZoneSharedMap = tsl::bhopscotch_map<cgsize_t, cgsize_t>;
147 #elif defined USE_ROBIN
148 using ZoneSharedMap = tsl::robin_map<cgsize_t, cgsize_t>;
149 #endif
150 ZoneSharedMap m_zoneSharedMap;
151 };
152
153 template <typename INT> class DecompositionData : public DecompositionDataBase
154 {
155 public:
156 DecompositionData(const Ioss::PropertyManager &props, MPI_Comm communicator);
157 ~DecompositionData() {}
158
159 int int_size() const { return sizeof(INT); }
160
161 void decompose_model(int filePtr, Ioss::MeshType mesh_type);
162
163 int spatial_dimension() const { return m_decomposition.m_spatialDimension; }
164
165 size_t global_node_count() const { return m_decomposition.global_node_count(); }
166 size_t global_elem_count() const { return m_decomposition.global_elem_count(); }
167
168 size_t ioss_node_count() const { return m_decomposition.ioss_node_count(); }
169 size_t ioss_elem_count() const { return m_decomposition.ioss_elem_count(); }
170
171 size_t decomp_node_offset() const { return m_decomposition.file_node_offset(); }
172 size_t decomp_node_count() const { return m_decomposition.file_node_count(); }
173 size_t decomp_elem_offset() const { return m_decomposition.file_elem_offset(); }
174 size_t decomp_elem_count() const { return m_decomposition.file_elem_count(); }
175
176 std::vector<double> &centroids() { return m_decomposition.m_centroids; }
177
178 template <typename T>
179 void communicate_element_data(T *file_data, T *ioss_data, size_t comp_count) const
180 {
181 m_decomposition.communicate_element_data(file_data, ioss_data, comp_count);
182 }
183
184 void communicate_set_data(INT *file_data, INT *ioss_data, const Ioss::SetDecompositionData &set,
185 size_t comp_count) const
186 {
187 m_decomposition.communicate_set_data(file_data, ioss_data, set, comp_count);
188 }
189
190 template <typename T>
191 void communicate_node_data(T *file_data, T *ioss_data, size_t comp_count) const
192 {
193 m_decomposition.communicate_node_data(file_data, ioss_data, comp_count);
194 }
195
196 template <typename U, typename T>
197 void communicate_block_data(U *file_data, T *ioss_data,
198 const Ioss::BlockDecompositionData &block, size_t comp_count) const
199 {
200 m_decomposition.communicate_block_data(file_data, ioss_data, block, comp_count);
201 }
202
203 void get_block_connectivity(int filePtr, INT *data, int blk_seq) const;
204
205 void get_element_field(int filePtr, int solution_index, int blk_seq, int field_index,
206 double *data) const;
207
208 void get_node_field(int filePtr, int solution_index, int field_index, double *data) const;
209
210 size_t get_commset_node_size() const { return m_decomposition.m_nodeCommMap.size() / 2; }
211
212 void get_sideset_element_side(int filePtr, const Ioss::SetDecompositionData &sset,
213 INT *data) const;
214
215 private:
216 void decompose_structured(int filePtr);
217 void decompose_unstructured(int filePtr);
218
219 void get_sideset_data(int filePtr);
220 void generate_zone_shared_nodes(int filePtr, INT min_node, INT max_node);
221
222 bool i_own_node(size_t node)
223 const // T/F if node with global index node owned by this processors ioss-decomp.
224 {
225 return m_decomposition.i_own_node(node);
226 }
227
228 bool i_own_elem(size_t elem)
229 const // T/F if node with global index elem owned by this processors ioss-decomp.
230 {
231 return m_decomposition.i_own_elem(elem);
232 }
233
234 // global_index is 1-based index into global list of nodes [1..global_node_count]
235 // return value is 1-based index into local list of nodes on this
236 // processor (ioss-decomposition)
237 size_t node_global_to_local(size_t global_index) const
238 {
239 return m_decomposition.node_global_to_local(global_index);
240 }
241
242 size_t elem_global_to_local(size_t global_index) const
243 {
244 return m_decomposition.elem_global_to_local(global_index);
245 }
246
247 void build_global_to_local_elem_map()
248 {
249 return m_decomposition.build_global_to_local_elem_map();
250 }
251
252 void get_element_block_communication()
253 {
254 m_decomposition.get_element_block_communication(m_elementBlocks);
255 }
256
257 void generate_adjacency_list(int fileId, Ioss::Decomposition<INT> &decomposition);
258
259 void calculate_element_centroids(int filePtr, std::vector<double> &centroids);
260
261 void get_shared_node_list() { m_decomposition.get_shared_node_list(); }
262
263 void get_local_node_list() { m_decomposition.get_local_node_list(); }
264
265 void get_file_node_coordinates(int filePtr, int direction, double *ioss_data) const;
266 void get_node_coordinates(int filePtr, double *ioss_data, const Ioss::Field &field) const;
267
268 double m_loadBalanceThreshold{1.4};
269 std::string m_lineDecomposition{};
270
271 public:
272 Ioss::Decomposition<INT> m_decomposition;
273 };
274
275 } // namespace Iocgns
276 #endif
Iocgns::DecompositionDataBase::m_structuredZones
std::vector< Iocgns::StructuredZoneData * > m_structuredZones
Definition: Iocgns_DecompositionData.h:138
Iocgns::DecompositionDataBase::m_zoneSharedMap
ZoneSharedMap m_zoneSharedMap
Definition: Iocgns_DecompositionData.h:150
Iocgns::DecompositionDataBase::m_elementBlocks
std::vector< Ioss::BlockDecompositionData > m_elementBlocks
Definition: Iocgns_DecompositionData.h:136
Ioss::MeshType
MeshType
The mesh type – structured, unstructured, hybrid (future), or unknown.
Definition: Ioss_MeshType.h:39
Iocgns::DecompositionData::generate_adjacency_list
void generate_adjacency_list(int fileId, Ioss::Decomposition< INT > &decomposition)
Definition: Iocgns_DecompositionData.C:606
Iocgns::DecompositionData::decomp_elem_offset
size_t decomp_elem_offset() const
Definition: Iocgns_DecompositionData.h:173
Iocgns::DecompositionDataBase::ioss_node_count
virtual size_t ioss_node_count() const =0
Iocgns::DecompositionData::m_lineDecomposition
std::string m_lineDecomposition
Definition: Iocgns_DecompositionData.h:269
Ioss::Field
Holds metadata for bulk data associated with a GroupingEntity.
Definition: Ioss_Field.h:47
Iocgns::DecompositionData::get_sideset_element_side
void get_sideset_element_side(int filePtr, const Ioss::SetDecompositionData &sset, INT *data) const
Definition: Iocgns_DecompositionData.C:1022
Iocgns::DecompositionData::global_elem_count
size_t global_elem_count() const
Definition: Iocgns_DecompositionData.h:166
Iocgns_StructuredZoneData.h
Iocgns::DecompositionData::decomp_elem_count
size_t decomp_elem_count() const
Definition: Iocgns_DecompositionData.h:174
Iocgns::DecompositionData::decompose_unstructured
void decompose_unstructured(int filePtr)
Definition: Iocgns_DecompositionData.C:367
Iocgns::DecompositionDataBase::global_elem_count
virtual size_t global_elem_count() const =0
Iocgns::DecompositionDataBase::communicate_node_data
void communicate_node_data(T *file_data, T *ioss_data, size_t comp_count) const
Definition: Iocgns_DecompositionData.C:1134
Iocgns::DecompositionDataBase::m_zones
std::vector< ZoneData > m_zones
Definition: Iocgns_DecompositionData.h:135
robin_map.h
Iocgns::ZoneData
Definition: Iocgns_DecompositionData.h:80
Iocgns::DecompositionData::get_element_field
void get_element_field(int filePtr, int solution_index, int blk_seq, int field_index, double *data) const
Definition: Iocgns_DecompositionData.C:1102
Iocgns::DecompositionData::get_node_field
void get_node_field(int filePtr, int solution_index, int field_index, double *data) const
Definition: Iocgns_DecompositionData.C:978
Iocgns::DecompositionData::decomp_node_offset
size_t decomp_node_offset() const
Definition: Iocgns_DecompositionData.h:171
Ioss
The main namespace for the Ioss library.
Definition: Ioad_DatabaseIO.C:66
Iocgns::DecompositionData::DecompositionData
DecompositionData(const Ioss::PropertyManager &props, MPI_Comm communicator)
Definition: Iocgns_DecompositionData.C:215
Iocgns::DecompositionData::i_own_node
bool i_own_node(size_t node) const
Definition: Iocgns_DecompositionData.h:222
Iocgns::DecompositionData
Definition: Iocgns_DecompositionData.h:153
Iocgns::DecompositionDataBase::get_block_connectivity
void get_block_connectivity(int filePtr, void *data, int blk_seq) const
Definition: Iocgns_DecompositionData.C:1195
Iocgns::DecompositionDataBase::get_node_entity_proc_data
void get_node_entity_proc_data(void *entity_proc, const Ioss::MapContainer &node_map, bool do_map) const
Definition: Iocgns_DecompositionData.C:1178
Iocgns::DecompositionDataBase::~DecompositionDataBase
virtual ~DecompositionDataBase()
Definition: Iocgns_DecompositionData.C:1117
Iocgns::DecompositionData::decompose_structured
void decompose_structured(int filePtr)
Definition: Iocgns_DecompositionData.C:260
INT
int INT
Definition: Ioss_StructuredBlock.h:53
Ioss::PropertyManager
A collection of Ioss::Property objects.
Definition: Ioss_PropertyManager.h:49
Iocgns::DecompositionData::build_global_to_local_elem_map
void build_global_to_local_elem_map()
Definition: Iocgns_DecompositionData.h:247
Ioss_StructuredBlock.h
Iocgns::DecompositionData::global_node_count
size_t global_node_count() const
Definition: Iocgns_DecompositionData.h:165
Iocgns::DecompositionData::m_loadBalanceThreshold
double m_loadBalanceThreshold
Definition: Iocgns_DecompositionData.h:268
Ioss::BlockDecompositionData
Definition: Ioss_Decomposition.h:57
Iocgns::DecompositionDataBase::decomp_node_offset
virtual size_t decomp_node_offset() const =0
Iocgns::DecompositionData::get_node_coordinates
void get_node_coordinates(int filePtr, double *ioss_data, const Ioss::Field &field) const
Definition: Iocgns_DecompositionData.C:928
Ioss_MeshType.h
Iocgns::DecompositionDataBase::m_sideSets
std::vector< Ioss::SetDecompositionData > m_sideSets
Definition: Iocgns_DecompositionData.h:137
Iocgns::DecompositionData::get_commset_node_size
size_t get_commset_node_size() const
Definition: Iocgns_DecompositionData.h:210
Iocgns::DecompositionData::i_own_elem
bool i_own_elem(size_t elem) const
Definition: Iocgns_DecompositionData.h:228
Iocgns::DecompositionDataBase::get_element_field
void get_element_field(int filePtr, int solution_index, int blk_seq, int field_index, double *data) const
Definition: Iocgns_DecompositionData.C:1210
Iocgns::DecompositionDataBase::decomp_elem_offset
virtual size_t decomp_elem_offset() const =0
Iocgns::DecompositionData::int_size
int int_size() const
Definition: Iocgns_DecompositionData.h:159
Iocgns::DecompositionData::communicate_element_data
void communicate_element_data(T *file_data, T *ioss_data, size_t comp_count) const
Definition: Iocgns_DecompositionData.h:179
Iocgns::DecompositionData::spatial_dimension
int spatial_dimension() const
Definition: Iocgns_DecompositionData.h:163
Iocgns::DecompositionData::get_file_node_coordinates
void get_file_node_coordinates(int filePtr, int direction, double *ioss_data) const
Definition: Iocgns_DecompositionData.C:884
Iocgns::DecompositionData::decompose_model
void decompose_model(int filePtr, Ioss::MeshType mesh_type)
Definition: Iocgns_DecompositionData.C:236
Ioss_Field.h
Iocgns::DecompositionData::decomp_node_count
size_t decomp_node_count() const
Definition: Iocgns_DecompositionData.h:172
Iocgns::DecompositionDataBase::get_sideset_element_side
void get_sideset_element_side(int filePtr, const Ioss::SetDecompositionData &sset, void *data) const
Definition: Iocgns_DecompositionData.C:1242
Iocgns::ZoneData::m_elementOffset
size_t m_elementOffset
Definition: Iocgns_DecompositionData.h:86
Iocgns::DecompositionData::ioss_node_count
size_t ioss_node_count() const
Definition: Iocgns_DecompositionData.h:168
Iocgns::DecompositionDataBase::DecompositionDataBase
DecompositionDataBase()
Definition: Iocgns_DecompositionData.h:92
Ioss::SetDecompositionData
Definition: Ioss_Decomposition.h:104
Iocgns::DecompositionDataBase
Definition: Iocgns_DecompositionData.h:89
Iocgns::DecompositionDataBase::decomp_elem_count
virtual size_t decomp_elem_count() const =0
Iocgns::DecompositionDataBase::decompose_model
virtual void decompose_model(int filePtr, Ioss::MeshType mesh_type)=0
Ioss_PropertyManager.h
Iocgns::DecompositionData::communicate_set_data
void communicate_set_data(INT *file_data, INT *ioss_data, const Ioss::SetDecompositionData &set, size_t comp_count) const
Definition: Iocgns_DecompositionData.h:184
Iocgns::DecompositionData::ioss_elem_count
size_t ioss_elem_count() const
Definition: Iocgns_DecompositionData.h:169
Iocgns::DecompositionData::get_sideset_data
void get_sideset_data(int filePtr)
Definition: Iocgns_DecompositionData.C:773
Iocgns::DecompositionDataBase::communicate_element_data
void communicate_element_data(T *file_data, T *ioss_data, size_t comp_count) const
Definition: Iocgns_DecompositionData.C:1162
Iocgns::DecompositionData::centroids
std::vector< double > & centroids()
Definition: Iocgns_DecompositionData.h:176
Iocgns::DecompositionDataBase::int_size
virtual int int_size() const =0
Iocgns::DecompositionData::communicate_block_data
void communicate_block_data(U *file_data, T *ioss_data, const Ioss::BlockDecompositionData &block, size_t comp_count) const
Definition: Iocgns_DecompositionData.h:197
tsl::bhopscotch_map
Definition: bhopscotch_map.h:58
Iocgns::DecompositionDataBase::ioss_elem_count
virtual size_t ioss_elem_count() const =0
Iocgns::ZoneData::m_nodeOffset
size_t m_nodeOffset
Definition: Iocgns_DecompositionData.h:84
Iocgns::DecompositionData::m_decomposition
Ioss::Decomposition< INT > m_decomposition
Definition: Iocgns_DecompositionData.h:272
Ioss::Decomposition
Definition: Iocgns_DecompositionData.h:75
Iocgns::DecompositionData::get_shared_node_list
void get_shared_node_list()
Definition: Iocgns_DecompositionData.h:261
anonymous_namespace{cth_pressure_map.C}::data
std::vector< char > data
Definition: cth_pressure_map.C:74
Iocgns::DecompositionDataBase::decomp_node_count
virtual size_t decomp_node_count() const =0
Iocgns::DecompositionData::get_element_block_communication
void get_element_block_communication()
Definition: Iocgns_DecompositionData.h:252
Iocgns::DecompositionData::elem_global_to_local
size_t elem_global_to_local(size_t global_index) const
Definition: Iocgns_DecompositionData.h:242
Iocgns::DecompositionData::get_local_node_list
void get_local_node_list()
Definition: Iocgns_DecompositionData.h:263
Iocgns::DecompositionData::node_global_to_local
size_t node_global_to_local(size_t global_index) const
Definition: Iocgns_DecompositionData.h:237
tsl::robin_map< cgsize_t, cgsize_t >
Iocgns::DecompositionDataBase::get_node_field
void get_node_field(int filePtr, int solution_index, int field_index, double *data) const
Definition: Iocgns_DecompositionData.C:1226
Iocgns::DecompositionData::communicate_node_data
void communicate_node_data(T *file_data, T *ioss_data, size_t comp_count) const
Definition: Iocgns_DecompositionData.h:191
bhopscotch_map.h
Iocgns::DecompositionData::get_block_connectivity
void get_block_connectivity(int filePtr, INT *data, int blk_seq) const
Definition: Iocgns_DecompositionData.C:1066
Iocgns::DecompositionDataBase::centroids
virtual std::vector< double > & centroids()=0
Ioss::MapContainer
std::vector< int64_t > MapContainer
Definition: Ioss_Map.h:48
Iocgns::DecompositionDataBase::global_node_count
virtual size_t global_node_count() const =0
Ioss_Decomposition.h
MPI_Comm
int MPI_Comm
Definition: Ioss_CodeTypes.h:96
Iocgns::DecompositionDataBase::get_commset_node_size
virtual size_t get_commset_node_size() const =0
Iocgns::DecompositionData::calculate_element_centroids
void calculate_element_centroids(int filePtr, std::vector< double > &centroids)
Iocgns::DecompositionDataBase::get_node_coordinates
virtual void get_node_coordinates(int filePtr, double *ioss_data, const Ioss::Field &field) const =0
Iocgns::DecompositionData::~DecompositionData
~DecompositionData()
Definition: Iocgns_DecompositionData.h:157
Iocgns
A namespace for the CGNS database format.
Definition: Iocgns_DatabaseIO.C:444
Iocgns::ZoneData::m_nodeCount
size_t m_nodeCount
Definition: Iocgns_DecompositionData.h:85
Ioss_CodeTypes.h
Iocgns::ZoneData::m_name
std::string m_name
Definition: Iocgns_DecompositionData.h:83
Iocgns::DecompositionData::generate_zone_shared_nodes
void generate_zone_shared_nodes(int filePtr, INT min_node, INT max_node)
Definition: Iocgns_DecompositionData.C:494
Iocgns::DecompositionDataBase::spatial_dimension
virtual int spatial_dimension() const =0