IOSS 2.0

Iopx_DecompositionData.h
Go to the documentation of this file.
1 /*
2 * Copyright(C) 1999-2017 National Technology & Engineering Solutions
3 * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
4 * NTESS, the U.S. Government retains certain rights in this software.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are
8 * met:
9 *
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 *
13 * * Redistributions in binary form must reproduce the above
14 * copyright notice, this list of conditions and the following
15 * disclaimer in the documentation and/or other materials provided
16 * with the distribution.
17 *
18 * * Neither the name of NTESS nor the names of its
19 * contributors may be used to endorse or promote products derived
20 * from this software without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 */
34 #ifndef IOPX_DECOMPOSITONDATA_H
35 #define IOPX_DECOMPOSITONDATA_H
36
37 #include <Ioss_CodeTypes.h>
38 #include <vector>
39 #if !defined(NO_PARMETIS_SUPPORT)
40 #include <parmetis.h>
41 #endif
42
43 #undef MPICPP
44 #if !defined(NO_ZOLTAN_SUPPORT)
45 #include <zoltan_cpp.h>
46 #endif
47 #include <Ioss_Decomposition.h>
48 #include <Ioss_Map.h>
49 #include <Ioss_PropertyManager.h>
50 #include <exodusII.h>
51
52 namespace Ioss {
53 class Field;
54 }
55 namespace Iopx {
56
57 class DecompositionDataBase
58 {
59 public:
60 DecompositionDataBase(MPI_Comm comm) : comm_(comm), m_processor(0), m_processorCount(0) {}
61
62 virtual ~DecompositionDataBase() {}
63 virtual int int_size() const = 0;
64 virtual void decompose_model(int filePtr) = 0;
65 virtual size_t ioss_node_count() const = 0;
66 virtual size_t ioss_elem_count() const = 0;
67
68 virtual int spatial_dimension() const = 0;
69 virtual size_t global_node_count() const = 0;
70 virtual size_t global_elem_count() const = 0;
71
72 virtual size_t decomp_node_offset() const = 0;
73 virtual size_t decomp_node_count() const = 0;
74 virtual size_t decomp_elem_offset() const = 0;
75 virtual size_t decomp_elem_count() const = 0;
76
77 virtual std::vector<double> &centroids() = 0;
78
79 MPI_Comm comm_;
80
81 int m_processor;
82 int m_processorCount;
83
84 std::vector<Ioss::BlockDecompositionData> el_blocks;
85 std::vector<Ioss::SetDecompositionData> node_sets;
86 std::vector<Ioss::SetDecompositionData> side_sets;
87
88 const Ioss::SetDecompositionData &get_decomp_set(ex_entity_type type, ex_entity_id id) const;
89
90 template <typename T>
91 void communicate_node_data(T *file_data, T *ioss_data, size_t comp_count) const;
92
93 template <typename T>
94 void communicate_element_data(T *file_data, T *ioss_data, size_t comp_count) const;
95
96 void get_block_connectivity(int filePtr, void *data, int64_t id, size_t blk_seq,
97 size_t nnpe) const;
98
99 void get_node_entity_proc_data(void *entity_proc, const Ioss::MapContainer &node_map,
100 bool do_map) const;
101
102 int get_set_mesh_var(int filePtr, ex_entity_type type, ex_entity_id id,
103 const Ioss::Field &field, void *ioss_data) const;
104
105 int get_set_mesh_double(int filePtr, ex_entity_type type, ex_entity_id id,
106 const Ioss::Field &field, double *ioss_data) const;
107
108 virtual size_t get_commset_node_size() const = 0;
109
110 virtual int get_node_coordinates(int filePtr, double *ioss_data,
111 const Ioss::Field &field) const = 0;
112 virtual int get_one_attr(int exoid, ex_entity_type obj_type, ex_entity_id obj_id,
113 int attrib_index, double *attrib) const = 0;
114 virtual int get_attr(int exoid, ex_entity_type obj_type, ex_entity_id obj_id, size_t attr_count,
115 double *attrib) const = 0;
116 virtual int get_var(int filePtr, int step, ex_entity_type type, int var_index, ex_entity_id id,
117 int64_t num_entity, std::vector<double> &data) const = 0;
118 };
119
120 template <typename INT> class DecompositionData : public DecompositionDataBase
121 {
122 public:
123 DecompositionData(const Ioss::PropertyManager &props, MPI_Comm communicator);
124 ~DecompositionData() {}
125
126 int int_size() const { return sizeof(INT); }
127
128 void decompose_model(int filePtr);
129
130 int spatial_dimension() const { return m_decomposition.m_spatialDimension; }
131
132 size_t global_node_count() const { return m_decomposition.global_node_count(); }
133 size_t global_elem_count() const { return m_decomposition.global_elem_count(); }
134
135 size_t ioss_node_count() const { return m_decomposition.ioss_node_count(); }
136 size_t ioss_elem_count() const { return m_decomposition.ioss_elem_count(); }
137
138 size_t decomp_node_offset() const { return m_decomposition.file_node_offset(); }
139 size_t decomp_node_count() const { return m_decomposition.file_node_count(); }
140 size_t decomp_elem_offset() const { return m_decomposition.file_elem_offset(); }
141 size_t decomp_elem_count() const { return m_decomposition.file_elem_count(); }
142
143 std::vector<double> &centroids() { return m_decomposition.m_centroids; }
144
145 template <typename T>
146 void communicate_element_data(T *file_data, T *ioss_data, size_t comp_count) const
147 {
148 m_decomposition.communicate_element_data(file_data, ioss_data, comp_count);
149 }
150
151 template <typename T>
152 void communicate_set_data(T *file_data, T *ioss_data, const Ioss::SetDecompositionData &set,
153 size_t comp_count) const
154 {
155 m_decomposition.communicate_set_data(file_data, ioss_data, set, comp_count);
156 }
157
158 template <typename T>
159 void communicate_node_data(T *file_data, T *ioss_data, size_t comp_count) const
160 {
161 m_decomposition.communicate_node_data(file_data, ioss_data, comp_count);
162 }
163
164 void get_block_connectivity(int filePtr, INT *data, int64_t id, size_t blk_seq,
165 size_t nnpe) const;
166 size_t get_commset_node_size() const { return m_decomposition.m_nodeCommMap.size() / 2; }
167
168 int get_attr(int filePtr, ex_entity_type obj_type, ex_entity_id id, size_t attr_count,
169 double *attrib) const;
170 int get_one_attr(int filePtr, ex_entity_type obj_type, ex_entity_id id, int attrib_index,
171 double *attrib) const;
172
173 int get_var(int filePtr, int step, ex_entity_type type, int var_index, ex_entity_id id,
174 int64_t num_entity, std::vector<double> &data) const;
175
176 template <typename T>
177 int get_set_mesh_var(int filePtr, ex_entity_type type, ex_entity_id id,
178 const Ioss::Field &field, T *ioss_data) const;
179
180 size_t get_block_seq(ex_entity_type type, ex_entity_id id) const;
181 size_t get_block_element_count(size_t blk_seq) const;
182 size_t get_block_element_offset(size_t blk_seq) const;
183
184 void create_implicit_global_map(const std::vector<int> &owning_proc,
185 std::vector<int64_t> &global_implicit_map, Ioss::Map &node_map,
186 int64_t *locally_owned_count, int64_t *processor_offset);
187
188 private:
189 #if !defined(NO_ZOLTAN_SUPPORT)
190 void zoltan_decompose(const std::string &method);
191 #endif
192
193 #if !defined(NO_PARMETIS_SUPPORT)
194 void metis_decompose(const std::string &method, const std::vector<INT> &element_dist);
195
196 void internal_metis_decompose(const std::string &method, idx_t *element_dist, idx_t *pointer,
197 idx_t *adjacency, idx_t *elem_partition);
198 #endif
199
200 void simple_decompose(const std::string &method, const std::vector<INT> &element_dist)
201 {
202 m_decomposition.simple_decompose(method, element_dist);
203 }
204
205 void simple_node_decompose(const std::string &method, const std::vector<INT> &node_dist);
206
207 template <typename T>
208 int handle_sset_df(int filePtr, ex_entity_id id, const Ioss::Field &field, T *ioss_data) const;
209
210 int get_one_set_attr(int filePtr, ex_entity_type type, ex_entity_id id, int attr_index,
211 double *ioss_data) const;
212 int get_one_node_attr(int filePtr, ex_entity_id id, int attr_index, double *ioss_data) const;
213 int get_one_elem_attr(int filePtr, ex_entity_id id, int attr_index, double *ioss_data) const;
214
215 int get_set_attr(int filePtr, ex_entity_type type, ex_entity_id id, size_t comp_count,
216 double *ioss_data) const;
217 int get_node_attr(int filePtr, ex_entity_id id, size_t comp_count, double *ioss_data) const;
218 int get_elem_attr(int filePtr, ex_entity_id id, size_t comp_count, double *ioss_data) const;
219
220 int get_node_var(int filePtr, int step, int var_index, ex_entity_id id, int64_t num_entity,
221 std::vector<double> &ioss_data) const;
222
223 int get_elem_var(int filePtr, int step, int var_index, ex_entity_id id, int64_t num_entity,
224 std::vector<double> &ioss_data) const;
225
226 int get_set_var(int filePtr, int step, int var_index, ex_entity_type type, ex_entity_id id,
227 int64_t num_entity, std::vector<double> &ioss_data) const;
228
229 bool i_own_node(size_t node)
230 const // T/F if node with global index node owned by this processors ioss-decomp.
231 {
232 return m_decomposition.i_own_node(node);
233 }
234
235 bool i_own_elem(size_t elem)
236 const // T/F if node with global index elem owned by this processors ioss-decomp.
237 {
238 return m_decomposition.i_own_elem(elem);
239 }
240
241 // global_index is 1-based index into global list of nodes [1..global_node_count]
242 // return value is 1-based index into local list of nodes on this
243 // processor (ioss-decomposition)
244 size_t node_global_to_local(size_t global_index) const
245 {
246 return m_decomposition.node_global_to_local(global_index);
247 }
248
249 size_t elem_global_to_local(size_t global_index) const
250 {
251 return m_decomposition.elem_global_to_local(global_index);
252 }
253
254 void build_global_to_local_elem_map()
255 {
256 return m_decomposition.build_global_to_local_elem_map();
257 }
258
259 void get_element_block_communication()
260 {
261 m_decomposition.get_element_block_communication(el_blocks);
262 }
263
264 void generate_adjacency_list(int filePtr, Ioss::Decomposition<INT> &decomposition);
265
266 void get_nodeset_data(int filePtr, size_t set_count);
267
268 void get_sideset_data(int filePtr, size_t set_count);
269
270 void calculate_element_centroids(int filePtr, const std::vector<INT> &node_dist);
271 #if !defined(NO_ZOLTAN_SUPPORT)
272 void get_local_element_list(const ZOLTAN_ID_PTR &export_global_ids, size_t export_count);
273 #endif
274
275 void get_shared_node_list() { m_decomposition.get_shared_node_list(); }
276
277 int get_node_coordinates(int filePtr, double *ioss_data, const Ioss::Field &field) const;
278
279 void get_local_node_list() { m_decomposition.get_local_node_list(); }
280
281 public:
282 Ioss::Decomposition<INT> m_decomposition;
283 };
284 } // namespace Iopx
285 #endif
Ioss_CodeTypes.h
Iopx::DecompositionData::get_element_block_communication
void get_element_block_communication()
Definition: Iopx_DecompositionData.h:259
Iopx::DecompositionDataBase
Definition: Iopx_DecompositionData.h:57
Iopx::DecompositionData::get_shared_node_list
void get_shared_node_list()
Definition: Iopx_DecompositionData.h:275
Iopx::DecompositionDataBase::m_processor
int m_processor
Definition: Iopx_DecompositionData.h:81
Ioss
The main namespace for the Ioss library.
Definition: Iocgns_DatabaseIO.h:50
Iopx::DecompositionData::node_global_to_local
size_t node_global_to_local(size_t global_index) const
Definition: Iopx_DecompositionData.h:244
Iopx::DecompositionData::ioss_elem_count
size_t ioss_elem_count() const
Definition: Iopx_DecompositionData.h:136
Ioss::SetDecompositionData
Definition: Ioss_Decomposition.h:104
Iopx::DecompositionData::i_own_elem
bool i_own_elem(size_t elem) const
Definition: Iopx_DecompositionData.h:235
Ioss_Map.h
Iopx::DecompositionData::get_commset_node_size
size_t get_commset_node_size() const
Definition: Iopx_DecompositionData.h:166
Iopx::DecompositionData::spatial_dimension
int spatial_dimension() const
Definition: Iopx_DecompositionData.h:130
Iopx::DecompositionDataBase::DecompositionDataBase
DecompositionDataBase(MPI_Comm comm)
Definition: Iopx_DecompositionData.h:60
Ioss_PropertyManager.h
Iopx::DecompositionDataBase::node_sets
std::vector< Ioss::SetDecompositionData > node_sets
Definition: Iopx_DecompositionData.h:85
Iopx::DecompositionData::decomp_node_offset
size_t decomp_node_offset() const
Definition: Iopx_DecompositionData.h:138
Iopx::DecompositionData::get_local_node_list
void get_local_node_list()
Definition: Iopx_DecompositionData.h:279
Iopx::DecompositionData::elem_global_to_local
size_t elem_global_to_local(size_t global_index) const
Definition: Iopx_DecompositionData.h:249
Iopx::DecompositionDataBase::comm_
MPI_Comm comm_
Definition: Iopx_DecompositionData.h:79
Iopx::DecompositionDataBase::el_blocks
std::vector< Ioss::BlockDecompositionData > el_blocks
Definition: Iopx_DecompositionData.h:84
Iopx::DecompositionData::communicate_node_data
void communicate_node_data(T *file_data, T *ioss_data, size_t comp_count) const
Definition: Iopx_DecompositionData.h:159
Iopx::DecompositionData::int_size
int int_size() const
Definition: Iopx_DecompositionData.h:126
Iopx::DecompositionData::simple_decompose
void simple_decompose(const std::string &method, const std::vector< INT > &element_dist)
Definition: Iopx_DecompositionData.h:200
Iopx::DecompositionData
Definition: Iopx_DatabaseIO.h:57
Ioss::MapContainer
std::vector< int64_t > MapContainer
Definition: Ioss_Map.h:48
Ioss_Decomposition.h
Iopx::DecompositionData::communicate_set_data
void communicate_set_data(T *file_data, T *ioss_data, const Ioss::SetDecompositionData &set, size_t comp_count) const
Definition: Iopx_DecompositionData.h:152
Iopx::DecompositionData::decomp_node_count
size_t decomp_node_count() const
Definition: Iopx_DecompositionData.h:139
Iopx::DecompositionData::communicate_element_data
void communicate_element_data(T *file_data, T *ioss_data, size_t comp_count) const
Definition: Iopx_DecompositionData.h:146
Iopx::DecompositionData::global_elem_count
size_t global_elem_count() const
Definition: Iopx_DecompositionData.h:133
Iopx::DecompositionData::global_node_count
size_t global_node_count() const
Definition: Iopx_DecompositionData.h:132
INT
int INT
Definition: Ioss_StructuredBlock.h:53
Ioss::Map
Definition: Ioss_Map.h:52
Iopx::DecompositionData::decomp_elem_count
size_t decomp_elem_count() const
Definition: Iopx_DecompositionData.h:141
Ioss::Decomposition
Definition: Iocgns_DecompositionData.h:65
Iopx::DecompositionDataBase::side_sets
std::vector< Ioss::SetDecompositionData > side_sets
Definition: Iopx_DecompositionData.h:86
Iopx
A namespace for the decompose-on-the-fly version of the parallel exodus database format.
Definition: Iopx_DatabaseIO.C:249
Iopx::DecompositionData::m_decomposition
Ioss::Decomposition< INT > m_decomposition
Definition: Iopx_DecompositionData.h:282
Iopx::DecompositionData::ioss_node_count
size_t ioss_node_count() const
Definition: Iopx_DecompositionData.h:135
Iopx::DecompositionData::build_global_to_local_elem_map
void build_global_to_local_elem_map()
Definition: Iopx_DecompositionData.h:254
Iopx::DecompositionDataBase::~DecompositionDataBase
virtual ~DecompositionDataBase()
Definition: Iopx_DecompositionData.h:62
Iopx::DecompositionData::centroids
std::vector< double > & centroids()
Definition: Iopx_DecompositionData.h:143
Ioss::Field
Holds metadata for bulk data associated with a GroupingEntity.
Definition: Ioss_Field.h:47
Iopx::DecompositionData::decomp_elem_offset
size_t decomp_elem_offset() const
Definition: Iopx_DecompositionData.h:140
MPI_Comm
int MPI_Comm
Definition: Ioss_CodeTypes.h:79
Ioss::PropertyManager
A collection of Ioss::Property objects.
Definition: Ioss_PropertyManager.h:49
Iopx::DecompositionDataBase::m_processorCount
int m_processorCount
Definition: Iopx_DecompositionData.h:82
Iopx::DecompositionData::~DecompositionData
~DecompositionData()
Definition: Iopx_DecompositionData.h:124
Iopx::DecompositionData::i_own_node
bool i_own_node(size_t node) const
Definition: Iopx_DecompositionData.h:229
anonymous_namespace{Ioss_Utils.C}::data
std::vector< char > data
Definition: Ioss_Utils.C:78