IOSS 2.0

Ioss_Decomposition.h
Go to the documentation of this file.
1 /*
2 * Copyright(C) 1999-2017 National Technology & Engineering Solutions
3 * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
4 * NTESS, the U.S. Government retains certain rights in this software.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are
8 * met:
9 *
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 *
13 * * Redistributions in binary form must reproduce the above
14 * copyright notice, this list of conditions and the following
15 * disclaimer in the documentation and/or other materials provided
16 * with the distribution.
17 *
18 * * Neither the name of NTESS nor the names of its
19 * contributors may be used to endorse or promote products derived
20 * from this software without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 */
34 #ifndef IOSS_DECOMPOSITON_H
35 #define IOSS_DECOMPOSITON_H
36
37 #include <Ioss_CodeTypes.h>
38 #include <Ioss_Map.h>
39 #include <Ioss_ParallelUtils.h>
40 #include <Ioss_PropertyManager.h>
41 #include <algorithm>
42 #include <assert.h>
43 #include <map>
44 #include <string>
45 #include <vector>
46
47 #if !defined(NO_PARMETIS_SUPPORT)
48 #include <parmetis.h>
49 #endif
50
51 #if !defined(NO_ZOLTAN_SUPPORT)
52 #undef MPICPP
53 #include <zoltan_cpp.h>
54 #endif
55
56 namespace Ioss {
57 class BlockDecompositionData
58 {
59 public:
60 BlockDecompositionData() = default;
61
62 const std::string &name() const { return name_; }
63 int zone() const { return zone_; }
64 int section() const { return section_; }
65 int64_t id() const { return id_; }
66 size_t file_count() const { return fileCount; }
67 size_t ioss_count() const { return iossCount; }
68 size_t global_count() const { return globalCount; }
69
70 std::string name_{};
71 int zone_{0};
72 int section_{0};
73
74 size_t fileSectionOffset{0}; // In partial read, where start
75 int64_t id_{0};
76 size_t fileCount{0};
77 size_t iossCount{0};
78 size_t globalCount{0};
79
80 size_t zoneNodeOffset{0};
81 std::string topologyType{"unknown"};
82 int nodesPerEntity{0};
83 int attributeCount{0};
84
85 // maps from file-block data to ioss-block data
86 // The local_map.size() elements starting at localIossOffset are local.
87 // ioss[localIossOffset+i] = file[local_map[i]];
88 size_t localIossOffset{0};
89 std::vector<int> localMap;
90
91 // Maps from file-block data to export list.
92 // export[i] = file[export_map[i]
93 std::vector<int> exportMap;
94 std::vector<int> exportCount;
95 std::vector<int> exportIndex;
96
97 // Maps from import data to ioss-block data.
98 // ioss[import_map[i] = local_map[i];
99 std::vector<int> importMap;
100 std::vector<int> importCount;
101 std::vector<int> importIndex;
102 };
103
104 class SetDecompositionData
105 {
106 public:
107 SetDecompositionData() = default;
108
109 const std::string &name() const { return name_; }
110 int64_t id() const { return id_; }
111 int zone() const { return zone_; }
112 int section() const { return section_; }
113 size_t file_count() const { return fileCount; }
114 size_t ioss_count() const { return entitylist_map.size(); }
115 size_t df_count() const { return distributionFactorCount; }
116
117 // contains global entity-list positions for all entities in this set on this processor.
118 std::vector<size_t> entitylist_map;
119 std::vector<bool> hasEntities; // T/F if this set exists on processor p
120
121 std::string name_{};
122 int64_t id_{0};
123 int zone_{0};
124 int section_{0};
125 size_t fileCount{0}; // Number of nodes in nodelist for file decomposition
126 int root_{0}; // Lowest number processor that has nodes for this nodest
127 std::string topologyType{};
128 size_t parentBlockIndex{0};
129
130 int distributionFactorValsPerEntity{-1}; // number of df / element or node. -1 if nonconstant.
131 size_t distributionFactorCount{0};
132 double distributionFactorValue{
133 0.0}; // If distributionFactorConstant == true, the constant value
134 bool distributionFactorConstant{false}; // T if all distribution factors the same value.
135 };
136
137 template <typename INT> class Decomposition
138 {
139 public:
140 Decomposition(const Ioss::PropertyManager &props, MPI_Comm comm);
141
142 size_t global_node_count() const { return m_globalNodeCount; }
143 size_t global_elem_count() const { return m_globalElementCount; }
144 size_t ioss_node_count() const { return nodeGTL.size(); }
145 size_t ioss_elem_count() const { return localElementMap.size() + importElementMap.size(); }
146 size_t file_node_count() const { return m_nodeCount; }
147 size_t file_elem_count() const { return m_elementCount; }
148 size_t file_node_offset() const { return m_nodeOffset; }
149 size_t file_elem_offset() const { return m_elementOffset; }
150
151 bool needs_centroids() const;
152
153 void generate_entity_distributions(size_t globalNodeCount, size_t globalElementCount);
154
155 // T/F if node with global index node owned by this processors ioss-decomp.
156 bool i_own_node(size_t global_index) const
157 {
158 // global_index is 1-based index into global list of nodes [1..global_node_count]
159 return std::binary_search(nodeGTL.begin(), nodeGTL.end(), global_index);
160 }
161
162 // T/F if element with global index elem owned by this processors ioss-decomp.
163 bool i_own_elem(size_t global_index) const
164 {
165 // global_index is 1-based index into global list of elements [1..global_element_count]
166 return elemGTL.count(global_index) != 0;
167 }
168
169 size_t node_global_to_local(size_t global_index) const
170 {
171 // global_index is 1-based index into global list of nodes [1..global_node_count]
172 // return value is 1-based index into local list of nodes on this
173 // processor (ioss-decomposition)
174 // Note that for 'int', equivalence and equality are the same, so
175 // lower_bound is OK here (EffectiveSTL, Item 19)
176 typename std::vector<INT>::const_iterator I =
177 lower_bound(nodeGTL.begin(), nodeGTL.end(), global_index);
178 assert(I != nodeGTL.end());
179 return std::distance(nodeGTL.begin(), I) + 1; // Convert to 1-based index.
180 }
181
182 size_t elem_global_to_local(size_t global_index) const
183 {
184 // global_index is 1-based index into global list of elements [1..global_node_count]
185 // return value is 1-based index into local list of elements on this
186 // processor (ioss-decomposition)
187 typename std::map<INT, INT>::const_iterator I = elemGTL.find(global_index);
188 assert(I != elemGTL.end());
189 return I->second;
190 }
191
192 void show_progress(const std::string &message) const
193 {
194 if (m_showProgress) {
195 // Use the output below for debugging...
196 // std::cerr << "[" << m_processor << "].... " << message << "\n";
197 Ioss::ParallelUtils pu(m_comm);
198 pu.progress(message);
199 }
200 }
201
202 void decompose_model(
203 #if !defined(NO_ZOLTAN_SUPPORT)
204 Zoltan &zz,
205 #endif
206 std::vector<BlockDecompositionData> &element_blocks);
207
208 void simple_decompose();
209
210 void simple_node_decompose();
211
212 void calculate_element_centroids(const std::vector<double> &x, const std::vector<double> &y,
213 const std::vector<double> &z);
214
215 #if !defined(NO_ZOLTAN_SUPPORT)
216 void zoltan_decompose(Zoltan &zz);
217
218 void get_local_element_list(const ZOLTAN_ID_PTR &export_global_ids, size_t export_count);
219 #endif
220
221 #if !defined(NO_PARMETIS_SUPPORT)
222 void metis_decompose(idx_t *pointer, idx_t *adjacency,
223 std::vector<BlockDecompositionData> &el_blocks);
224
225 void internal_metis_decompose(std::vector<BlockDecompositionData> &el_blocks,
226 idx_t *element_dist, idx_t *pointer, idx_t *adjacency,
227 idx_t *elem_partition);
228 #endif
229
230 void get_node_entity_proc_data(INT *entity_proc, const Ioss::MapContainer &node_map,
231 bool do_map) const;
232
233 void get_element_block_communication(std::vector<BlockDecompositionData> &el_blocks);
234 void build_global_to_local_elem_map();
235 void get_local_node_list();
236 void get_shared_node_list();
237 template <typename T>
238 void communicate_element_data(T *file_data, T *ioss_data, size_t comp_count) const;
239
240 template <typename T>
241 void communicate_set_data(T *file_data, T *ioss_data, const SetDecompositionData &set,
242 size_t comp_count) const;
243
244 template <typename T, typename U>
245 void communicate_block_data(T *file_data, U *ioss_data, const BlockDecompositionData &block,
246 size_t comp_count) const;
247
248 template <typename T>
249 void communicate_node_data(T *file_data, T *ioss_data, size_t comp_count) const;
250
251 MPI_Comm m_comm;
252 int m_processor{};
253 int m_processorCount{};
254 std::string m_method;
255
256 // Values for the file decomposition
257 int m_spatialDimension{3};
258 size_t m_globalElementCount{0};
259 size_t m_elementCount{0};
260 size_t m_elementOffset{0};
261 size_t m_importPreLocalElemIndex{0};
262
263 size_t m_globalNodeCount{0};
264 size_t m_nodeCount{0};
265 size_t m_nodeOffset{0};
266 size_t m_importPreLocalNodeIndex{0};
267
268 bool m_retainFreeNodes{true};
269 bool m_showProgress{false};
270 bool m_showHWM{false};
271
272 std::vector<double> m_centroids;
273 std::vector<INT> m_pointer; // Index into adjacency, processor list for each element...
274 std::vector<INT> m_adjacency; // Size is sum of element connectivity sizes
275
276 std::vector<INT> m_nodeCommMap; // node/processor pair of the
277 // nodes I communicate with. Stored node#,proc,node#,proc, ...
278
279 // The global element at index 'I' (0-based) is on block B in the
280 // file decomposition.
281 // if m_fileBlockIndex[B] <= I && m_fileBlockIndex[B+1] < I
282 std::vector<size_t> m_fileBlockIndex;
283
284 private:
285 // This processor "manages" the elements on the exodus mesh file from
286 // element_offset to element_offset+count (0-based). This is
287 // 'file' data
288 //
289 // This processor also appears to the Ioss clients to own other
290 // element and node data based on the decomposition. This is the
291 // 'ioss' data.
292 //
293 // The indices in 'local_element_map' are the elements that are
294 // common to both the 'file' data and the 'ioss' data.
295 // local_element_map[i] contains the location in 'file' data for
296 // the 'ioss' data at location 'i+import_pre_local_elem_index'
297 //
298 // local_element_map[i]+m_elementOffset is the 0-based global index
299 //
300 // The indices in 'import_element_map' map the data received via
301 // mpi communication from other processors into 'ioss' data.
302 // if 'ind=import_element_map[i]', then ioss[ind] = comm_recv[i]
303 // Note that this is the reverse direction of the
304 // local_element_map mapping.
305 //
306 // The indices in 'export_element_map' are used to pull from
307 // 'file' data into the comm_send vector. if 'ind =
308 // export_element_map[i]', then 'comm_send[i] = file[ind]' for i =
309 // 0..#exported_elements
310 //
311 // local_element_map.size() + import_element_map.size() == #
312 // ioss elements on this processor.
313 //
314 // local_element_map.size() + export_element_map.size() == #
315 // file elements on this processor.
316 //
317 // export_element_map and import_element_map are sorted.
318 // The primary key is processor order followed by global id.
319 // The processor association is via 'export_proc_disp' and
320 // 'import_proc_disp' Both are of size '#processors+1' and
321 // the elements for processor p range from [X_proc_disp[p] to
322 // X_proc_disp[p+1])
323
324 std::vector<INT> localElementMap;
325
326 std::vector<INT> importElementMap;
327 std::vector<INT> importElementCount;
328 std::vector<INT> importElementIndex;
329
330 std::vector<INT> exportElementMap;
331 std::vector<INT> exportElementCount;
332 std::vector<INT> exportElementIndex;
333
334 std::vector<INT> nodeIndex;
335
336 std::vector<INT> exportNodeMap;
337 std::vector<INT> exportNodeCount;
338 std::vector<INT> exportNodeIndex;
339
340 std::vector<INT>
341 importNodeMap; // Where to put each imported nodes data in the list of all data...
342 std::vector<INT> importNodeCount;
343 std::vector<INT> importNodeIndex;
344
345 std::vector<INT> localNodeMap;
346
347 std::vector<INT> m_elementDist;
348 std::vector<INT> m_nodeDist;
349
350 // Note that nodeGTL is a sorted vector.
351 std::vector<INT> nodeGTL; // Convert from global index to local index (1-based)
352 std::map<INT, INT> elemGTL; // Convert from global index to local index (1-based)
353 };
354 } // namespace Ioss
355 #endif
Ioss_CodeTypes.h
Ioss::Decomposition::m_pointer
std::vector< INT > m_pointer
Definition: Ioss_Decomposition.h:273
Ioss::Decomposition::exportNodeCount
std::vector< INT > exportNodeCount
Definition: Ioss_Decomposition.h:337
Ioss::SetDecompositionData::file_count
size_t file_count() const
Definition: Ioss_Decomposition.h:113
Ioss::Decomposition::importElementIndex
std::vector< INT > importElementIndex
Definition: Ioss_Decomposition.h:328
Ioss::Decomposition::exportNodeIndex
std::vector< INT > exportNodeIndex
Definition: Ioss_Decomposition.h:338
Ioss::BlockDecompositionData::id_
int64_t id_
Definition: Ioss_Decomposition.h:75
Ioss::Decomposition::show_progress
void show_progress(const std::string &message) const
Definition: Ioss_Decomposition.h:192
Ioss::BlockDecompositionData::attributeCount
int attributeCount
Definition: Ioss_Decomposition.h:83
Ioss::BlockDecompositionData::fileSectionOffset
size_t fileSectionOffset
Definition: Ioss_Decomposition.h:74
Ioss
The main namespace for the Ioss library.
Definition: Iocgns_DatabaseIO.h:50
Ioss::Decomposition::file_node_offset
size_t file_node_offset() const
Definition: Ioss_Decomposition.h:148
Ioss::Decomposition::nodeGTL
std::vector< INT > nodeGTL
Definition: Ioss_Decomposition.h:351
Ioss::BlockDecompositionData::section
int section() const
Definition: Ioss_Decomposition.h:64
Ioss::BlockDecompositionData::topologyType
std::string topologyType
Definition: Ioss_Decomposition.h:81
Ioss::Decomposition::nodeIndex
std::vector< INT > nodeIndex
Definition: Ioss_Decomposition.h:334
Ioss::Decomposition::m_adjacency
std::vector< INT > m_adjacency
Definition: Ioss_Decomposition.h:274
Ioss::BlockDecompositionData::nodesPerEntity
int nodesPerEntity
Definition: Ioss_Decomposition.h:82
Ioss::SetDecompositionData
Definition: Ioss_Decomposition.h:104
Ioss::Decomposition::localNodeMap
std::vector< INT > localNodeMap
Definition: Ioss_Decomposition.h:345
Ioss::Decomposition::ioss_elem_count
size_t ioss_elem_count() const
Definition: Ioss_Decomposition.h:145
Ioss::Decomposition::importNodeIndex
std::vector< INT > importNodeIndex
Definition: Ioss_Decomposition.h:343
Ioss::BlockDecompositionData::file_count
size_t file_count() const
Definition: Ioss_Decomposition.h:66
Ioss::BlockDecompositionData::BlockDecompositionData
BlockDecompositionData()=default
Ioss::Decomposition::m_nodeDist
std::vector< INT > m_nodeDist
Definition: Ioss_Decomposition.h:348
Ioss_Map.h
Ioss::Decomposition::file_node_count
size_t file_node_count() const
Definition: Ioss_Decomposition.h:146
Ioss::Decomposition::importElementCount
std::vector< INT > importElementCount
Definition: Ioss_Decomposition.h:327
Ioss::Decomposition::exportElementMap
std::vector< INT > exportElementMap
Definition: Ioss_Decomposition.h:330
Ioss_PropertyManager.h
Ioss::BlockDecompositionData::zone_
int zone_
Definition: Ioss_Decomposition.h:71
Ioss::Decomposition::importNodeMap
std::vector< INT > importNodeMap
Definition: Ioss_Decomposition.h:341
Ioss::Decomposition::m_comm
MPI_Comm m_comm
Definition: Ioss_Decomposition.h:251
Ioss::Decomposition::exportElementCount
std::vector< INT > exportElementCount
Definition: Ioss_Decomposition.h:331
Ioss::ParallelUtils
Definition: Ioss_ParallelUtils.h:45
Ioss::Decomposition::m_method
std::string m_method
Definition: Ioss_Decomposition.h:254
Ioss::Decomposition::i_own_node
bool i_own_node(size_t global_index) const
Definition: Ioss_Decomposition.h:156
Ioss::BlockDecompositionData::id
int64_t id() const
Definition: Ioss_Decomposition.h:65
Ioss::BlockDecompositionData::localIossOffset
size_t localIossOffset
Definition: Ioss_Decomposition.h:88
Ioss::BlockDecompositionData::importCount
std::vector< int > importCount
Definition: Ioss_Decomposition.h:100
Ioss::BlockDecompositionData::ioss_count
size_t ioss_count() const
Definition: Ioss_Decomposition.h:67
Ioss::SetDecompositionData::entitylist_map
std::vector< size_t > entitylist_map
Definition: Ioss_Decomposition.h:118
Ioss::BlockDecompositionData::exportIndex
std::vector< int > exportIndex
Definition: Ioss_Decomposition.h:95
Ioss::Decomposition::global_elem_count
size_t global_elem_count() const
Definition: Ioss_Decomposition.h:143
Ioss::BlockDecompositionData::global_count
size_t global_count() const
Definition: Ioss_Decomposition.h:68
Ioss::SetDecompositionData::zone
int zone() const
Definition: Ioss_Decomposition.h:111
Ioss::Decomposition::file_elem_offset
size_t file_elem_offset() const
Definition: Ioss_Decomposition.h:149
Ioss::MapContainer
std::vector< int64_t > MapContainer
Definition: Ioss_Map.h:48
Ioss::BlockDecompositionData::localMap
std::vector< int > localMap
Definition: Ioss_Decomposition.h:89
Ioss::SetDecompositionData::section
int section() const
Definition: Ioss_Decomposition.h:112
Ioss::SetDecompositionData::name
const std::string & name() const
Definition: Ioss_Decomposition.h:109
Ioss::BlockDecompositionData::name
const std::string & name() const
Definition: Ioss_Decomposition.h:62
Ioss::Decomposition::ioss_node_count
size_t ioss_node_count() const
Definition: Ioss_Decomposition.h:144
Ioss::Decomposition::node_global_to_local
size_t node_global_to_local(size_t global_index) const
Definition: Ioss_Decomposition.h:169
Ioss::BlockDecompositionData::exportCount
std::vector< int > exportCount
Definition: Ioss_Decomposition.h:94
INT
int INT
Definition: Ioss_StructuredBlock.h:53
Ioss::Decomposition
Definition: Iocgns_DecompositionData.h:65
Ioss::ParallelUtils::progress
void progress(const std::string &output) const
Definition: Ioss_ParallelUtils.C:512
Ioss::SetDecompositionData::ioss_count
size_t ioss_count() const
Definition: Ioss_Decomposition.h:114
Ioss::Decomposition::m_fileBlockIndex
std::vector< size_t > m_fileBlockIndex
Definition: Ioss_Decomposition.h:282
Ioss::BlockDecompositionData::name_
std::string name_
Definition: Ioss_Decomposition.h:70
Ioss_ParallelUtils.h
Ioss::SetDecompositionData::hasEntities
std::vector< bool > hasEntities
Definition: Ioss_Decomposition.h:119
Ioss::BlockDecompositionData::fileCount
size_t fileCount
Definition: Ioss_Decomposition.h:76
Ioss::Decomposition::i_own_elem
bool i_own_elem(size_t global_index) const
Definition: Ioss_Decomposition.h:163
Ioss::BlockDecompositionData::iossCount
size_t iossCount
Definition: Ioss_Decomposition.h:77
Ioss::BlockDecompositionData::section_
int section_
Definition: Ioss_Decomposition.h:72
Ioss::BlockDecompositionData::zone
int zone() const
Definition: Ioss_Decomposition.h:63
Ioss::BlockDecompositionData
Definition: Ioss_Decomposition.h:57
Ioss::Decomposition::m_elementDist
std::vector< INT > m_elementDist
Definition: Ioss_Decomposition.h:347
Ioss::Decomposition::file_elem_count
size_t file_elem_count() const
Definition: Ioss_Decomposition.h:147
Ioss::Decomposition::exportElementIndex
std::vector< INT > exportElementIndex
Definition: Ioss_Decomposition.h:332
Ioss::BlockDecompositionData::importMap
std::vector< int > importMap
Definition: Ioss_Decomposition.h:99
MPI_Comm
int MPI_Comm
Definition: Ioss_CodeTypes.h:79
Ioss::Decomposition::importElementMap
std::vector< INT > importElementMap
Definition: Ioss_Decomposition.h:326
Ioss::Decomposition::m_nodeCommMap
std::vector< INT > m_nodeCommMap
Definition: Ioss_Decomposition.h:276
Ioss::Decomposition::global_node_count
size_t global_node_count() const
Definition: Ioss_Decomposition.h:142
Ioss::Decomposition::exportNodeMap
std::vector< INT > exportNodeMap
Definition: Ioss_Decomposition.h:336
Ioss::BlockDecompositionData::exportMap
std::vector< int > exportMap
Definition: Ioss_Decomposition.h:93
Ioss::Decomposition::elem_global_to_local
size_t elem_global_to_local(size_t global_index) const
Definition: Ioss_Decomposition.h:182
Ioss::SetDecompositionData::id
int64_t id() const
Definition: Ioss_Decomposition.h:110
Ioss::PropertyManager
A collection of Ioss::Property objects.
Definition: Ioss_PropertyManager.h:49
Ioss::Decomposition::localElementMap
std::vector< INT > localElementMap
Definition: Ioss_Decomposition.h:324
Ioss::BlockDecompositionData::globalCount
size_t globalCount
Definition: Ioss_Decomposition.h:78
Ioss::BlockDecompositionData::zoneNodeOffset
size_t zoneNodeOffset
Definition: Ioss_Decomposition.h:80
Ioss::Decomposition::m_centroids
std::vector< double > m_centroids
Definition: Ioss_Decomposition.h:272
Ioss::BlockDecompositionData::importIndex
std::vector< int > importIndex
Definition: Ioss_Decomposition.h:101
Ioss::Decomposition::importNodeCount
std::vector< INT > importNodeCount
Definition: Ioss_Decomposition.h:342
Ioss::SetDecompositionData::df_count
size_t df_count() const
Definition: Ioss_Decomposition.h:115
Ioss::Decomposition::elemGTL
std::map< INT, INT > elemGTL
Definition: Ioss_Decomposition.h:352