IOSS 2.0

Ioss_ParallelUtils.h
Go to the documentation of this file.
1 // Copyright(C) 1999-2017 National Technology & Engineering Solutions
2 // of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
3 // NTESS, the U.S. Government retains certain rights in this software.
4 //
5 // Redistribution and use in source and binary forms, with or without
6 // modification, are permitted provided that the following conditions are
7 // met:
8 //
9 // * Redistributions of source code must retain the above copyright
10 // notice, this list of conditions and the following disclaimer.
11 //
12 // * Redistributions in binary form must reproduce the above
13 // copyright notice, this list of conditions and the following
14 // disclaimer in the documentation and/or other materials provided
15 // with the distribution.
16 //
17 // * Neither the name of NTESS nor the names of its
18 // contributors may be used to endorse or promote products derived
19 // from this software without specific prior written permission.
20 //
21 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32
33 #ifndef IOSS_Ioss_ParallelUtils_h
34 #define IOSS_Ioss_ParallelUtils_h
35
36 #include <Ioss_CodeTypes.h> // for Int64Vector, IntVector
37 #include <Ioss_Utils.h>
38 #include <cassert>
39 #include <cstddef> // for size_t
40 #include <string> // for string
41 #include <vector> // for vector
42
43 #ifdef SEACAS_HAVE_MPI
44 #include <Ioss_SerializeIO.h>
45 #endif
46
47 namespace Ioss {
48
49 class ParallelUtils
50 {
51 public:
52 explicit ParallelUtils(MPI_Comm the_communicator);
53 ~ParallelUtils() = default;
54
55 // Assignment operator
56 // Copy constructor
57
58 enum MinMax { DO_MAX, DO_MIN, DO_SUM };
59
60 /*!
61 * Returns 'true' if 'name' is defined in the environment.
62 * The value of the environment variable is returned in 'value'.
63 * getenv system call is only done on processor 0.
64 * If '!sync_parallel', then don't push to other processors.
65 */
66 bool get_environment(const std::string &name, std::string &value, bool sync_parallel) const;
67
68 /*!
69 * See if any external properties specified via the
70 * IOSS_PROPERTIES environment variable. If any found, add to
71 * `properties`.
72 */
73 void add_environment_properties(Ioss::PropertyManager &properties);
74
75 /*!
76 * Returns 'true' if 'name' is defined in the environment.
77 * The value of the environment variable is converted to an
78 * integer via the atoi library call and returned in 'value'.
79 * No checking is done to ensure that the environment variable
80 * points to a valid integer.
81 * getenv system call is only done on processor 0.
82 * If '!sync_parallel', then don't push to other processors.
83 */
84 bool get_environment(const std::string &name, int &value, bool sync_parallel) const;
85
86 /*!
87 * Returns 'true' if 'name' is defined in the environment no
88 * matter what the value. Returns false otherwise.
89 * getenv system call is only done on processor 0.
90 * If '!sync_parallel', then don't push to other processors.
91 */
92 bool get_environment(const std::string &name, bool sync_parallel) const;
93
94 std::string decode_filename(const std::string &filename, bool is_parallel) const;
95
96 MPI_Comm communicator() const { return communicator_; }
97 int parallel_size() const;
98 int parallel_rank() const;
99
100 /*!
101 * Global OR of attribute strings, the processors which have no
102 * knowledge of the value should initialize to '0' and the
103 * processors with knowledge set the appropriate values.
104 */
105 void attribute_reduction(int length, char buffer[]) const;
106
107 /*!
108 * Generate a "globally unique id" which is unique over all entities
109 * of a specific type over all processors.
110 * Used by some applications for uniquely identifying an entity.
111 * If `rank` == -1, then use parallel_rank; otherwise use rank
112 */
113 int64_t generate_guid(size_t id, int rank = -1) const;
114
115 /*! Return min, max, average memory used by any process */
116 void memory_stats(int64_t &min, int64_t &max, int64_t &avg) const;
117
118 /*! Return high-water-mark min, max, average memory used by any process */
119 /* May be inaccurate unless system maintains this information */
120 void hwm_memory_stats(int64_t &min, int64_t &max, int64_t &avg) const;
121
122 /*! Vector 'local_counts' contains the number of objects
123 * local to this processor. On exit, global_counts
124 * contains the total number of objects on all processors.
125 * Assumes that ordering is the same on all processors
126 */
127 void global_count(const IntVector &local_counts, IntVector &global_counts) const;
128 void global_count(const Int64Vector &local_counts, Int64Vector &global_counts) const;
129
130 template <typename T> T global_minmax(T local_minmax, MinMax which) const;
131
132 template <typename T>
133 void global_array_minmax(std::vector<T> &local_minmax, MinMax which) const;
134
135 template <typename T> void gather(T my_value, std::vector<T> &result) const;
136 template <typename T> void all_gather(T my_value, std::vector<T> &result) const;
137 template <typename T> void gather(std::vector<T> &my_values, std::vector<T> &result) const;
138 template <typename T> void all_gather(std::vector<T> &my_values, std::vector<T> &result) const;
139 template <typename T>
140 int gather(int vals_count, int size_per_val, std::vector<T> &my_values,
141 std::vector<T> &result) const;
142
143 void progress(const std::string &output) const;
144
145 private:
146 MPI_Comm communicator_;
147 };
148
149 #ifdef SEACAS_HAVE_MPI
150 inline MPI_Datatype mpi_type(double /*dummy*/) { return MPI_DOUBLE; }
151 inline MPI_Datatype mpi_type(float /*dummy*/) { return MPI_FLOAT; }
152 inline MPI_Datatype mpi_type(int /*dummy*/) { return MPI_INT; }
153 inline MPI_Datatype mpi_type(char /*dummy*/) { return MPI_CHAR; }
154 inline MPI_Datatype mpi_type(long int /*dummy*/) { return MPI_LONG_LONG_INT; }
155 inline MPI_Datatype mpi_type(long long int /*dummy*/) { return MPI_LONG_LONG_INT; }
156 inline MPI_Datatype mpi_type(unsigned int /*dummy*/) { return MPI_UNSIGNED; }
157 inline MPI_Datatype mpi_type(unsigned long int /*dummy*/) { return MPI_UNSIGNED_LONG; }
158
159 template <typename T>
160 int MY_Alltoallv64(const std::vector<T> &sendbuf, const std::vector<int64_t> &sendcounts,
161 const std::vector<int64_t> &senddisp, std::vector<T> &recvbuf,
162 const std::vector<int64_t> &recvcounts, const std::vector<int64_t> &recvdisp,
163 MPI_Comm comm)
164 {
165 int processor_count = 0;
166 int my_processor = 0;
167 MPI_Comm_size(comm, &processor_count);
168 MPI_Comm_rank(comm, &my_processor);
169
170 // Verify that all 'counts' can fit in an integer. Symmetric
171 // communication, so recvcounts are sendcounts on another processor.
172 for (int i = 0; i < processor_count; i++) {
173 int snd_cnt = static_cast<int>(sendcounts[i]);
174 if (static_cast<int64_t>(snd_cnt) != sendcounts[i]) {
175 std::ostringstream errmsg;
176 errmsg << "ERROR: The number of items that must be communicated via MPI calls from\n"
177 << " processor " << my_processor << " to processor " << i << " is "
178 << sendcounts[i]
179 << "\n which exceeds the storage capacity of the integers "
180 "used by MPI functions.\n";
181 std::cerr << errmsg.str();
182 exit(EXIT_FAILURE);
183 }
184 }
185
186 size_t pow_2 = Ioss::Utils::power_2(processor_count);
187
188 for (size_t i = 1; i < pow_2; i++) {
189 MPI_Status status{};
190
191 int tag = 24713;
192 size_t exchange_proc = i ^ my_processor;
193 if (exchange_proc < static_cast<size_t>(processor_count)) {
194 int snd_cnt = static_cast<int>(
195 sendcounts[exchange_proc]); // Converts from int64_t to int as needed by mpi
196 int rcv_cnt = static_cast<int>(recvcounts[exchange_proc]);
197 if (static_cast<size_t>(my_processor) < exchange_proc) {
198 MPI_Send((void *)&sendbuf[senddisp[exchange_proc]], snd_cnt, mpi_type(T(0)),
199 exchange_proc, tag, comm);
200 MPI_Recv(&recvbuf[recvdisp[exchange_proc]], rcv_cnt, mpi_type(T(0)), exchange_proc, tag,
201 comm, &status);
202 }
203 else {
204 MPI_Recv(&recvbuf[recvdisp[exchange_proc]], rcv_cnt, mpi_type(T(0)), exchange_proc, tag,
205 comm, &status);
206 MPI_Send((void *)&sendbuf[senddisp[exchange_proc]], snd_cnt, mpi_type(T(0)),
207 exchange_proc, tag, comm);
208 }
209 }
210 }
211
212 // Take care of this processor's data movement...
213 std::copy(&sendbuf[senddisp[my_processor]],
214 &sendbuf[senddisp[my_processor] + sendcounts[my_processor]],
215 &recvbuf[recvdisp[my_processor]]);
216 return 0;
217 }
218
219 template <typename T>
220 int MY_Alltoallv(const std::vector<T> &sendbuf, const std::vector<int64_t> &sendcnts,
221 const std::vector<int64_t> &senddisp, std::vector<T> &recvbuf,
222 const std::vector<int64_t> &recvcnts, const std::vector<int64_t> &recvdisp,
223 MPI_Comm comm)
224 {
225 // Wrapper to handle case where send/recv counts and displacements are 64-bit integers.
226 // Two cases:
227 // 1) They are of type 64-bit integers, but only storing data in the 32-bit integer range.
228 // -- if (sendcnts[#proc-1] + senddisp[#proc-1] < 2^31, then we are ok
229 // 2) They are of type 64-bit integers, and storing data in the 64-bit integer range.
230 // -- call special alltoallv which does point-to-point sends
231 #if 1
232 int processor_count = 0;
233 MPI_Comm_size(comm, &processor_count);
234 size_t max_comm = sendcnts[processor_count - 1] + senddisp[processor_count - 1];
235 size_t one = 1;
236 if (max_comm < one << 31) {
237 // count and displacement data in range, need to copy to integer vector.
238 std::vector<int> send_cnt(sendcnts.begin(), sendcnts.end());
239 std::vector<int> send_dis(senddisp.begin(), senddisp.end());
240 std::vector<int> recv_cnt(recvcnts.begin(), recvcnts.end());
241 std::vector<int> recv_dis(recvdisp.begin(), recvdisp.end());
242 return MPI_Alltoallv((void *)sendbuf.data(), send_cnt.data(), send_dis.data(), mpi_type(T(0)),
243 (void *)recvbuf.data(), recv_cnt.data(), recv_dis.data(), mpi_type(T(0)),
244 comm);
245 }
246 else {
247 #endif
248 // Same as if each processor sent a message to every other process with:
249 // MPI_Send(sendbuf+senddisp[i]*sizeof(sendtype),sendcnts[i], sendtype, i, tag, comm);
250 // And received a message from each processor with a call to:
251 // MPI_Recv(recvbuf+recvdisp[i]*sizeof(recvtype),recvcnts[i], recvtype, i, tag, comm);
252 return MY_Alltoallv64(sendbuf, sendcnts, senddisp, recvbuf, recvcnts, recvdisp, comm);
253 #if 1
254 }
255 #endif
256 }
257
258 template <typename T>
259 int MY_Alltoallv(const std::vector<T> &sendbuf, const std::vector<int> &sendcnts,
260 const std::vector<int> &senddisp, std::vector<T> &recvbuf,
261 const std::vector<int> &recvcnts, const std::vector<int> &recvdisp,
262 MPI_Comm comm)
263 {
264 return MPI_Alltoallv((void *)sendbuf.data(), const_cast<int *>(sendcnts.data()),
265 const_cast<int *>(senddisp.data()), mpi_type(T(0)), recvbuf.data(),
266 const_cast<int *>(recvcnts.data()), const_cast<int *>(recvdisp.data()),
267 mpi_type(T(0)), comm);
268 }
269 #endif
270
271 template <typename T>
272 void ParallelUtils::global_array_minmax(std::vector<T> &local_minmax, MinMax which) const
273 {
274 PAR_UNUSED(local_minmax);
275 PAR_UNUSED(which);
276 #ifdef SEACAS_HAVE_MPI
277 if (parallel_size() > 1 && !local_minmax.empty()) {
278 if (Ioss::SerializeIO::isEnabled() && Ioss::SerializeIO::inBarrier()) {
279 std::ostringstream errmsg;
280 errmsg << "Attempting mpi while in barrier owned by " << Ioss::SerializeIO::getOwner();
281 IOSS_ERROR(errmsg);
282 }
283
284 std::vector<T> maxout(local_minmax.size());
285 MPI_Op oper = MPI_MAX;
286 if (which == Ioss::ParallelUtils::DO_MAX) {
287 oper = MPI_MAX;
288 }
289 else if (which == Ioss::ParallelUtils::DO_MIN) {
290 oper = MPI_MIN;
291 }
292 else if (which == Ioss::ParallelUtils::DO_SUM) {
293 oper = MPI_SUM;
294 }
295
296 const int success =
297 MPI_Allreduce((void *)(local_minmax.data()), maxout.data(),
298 static_cast<int>(local_minmax.size()), mpi_type(T()), oper, communicator_);
299 if (success != MPI_SUCCESS) {
300 std::ostringstream errmsg;
301 errmsg << "Ioss::ParallelUtils::global_array_minmax - MPI_Allreduce failed";
302 IOSS_ERROR(errmsg);
303 }
304 // Now copy back into passed in array...
305 for (size_t i = 0; i < local_minmax.size(); i++) {
306 local_minmax[i] = maxout[i];
307 }
308 }
309 #endif
310 }
311
312 } // namespace Ioss
313 #endif
Ioss_CodeTypes.h
Ioss::ParallelUtils::global_minmax
T global_minmax(T local_minmax, MinMax which) const
Definition: Ioss_ParallelUtils.C:402
Ioss::ParallelUtils::generate_guid
int64_t generate_guid(size_t id, int rank=-1) const
Definition: Ioss_ParallelUtils.C:287
Ioss::Utils::power_2
static int power_2(int count)
Definition: Ioss_Utils.h:211
Ioss::ParallelUtils::attribute_reduction
void attribute_reduction(int length, char buffer[]) const
Definition: Ioss_ParallelUtils.C:304
Ioss
The main namespace for the Ioss library.
Definition: Ioad_DatabaseIO.C:66
PAR_UNUSED
#define PAR_UNUSED(x)
Definition: Ioss_CodeTypes.h:81
Ioss::ParallelUtils::parallel_rank
int parallel_rank() const
Definition: Ioss_ParallelUtils.C:245
Ioss::ParallelUtils::parallel_size
int parallel_size() const
Definition: Ioss_ParallelUtils.C:234
Ioss::ParallelUtils::hwm_memory_stats
void hwm_memory_stats(int64_t &min, int64_t &max, int64_t &avg) const
Definition: Ioss_ParallelUtils.C:270
Ioss::IntVector
std::vector< int > IntVector
Definition: Ioss_CodeTypes.h:43
Ioss::ParallelUtils::decode_filename
std::string decode_filename(const std::string &filename, bool is_parallel) const
Definition: Ioss_ParallelUtils.C:218
Ioss::ParallelUtils::DO_MAX
Definition: Ioss_ParallelUtils.h:58
Ioss::ParallelUtils
Definition: Ioss_ParallelUtils.h:49
Ioss::ParallelUtils::DO_MIN
Definition: Ioss_ParallelUtils.h:58
Ioss_SerializeIO.h
Ioss::SerializeIO::isEnabled
static bool isEnabled()
Definition: Ioss_SerializeIO.h:84
Ioss_Utils.h
Ioss::ParallelUtils::global_array_minmax
void global_array_minmax(std::vector< T > &local_minmax, MinMax which) const
Definition: Ioss_ParallelUtils.h:272
Ioss::Int64Vector
std::vector< int64_t > Int64Vector
Definition: Ioss_CodeTypes.h:44
Ioss::ParallelUtils::memory_stats
void memory_stats(int64_t &min, int64_t &max, int64_t &avg) const
Definition: Ioss_ParallelUtils.C:256
Ioss::ParallelUtils::DO_SUM
Definition: Ioss_ParallelUtils.h:58
Ioss::ParallelUtils::add_environment_properties
void add_environment_properties(Ioss::PropertyManager &properties)
Definition: Ioss_ParallelUtils.C:78
Ioss::ParallelUtils::MinMax
MinMax
Definition: Ioss_ParallelUtils.h:58
Ioss::ParallelUtils::communicator
MPI_Comm communicator() const
Definition: Ioss_ParallelUtils.h:96
Ioss::ParallelUtils::progress
void progress(const std::string &output) const
Definition: Ioss_ParallelUtils.C:513
anonymous_namespace{Iocgns_DecompositionData.C}::rank
int rank
Definition: Iocgns_DecompositionData.C:55
Ioss::SerializeIO::getOwner
static int getOwner()
Definition: Ioss_SerializeIO.h:72
anonymous_namespace{io_info.C}::name
std::string name(Ioss::GroupingEntity *entity)
Definition: io_info.C:89
Ioss::ParallelUtils::global_count
void global_count(const IntVector &local_counts, IntVector &global_counts) const
Definition: Ioss_ParallelUtils.C:326
Ioss::ParallelUtils::get_environment
bool get_environment(const std::string &name, std::string &value, bool sync_parallel) const
Definition: Ioss_ParallelUtils.C:126
Ioss::SerializeIO::inBarrier
static bool inBarrier()
Definition: Ioss_SerializeIO.h:86
Ioss::ParallelUtils::~ParallelUtils
~ParallelUtils()=default
Ioss::ParallelUtils::ParallelUtils
ParallelUtils(MPI_Comm the_communicator)
Definition: Ioss_ParallelUtils.C:76
MPI_Comm
int MPI_Comm
Definition: Ioss_CodeTypes.h:88
Ioss::ParallelUtils::communicator_
MPI_Comm communicator_
Definition: Ioss_ParallelUtils.h:146
Ioss::ParallelUtils::gather
void gather(T my_value, std::vector< T > &result) const
Definition: Ioss_ParallelUtils.C:446
Ioss::PropertyManager
A collection of Ioss::Property objects.
Definition: Ioss_PropertyManager.h:49
Ioss::ParallelUtils::all_gather
void all_gather(T my_value, std::vector< T > &result) const
Definition: Ioss_ParallelUtils.C:469
IOSS_ERROR
#define IOSS_ERROR(errmsg)
Definition: Ioss_Utils.h:65