IOSS 2.0

Ioss_ParallelUtils.h
Go to the documentation of this file.
1 // Copyright(C) 1999-2017 National Technology & Engineering Solutions
2 // of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
3 // NTESS, the U.S. Government retains certain rights in this software.
4 //
5 // Redistribution and use in source and binary forms, with or without
6 // modification, are permitted provided that the following conditions are
7 // met:
8 //
9 // * Redistributions of source code must retain the above copyright
10 // notice, this list of conditions and the following disclaimer.
11 //
12 // * Redistributions in binary form must reproduce the above
13 // copyright notice, this list of conditions and the following
14 // disclaimer in the documentation and/or other materials provided
15 // with the distribution.
16 //
17 // * Neither the name of NTESS nor the names of its
18 // contributors may be used to endorse or promote products derived
19 // from this software without specific prior written permission.
20 //
21 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32
33 #ifndef IOSS_Ioss_ParallelUtils_h
34 #define IOSS_Ioss_ParallelUtils_h
35
36 #include <Ioss_CodeTypes.h> // for Int64Vector, IntVector
37 #include <Ioss_Utils.h>
38 #include <cassert>
39 #include <cstddef> // for size_t
40 #include <string> // for string
41 #include <vector> // for vector
42
43 namespace Ioss {
44
45 class ParallelUtils
46 {
47 public:
48 explicit ParallelUtils(MPI_Comm the_communicator);
49 ~ParallelUtils() = default;
50
51 // Assignment operator
52 // Copy constructor
53
54 enum MinMax { DO_MAX, DO_MIN, DO_SUM };
55
56 /*!
57 * Returns 'true' if 'name' is defined in the environment.
58 * The value of the environment variable is returned in 'value'.
59 * getenv system call is only done on processor 0.
60 * If '!sync_parallel', then don't push to other processors.
61 */
62 bool get_environment(const std::string &name, std::string &value, bool sync_parallel) const;
63
64 /*!
65 * See if any external properties specified via the
66 * IOSS_PROPERTIES environment variable. If any found, add to
67 * `properties`. If `do_print` then output to cerr which
68 * properties were set
69 */
70 void add_environment_properties(Ioss::PropertyManager &properties, bool do_print);
71
72 /*!
73 * Returns 'true' if 'name' is defined in the environment.
74 * The value of the environment variable is converted to an
75 * integer via the atoi library call and returned in 'value'.
76 * No checking is done to ensure that the environment variable
77 * points to a valid integer.
78 * getenv system call is only done on processor 0.
79 * If '!sync_parallel', then don't push to other processors.
80 */
81 bool get_environment(const std::string &name, int &value, bool sync_parallel) const;
82
83 /*!
84 * Returns 'true' if 'name' is defined in the environment no
85 * matter what the value. Returns false otherwise.
86 * getenv system call is only done on processor 0.
87 * If '!sync_parallel', then don't push to other processors.
88 */
89 bool get_environment(const std::string &name, bool sync_parallel) const;
90
91 std::string decode_filename(const std::string &filename, bool is_parallel) const;
92
93 MPI_Comm communicator() const { return communicator_; }
94 int parallel_size() const;
95 int parallel_rank() const;
96
97 /*!
98 * Global OR of attribute strings, the processors which have no
99 * knowledge of the value should initialize to '0' and the
100 * processors with knowledge set the appropriate values.
101 */
102 void attribute_reduction(int length, char buffer[]) const;
103
104 /*!
105 * Generate a "globally unique id" which is unique over all entities
106 * of a specific type over all processors.
107 * Used by some applications for uniquely identifying an entity.
108 * If `rank` == -1, then use parallel_rank; otherwise use rank
109 */
110 int64_t generate_guid(size_t id, int rank = -1) const;
111
112 /*! Return min, max, average memory used by any process */
113 void memory_stats(int64_t &min, int64_t &max, int64_t &avg) const;
114
115 /*! Return high-water-mark min, max, average memory used by any process */
116 /* May be inaccurate unless system maintains this information */
117 void hwm_memory_stats(int64_t &min, int64_t &max, int64_t &avg) const;
118
119 /*! Vector 'local_counts' contains the number of objects
120 * local to this processor. On exit, global_counts
121 * contains the total number of objects on all processors.
122 * Assumes that ordering is the same on all processors
123 */
124 void global_count(const IntVector &local_counts, IntVector &global_counts) const;
125 void global_count(const Int64Vector &local_counts, Int64Vector &global_counts) const;
126
127 template <typename T> T global_minmax(T local_minmax, MinMax which) const;
128 template <typename T>
129 void global_array_minmax(std::vector<T> &local_minmax, MinMax which) const;
130 template <typename T>
131 void global_array_minmax(T *local_minmax, size_t count, MinMax which) const;
132
133 template <typename T> void gather(T my_value, std::vector<T> &result) const;
134 template <typename T> void all_gather(T my_value, std::vector<T> &result) const;
135 template <typename T> void gather(std::vector<T> &my_values, std::vector<T> &result) const;
136 template <typename T> void all_gather(std::vector<T> &my_values, std::vector<T> &result) const;
137 template <typename T>
138 int gather(int vals_count, int size_per_val, std::vector<T> &my_values,
139 std::vector<T> &result) const;
140
141 void progress(const std::string &output) const;
142
143 private:
144 MPI_Comm communicator_;
145 };
146
147 #ifdef SEACAS_HAVE_MPI
148 inline MPI_Datatype mpi_type(double /*dummy*/) { return MPI_DOUBLE; }
149 inline MPI_Datatype mpi_type(float /*dummy*/) { return MPI_FLOAT; }
150 inline MPI_Datatype mpi_type(int /*dummy*/) { return MPI_INT; }
151 inline MPI_Datatype mpi_type(char /*dummy*/) { return MPI_CHAR; }
152 inline MPI_Datatype mpi_type(long int /*dummy*/) { return MPI_LONG_LONG_INT; }
153 inline MPI_Datatype mpi_type(long long int /*dummy*/) { return MPI_LONG_LONG_INT; }
154 inline MPI_Datatype mpi_type(unsigned int /*dummy*/) { return MPI_UNSIGNED; }
155 inline MPI_Datatype mpi_type(unsigned long int /*dummy*/) { return MPI_UNSIGNED_LONG; }
156
157 template <typename T>
158 int MY_Alltoallv64(const std::vector<T> &sendbuf, const std::vector<int64_t> &sendcounts,
159 const std::vector<int64_t> &senddisp, std::vector<T> &recvbuf,
160 const std::vector<int64_t> &recvcounts, const std::vector<int64_t> &recvdisp,
161 MPI_Comm comm)
162 {
163 int processor_count = 0;
164 int my_processor = 0;
165 MPI_Comm_size(comm, &processor_count);
166 MPI_Comm_rank(comm, &my_processor);
167
168 // Verify that all 'counts' can fit in an integer. Symmetric
169 // communication, so recvcounts are sendcounts on another processor.
170 for (int i = 0; i < processor_count; i++) {
171 int snd_cnt = static_cast<int>(sendcounts[i]);
172 if (static_cast<int64_t>(snd_cnt) != sendcounts[i]) {
173 std::ostringstream errmsg;
174 errmsg << "ERROR: The number of items that must be communicated via MPI calls from\n"
175 << " processor " << my_processor << " to processor " << i << " is "
176 << sendcounts[i]
177 << "\n which exceeds the storage capacity of the integers "
178 "used by MPI functions.\n";
179 std::cerr << errmsg.str();
180 exit(EXIT_FAILURE);
181 }
182 }
183
184 size_t pow_2 = Ioss::Utils::power_2(processor_count);
185
186 for (size_t i = 1; i < pow_2; i++) {
187 MPI_Status status{};
188
189 int tag = 24713;
190 size_t exchange_proc = i ^ my_processor;
191 if (exchange_proc < static_cast<size_t>(processor_count)) {
192 int snd_cnt = static_cast<int>(
193 sendcounts[exchange_proc]); // Converts from int64_t to int as needed by mpi
194 int rcv_cnt = static_cast<int>(recvcounts[exchange_proc]);
195 if (static_cast<size_t>(my_processor) < exchange_proc) {
196 MPI_Send((void *)&sendbuf[senddisp[exchange_proc]], snd_cnt, mpi_type(T(0)),
197 exchange_proc, tag, comm);
198 MPI_Recv(&recvbuf[recvdisp[exchange_proc]], rcv_cnt, mpi_type(T(0)), exchange_proc, tag,
199 comm, &status);
200 }
201 else {
202 MPI_Recv(&recvbuf[recvdisp[exchange_proc]], rcv_cnt, mpi_type(T(0)), exchange_proc, tag,
203 comm, &status);
204 MPI_Send((void *)&sendbuf[senddisp[exchange_proc]], snd_cnt, mpi_type(T(0)),
205 exchange_proc, tag, comm);
206 }
207 }
208 }
209
210 // Take care of this processor's data movement...
211 std::copy(&sendbuf[senddisp[my_processor]],
212 &sendbuf[senddisp[my_processor] + sendcounts[my_processor]],
213 &recvbuf[recvdisp[my_processor]]);
214 return 0;
215 }
216
217 template <typename T>
218 int MY_Alltoallv(const std::vector<T> &sendbuf, const std::vector<int64_t> &sendcnts,
219 const std::vector<int64_t> &senddisp, std::vector<T> &recvbuf,
220 const std::vector<int64_t> &recvcnts, const std::vector<int64_t> &recvdisp,
221 MPI_Comm comm)
222 {
223 // Wrapper to handle case where send/recv counts and displacements are 64-bit integers.
224 // Two cases:
225 // 1) They are of type 64-bit integers, but only storing data in the 32-bit integer range.
226 // -- if (sendcnts[#proc-1] + senddisp[#proc-1] < 2^31, then we are ok
227 // 2) They are of type 64-bit integers, and storing data in the 64-bit integer range.
228 // -- call special alltoallv which does point-to-point sends
229 #if 1
230 int processor_count = 0;
231 MPI_Comm_size(comm, &processor_count);
232 size_t max_comm = sendcnts[processor_count - 1] + senddisp[processor_count - 1];
233 size_t one = 1;
234 if (max_comm < one << 31) {
235 // count and displacement data in range, need to copy to integer vector.
236 std::vector<int> send_cnt(sendcnts.begin(), sendcnts.end());
237 std::vector<int> send_dis(senddisp.begin(), senddisp.end());
238 std::vector<int> recv_cnt(recvcnts.begin(), recvcnts.end());
239 std::vector<int> recv_dis(recvdisp.begin(), recvdisp.end());
240 return MPI_Alltoallv((void *)sendbuf.data(), send_cnt.data(), send_dis.data(), mpi_type(T(0)),
241 (void *)recvbuf.data(), recv_cnt.data(), recv_dis.data(), mpi_type(T(0)),
242 comm);
243 }
244 else {
245 #endif
246 // Same as if each processor sent a message to every other process with:
247 // MPI_Send(sendbuf+senddisp[i]*sizeof(sendtype),sendcnts[i], sendtype, i, tag, comm);
248 // And received a message from each processor with a call to:
249 // MPI_Recv(recvbuf+recvdisp[i]*sizeof(recvtype),recvcnts[i], recvtype, i, tag, comm);
250 return MY_Alltoallv64(sendbuf, sendcnts, senddisp, recvbuf, recvcnts, recvdisp, comm);
251 #if 1
252 }
253 #endif
254 }
255
256 template <typename T>
257 int MY_Alltoallv(const std::vector<T> &sendbuf, const std::vector<int> &sendcnts,
258 const std::vector<int> &senddisp, std::vector<T> &recvbuf,
259 const std::vector<int> &recvcnts, const std::vector<int> &recvdisp,
260 MPI_Comm comm)
261 {
262 return MPI_Alltoallv((void *)sendbuf.data(), const_cast<int *>(sendcnts.data()),
263 const_cast<int *>(senddisp.data()), mpi_type(T(0)), recvbuf.data(),
264 const_cast<int *>(recvcnts.data()), const_cast<int *>(recvdisp.data()),
265 mpi_type(T(0)), comm);
266 }
267 #endif
268 } // namespace Ioss
269 #endif
Ioss_CodeTypes.h
Ioss::ParallelUtils::global_minmax
T global_minmax(T local_minmax, MinMax which) const
Definition: Ioss_ParallelUtils.C:386
Ioss::ParallelUtils::generate_guid
int64_t generate_guid(size_t id, int rank=-1) const
Definition: Ioss_ParallelUtils.C:276
Ioss::Utils::power_2
static int power_2(int count)
Definition: Ioss_Utils.h:174
Ioss::ParallelUtils::attribute_reduction
void attribute_reduction(int length, char buffer[]) const
Definition: Ioss_ParallelUtils.C:292
Ioss
The main namespace for the Ioss library.
Definition: Iocgns_DatabaseIO.h:50
Ioss::ParallelUtils::parallel_rank
int parallel_rank() const
Definition: Ioss_ParallelUtils.C:234
Ioss::ParallelUtils::parallel_size
int parallel_size() const
Definition: Ioss_ParallelUtils.C:223
Ioss::ParallelUtils::hwm_memory_stats
void hwm_memory_stats(int64_t &min, int64_t &max, int64_t &avg) const
Definition: Ioss_ParallelUtils.C:259
Ioss::IntVector
std::vector< int > IntVector
Definition: Ioss_CodeTypes.h:43
Ioss::ParallelUtils::decode_filename
std::string decode_filename(const std::string &filename, bool is_parallel) const
Definition: Ioss_ParallelUtils.C:207
Ioss::ParallelUtils::DO_MAX
Definition: Ioss_ParallelUtils.h:54
Ioss::ParallelUtils
Definition: Ioss_ParallelUtils.h:45
Ioss::ParallelUtils::add_environment_properties
void add_environment_properties(Ioss::PropertyManager &properties, bool do_print)
Definition: Ioss_ParallelUtils.C:74
Ioss::ParallelUtils::DO_MIN
Definition: Ioss_ParallelUtils.h:54
Ioss_Utils.h
Ioss::ParallelUtils::global_array_minmax
void global_array_minmax(std::vector< T > &local_minmax, MinMax which) const
Definition: Ioss_ParallelUtils.C:455
Ioss::Int64Vector
std::vector< int64_t > Int64Vector
Definition: Ioss_CodeTypes.h:44
Ioss::ParallelUtils::memory_stats
void memory_stats(int64_t &min, int64_t &max, int64_t &avg) const
Definition: Ioss_ParallelUtils.C:245
Ioss::ParallelUtils::DO_SUM
Definition: Ioss_ParallelUtils.h:54
Ioss::ParallelUtils::MinMax
MinMax
Definition: Ioss_ParallelUtils.h:54
Ioss::ParallelUtils::communicator
MPI_Comm communicator() const
Definition: Ioss_ParallelUtils.h:93
Ioss::ParallelUtils::progress
void progress(const std::string &output) const
Definition: Ioss_ParallelUtils.C:538
anonymous_namespace{Iocgns_DecompositionData.C}::rank
int rank
Definition: Iocgns_DecompositionData.C:54
anonymous_namespace{io_info.C}::name
std::string name(Ioss::GroupingEntity *entity)
Definition: io_info.C:88
Ioss::ParallelUtils::global_count
void global_count(const IntVector &local_counts, IntVector &global_counts) const
Definition: Ioss_ParallelUtils.C:312
Ioss::ParallelUtils::get_environment
bool get_environment(const std::string &name, std::string &value, bool sync_parallel) const
Definition: Ioss_ParallelUtils.C:117
Ioss::ParallelUtils::~ParallelUtils
~ParallelUtils()=default
Ioss::ParallelUtils::ParallelUtils
ParallelUtils(MPI_Comm the_communicator)
Definition: Ioss_ParallelUtils.C:72
MPI_Comm
int MPI_Comm
Definition: Ioss_CodeTypes.h:80
Ioss::ParallelUtils::communicator_
MPI_Comm communicator_
Definition: Ioss_ParallelUtils.h:144
Ioss::ParallelUtils::gather
void gather(T my_value, std::vector< T > &result) const
Definition: Ioss_ParallelUtils.C:468
Ioss::PropertyManager
A collection of Ioss::Property objects.
Definition: Ioss_PropertyManager.h:49
Ioss::ParallelUtils::all_gather
void all_gather(T my_value, std::vector< T > &result) const
Definition: Ioss_ParallelUtils.C:491