IOSS 2.0

Ioss_ParallelUtils.h
Go to the documentation of this file.
1 // Copyright(C) 1999-2017 National Technology & Engineering Solutions
2 // of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
3 // NTESS, the U.S. Government retains certain rights in this software.
4 //
5 // Redistribution and use in source and binary forms, with or without
6 // modification, are permitted provided that the following conditions are
7 // met:
8 //
9 // * Redistributions of source code must retain the above copyright
10 // notice, this list of conditions and the following disclaimer.
11 //
12 // * Redistributions in binary form must reproduce the above
13 // copyright notice, this list of conditions and the following
14 // disclaimer in the documentation and/or other materials provided
15 // with the distribution.
16 //
17 // * Neither the name of NTESS nor the names of its
18 // contributors may be used to endorse or promote products derived
19 // from this software without specific prior written permission.
20 //
21 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32
33 #ifndef IOSS_Ioss_ParallelUtils_h
34 #define IOSS_Ioss_ParallelUtils_h
35
36 #include <Ioss_CodeTypes.h> // for Int64Vector, IntVector
37 #include <Ioss_Utils.h>
38 #include <cassert>
39 #include <cstddef> // for size_t
40 #include <string> // for string
41 #include <vector> // for vector
42
43 namespace Ioss {
44
45 class ParallelUtils
46 {
47 public:
48 explicit ParallelUtils(MPI_Comm the_communicator);
49 ~ParallelUtils() = default;
50
51 // Assignment operator
52 // Copy constructor
53
54 enum MinMax { DO_MAX, DO_MIN, DO_SUM };
55
56 /*!
57 * Returns 'true' if 'name' is defined in the environment.
58 * The value of the environment variable is returned in 'value'.
59 * getenv system call is only done on processor 0.
60 * If '!sync_parallel', then don't push to other processors.
61 */
62 bool get_environment(const std::string &name, std::string &value, bool sync_parallel) const;
63
64 /*!
65 * See if any external properties specified via the
66 * IOSS_PROPERTIES environment variable. If any found, add to
67 * `properties`. If `do_print` then output to cerr which
68 * properties were set
69 */
70 void add_environment_properties(Ioss::PropertyManager &properties, bool do_print);
71
72 /*!
73 * Returns 'true' if 'name' is defined in the environment.
74 * The value of the environment variable is converted to an
75 * integer via the atoi library call and returned in 'value'.
76 * No checking is done to ensure that the environment variable
77 * points to a valid integer.
78 * getenv system call is only done on processor 0.
79 * If '!sync_parallel', then don't push to other processors.
80 */
81 bool get_environment(const std::string &name, int &value, bool sync_parallel) const;
82
83 /*!
84 * Returns 'true' if 'name' is defined in the environment no
85 * matter what the value. Returns false otherwise.
86 * getenv system call is only done on processor 0.
87 * If '!sync_parallel', then don't push to other processors.
88 */
89 bool get_environment(const std::string &name, bool sync_parallel) const;
90
91 std::string decode_filename(const std::string &filename, bool is_parallel) const;
92
93 MPI_Comm communicator() const { return communicator_; }
94 int parallel_size() const;
95 int parallel_rank() const;
96
97 /*!
98 * Global OR of attribute strings, the processors which have no
99 * knowledge of the value should initialize to '0' and the
100 * processors with knowledge set the appropriate values.
101 */
102 void attribute_reduction(int length, char buffer[]) const;
103
104 /*!
105 * Generate a "globally unique id" which is unique over all entities
106 * of a specific type over all processors.
107 * Used by some applications for uniquely identifying an entity.
108 * If `rank` == -1, then use parallel_rank; otherwise use rank
109 */
110 int64_t generate_guid(size_t id, int rank = -1) const;
111
112 /*! Return min, max, average memory used by any process */
113 void memory_stats(int64_t &min, int64_t &max, int64_t &avg) const;
114
115 /*! Return high-water-mark min, max, average memory used by any process */
116 /* May be inaccurate unless system maintains this information */
117 void hwm_memory_stats(int64_t &min, int64_t &max, int64_t &avg) const;
118
119 /*! Vector 'local_counts' contains the number of objects
120 * local to this processor. On exit, global_counts
121 * contains the total number of objects on all processors.
122 * Assumes that ordering is the same on all processors
123 */
124 void global_count(const IntVector &local_counts, IntVector &global_counts) const;
125 void global_count(const Int64Vector &local_counts, Int64Vector &global_counts) const;
126
127 template <typename T> T global_minmax(T local_minmax, MinMax which) const;
128 template <typename T>
129 void global_array_minmax(std::vector<T> &local_minmax, MinMax which) const;
130 template <typename T>
131 void global_array_minmax(T *local_minmax, size_t count, MinMax which) const;
132
133 template <typename T> void gather(T my_value, std::vector<T> &result) const;
134 template <typename T> void all_gather(T my_value, std::vector<T> &result) const;
135 template <typename T> void gather(std::vector<T> &my_values, std::vector<T> &result) const;
136
137 void progress(const std::string &output) const;
138
139 private:
140 MPI_Comm communicator_;
141 };
142
143 #ifdef SEACAS_HAVE_MPI
144 inline MPI_Datatype mpi_type(double /*dummy*/) { return MPI_DOUBLE; }
145 inline MPI_Datatype mpi_type(float /*dummy*/) { return MPI_FLOAT; }
146 inline MPI_Datatype mpi_type(int /*dummy*/) { return MPI_INT; }
147 inline MPI_Datatype mpi_type(long int /*dummy*/) { return MPI_LONG_LONG_INT; }
148 inline MPI_Datatype mpi_type(long long int /*dummy*/) { return MPI_LONG_LONG_INT; }
149 inline MPI_Datatype mpi_type(unsigned int /*dummy*/) { return MPI_UNSIGNED; }
150 inline MPI_Datatype mpi_type(unsigned long int /*dummy*/) { return MPI_UNSIGNED_LONG; }
151
152 template <typename T>
153 int MY_Alltoallv64(const std::vector<T> &sendbuf, const std::vector<int64_t> &sendcounts,
154 const std::vector<int64_t> &senddisp, std::vector<T> &recvbuf,
155 const std::vector<int64_t> &recvcounts, const std::vector<int64_t> &recvdisp,
156 MPI_Comm comm)
157 {
158 int processor_count = 0;
159 int my_processor = 0;
160 MPI_Comm_size(comm, &processor_count);
161 MPI_Comm_rank(comm, &my_processor);
162
163 // Verify that all 'counts' can fit in an integer. Symmetric
164 // communication, so recvcounts are sendcounts on another processor.
165 for (int i = 0; i < processor_count; i++) {
166 int snd_cnt = static_cast<int>(sendcounts[i]);
167 if (static_cast<int64_t>(snd_cnt) != sendcounts[i]) {
168 std::ostringstream errmsg;
169 errmsg << "ERROR: The number of items that must be communicated via MPI calls from\n"
170 << " processor " << my_processor << " to processor " << i << " is "
171 << sendcounts[i]
172 << "\n which exceeds the storage capacity of the integers "
173 "used by MPI functions.\n";
174 std::cerr << errmsg.str();
175 exit(EXIT_FAILURE);
176 }
177 }
178
179 size_t pow_2 = Ioss::Utils::power_2(processor_count);
180
181 for (size_t i = 1; i < pow_2; i++) {
182 MPI_Status status{};
183
184 int tag = 24713;
185 size_t exchange_proc = i ^ my_processor;
186 if (exchange_proc < static_cast<size_t>(processor_count)) {
187 int snd_cnt = static_cast<int>(
188 sendcounts[exchange_proc]); // Converts from int64_t to int as needed by mpi
189 int rcv_cnt = static_cast<int>(recvcounts[exchange_proc]);
190 if (static_cast<size_t>(my_processor) < exchange_proc) {
191 MPI_Send((void *)&sendbuf[senddisp[exchange_proc]], snd_cnt, mpi_type(T(0)),
192 exchange_proc, tag, comm);
193 MPI_Recv(&recvbuf[recvdisp[exchange_proc]], rcv_cnt, mpi_type(T(0)), exchange_proc, tag,
194 comm, &status);
195 }
196 else {
197 MPI_Recv(&recvbuf[recvdisp[exchange_proc]], rcv_cnt, mpi_type(T(0)), exchange_proc, tag,
198 comm, &status);
199 MPI_Send((void *)&sendbuf[senddisp[exchange_proc]], snd_cnt, mpi_type(T(0)),
200 exchange_proc, tag, comm);
201 }
202 }
203 }
204
205 // Take care of this processor's data movement...
206 std::copy(&sendbuf[senddisp[my_processor]],
207 &sendbuf[senddisp[my_processor] + sendcounts[my_processor]],
208 &recvbuf[recvdisp[my_processor]]);
209 return 0;
210 }
211
212 template <typename T>
213 int MY_Alltoallv(const std::vector<T> &sendbuf, const std::vector<int64_t> &sendcnts,
214 const std::vector<int64_t> &senddisp, std::vector<T> &recvbuf,
215 const std::vector<int64_t> &recvcnts, const std::vector<int64_t> &recvdisp,
216 MPI_Comm comm)
217 {
218 // Wrapper to handle case where send/recv counts and displacements are 64-bit integers.
219 // Two cases:
220 // 1) They are of type 64-bit integers, but only storing data in the 32-bit integer range.
221 // -- if (sendcnts[#proc-1] + senddisp[#proc-1] < 2^31, then we are ok
222 // 2) They are of type 64-bit integers, and storing data in the 64-bit integer range.
223 // -- call special alltoallv which does point-to-point sends
224 #if 1
225 int processor_count = 0;
226 MPI_Comm_size(comm, &processor_count);
227 size_t max_comm = sendcnts[processor_count - 1] + senddisp[processor_count - 1];
228 size_t one = 1;
229 if (max_comm < one << 31) {
230 // count and displacement data in range, need to copy to integer vector.
231 std::vector<int> send_cnt(sendcnts.begin(), sendcnts.end());
232 std::vector<int> send_dis(senddisp.begin(), senddisp.end());
233 std::vector<int> recv_cnt(recvcnts.begin(), recvcnts.end());
234 std::vector<int> recv_dis(recvdisp.begin(), recvdisp.end());
235 return MPI_Alltoallv((void *)sendbuf.data(), send_cnt.data(), send_dis.data(), mpi_type(T(0)),
236 (void *)recvbuf.data(), recv_cnt.data(), recv_dis.data(), mpi_type(T(0)),
237 comm);
238 }
239 else {
240 #endif
241 // Same as if each processor sent a message to every other process with:
242 // MPI_Send(sendbuf+senddisp[i]*sizeof(sendtype),sendcnts[i], sendtype, i, tag, comm);
243 // And received a message from each processor with a call to:
244 // MPI_Recv(recvbuf+recvdisp[i]*sizeof(recvtype),recvcnts[i], recvtype, i, tag, comm);
245 return MY_Alltoallv64(sendbuf, sendcnts, senddisp, recvbuf, recvcnts, recvdisp, comm);
246 #if 1
247 }
248 #endif
249 }
250
251 template <typename T>
252 int MY_Alltoallv(const std::vector<T> &sendbuf, const std::vector<int> &sendcnts,
253 const std::vector<int> &senddisp, std::vector<T> &recvbuf,
254 const std::vector<int> &recvcnts, const std::vector<int> &recvdisp,
255 MPI_Comm comm)
256 {
257 return MPI_Alltoallv((void *)sendbuf.data(), const_cast<int *>(sendcnts.data()),
258 const_cast<int *>(senddisp.data()), mpi_type(T(0)), recvbuf.data(),
259 const_cast<int *>(recvcnts.data()), const_cast<int *>(recvdisp.data()),
260 mpi_type(T(0)), comm);
261 }
262 #endif
263 } // namespace Ioss
264 #endif
Ioss_CodeTypes.h
Ioss::ParallelUtils::global_minmax
T global_minmax(T local_minmax, MinMax which) const
Definition: Ioss_ParallelUtils.C:366
Ioss::ParallelUtils::generate_guid
int64_t generate_guid(size_t id, int rank=-1) const
Definition: Ioss_ParallelUtils.C:256
Ioss::Utils::power_2
static int power_2(int count)
Definition: Ioss_Utils.h:174
Ioss::ParallelUtils::attribute_reduction
void attribute_reduction(int length, char buffer[]) const
Definition: Ioss_ParallelUtils.C:272
Ioss
The main namespace for the Ioss library.
Definition: Iocgns_DatabaseIO.h:50
Ioss::ParallelUtils::parallel_rank
int parallel_rank() const
Definition: Ioss_ParallelUtils.C:214
Ioss::ParallelUtils::parallel_size
int parallel_size() const
Definition: Ioss_ParallelUtils.C:203
Ioss::ParallelUtils::hwm_memory_stats
void hwm_memory_stats(int64_t &min, int64_t &max, int64_t &avg) const
Definition: Ioss_ParallelUtils.C:239
Ioss::IntVector
std::vector< int > IntVector
Definition: Ioss_CodeTypes.h:42
Ioss::ParallelUtils::decode_filename
std::string decode_filename(const std::string &filename, bool is_parallel) const
Definition: Ioss_ParallelUtils.C:187
Ioss::ParallelUtils::DO_MAX
Definition: Ioss_ParallelUtils.h:54
Ioss::ParallelUtils
Definition: Ioss_ParallelUtils.h:45
Ioss::ParallelUtils::add_environment_properties
void add_environment_properties(Ioss::PropertyManager &properties, bool do_print)
Definition: Ioss_ParallelUtils.C:54
Ioss::ParallelUtils::DO_MIN
Definition: Ioss_ParallelUtils.h:54
Ioss_Utils.h
Ioss::ParallelUtils::global_array_minmax
void global_array_minmax(std::vector< T > &local_minmax, MinMax which) const
Definition: Ioss_ParallelUtils.C:453
Ioss::Int64Vector
std::vector< int64_t > Int64Vector
Definition: Ioss_CodeTypes.h:43
Ioss::ParallelUtils::memory_stats
void memory_stats(int64_t &min, int64_t &max, int64_t &avg) const
Definition: Ioss_ParallelUtils.C:225
Ioss::ParallelUtils::DO_SUM
Definition: Ioss_ParallelUtils.h:54
Ioss::ParallelUtils::MinMax
MinMax
Definition: Ioss_ParallelUtils.h:54
Ioss::ParallelUtils::communicator
MPI_Comm communicator() const
Definition: Ioss_ParallelUtils.h:93
Ioss::ParallelUtils::progress
void progress(const std::string &output) const
Definition: Ioss_ParallelUtils.C:512
anonymous_namespace{Iocgns_DecompositionData.C}::rank
int rank
Definition: Iocgns_DecompositionData.C:52
anonymous_namespace{io_info.C}::name
std::string name(Ioss::GroupingEntity *entity)
Definition: io_info.C:71
Ioss::ParallelUtils::global_count
void global_count(const IntVector &local_counts, IntVector &global_counts) const
Definition: Ioss_ParallelUtils.C:292
Ioss::ParallelUtils::get_environment
bool get_environment(const std::string &name, std::string &value, bool sync_parallel) const
Definition: Ioss_ParallelUtils.C:97
Ioss::ParallelUtils::~ParallelUtils
~ParallelUtils()=default
Ioss::ParallelUtils::ParallelUtils
ParallelUtils(MPI_Comm the_communicator)
Definition: Ioss_ParallelUtils.C:52
MPI_Comm
int MPI_Comm
Definition: Ioss_CodeTypes.h:79
Ioss::ParallelUtils::communicator_
MPI_Comm communicator_
Definition: Ioss_ParallelUtils.h:140
Ioss::ParallelUtils::gather
void gather(T my_value, std::vector< T > &result) const
Definition: Ioss_ParallelUtils.C:465
Ioss::PropertyManager
A collection of Ioss::Property objects.
Definition: Ioss_PropertyManager.h:49
Ioss::ParallelUtils::all_gather
void all_gather(T my_value, std::vector< T > &result) const
Definition: Ioss_ParallelUtils.C:488