DIY 3.0 data-parallel out-of-core C++ library

- Main Page
- Modules
- Namespaces
- Classes
- Files
- Examples

- File List

All Classes Namespaces Functions Typedefs Groups Pages
collectives.hpp
1 #include <vector>
2
3 #include "operations.hpp"
4
5 namespace diy
6 {
7 namespace mpi
8 {
11
12 template<class T, class Op>
13 struct Collectives
14 {
15 typedef detail::mpi_datatype<T> Datatype;
16
17 static void broadcast(const communicator& comm, T& x, int root)
18 {
19 MPI_Bcast(Datatype::address(x),
20 Datatype::count(x),
21 Datatype::datatype(), root, comm);
22 }
23
24 static void broadcast(const communicator& comm, std::vector<T>& x, int root)
25 {
26 size_t sz = x.size();
27 Collectives<size_t, void*>::broadcast(comm, sz, root);
28
29 if (comm.rank() != root)
30 x.resize(sz);
31
32 MPI_Bcast(Datatype::address(x[0]),
33 x.size(),
34 Datatype::datatype(), root, comm);
35 }
36
37 static request ibroadcast(const communicator& comm, T& x, int root)
38 {
39 request r;
40 MPI_Ibcast(Datatype::address(x),
41 Datatype::count(x),
42 Datatype::datatype(), root, comm, &r.r);
43 return r;
44 }
45
46 static void gather(const communicator& comm, const T& in, std::vector<T>& out, int root)
47 {
48 size_t s = comm.size();
49 s *= Datatype::count(in);
50 out.resize(s);
51 MPI_Gather(Datatype::address(const_cast<T&>(in)),
52 Datatype::count(in),
53 Datatype::datatype(),
54 Datatype::address(out[0]),
55 Datatype::count(in),
56 Datatype::datatype(),
57 root, comm);
58 }
59
60 static void gather(const communicator& comm, const std::vector<T>& in, std::vector< std::vector<T> >& out, int root)
61 {
62 std::vector<int> counts(comm.size());
63 Collectives<int,void*>::gather(comm, (int) in.size(), counts, root);
64
65 std::vector<int> offsets(comm.size(), 0);
66 for (unsigned i = 1; i < offsets.size(); ++i)
67 offsets[i] = offsets[i-1] + counts[i-1];
68
69 std::vector<T> buffer(offsets.back() + counts.back());
70 MPI_Gatherv(Datatype::address(const_cast<T&>(in[0])),
71 in.size(),
72 Datatype::datatype(),
73 Datatype::address(buffer[0]),
74 &counts[0],
75 &offsets[0],
76 Datatype::datatype(),
77 root, comm);
78
79 out.resize(comm.size());
80 size_t cur = 0;
81 for (unsigned i = 0; i < (unsigned)comm.size(); ++i)
82 {
83 out[i].reserve(counts[i]);
84 for (unsigned j = 0; j < (unsigned)counts[i]; ++j)
85 out[i].push_back(buffer[cur++]);
86 }
87 }
88
89 static void gather(const communicator& comm, const T& in, int root)
90 {
91 MPI_Gather(Datatype::address(const_cast<T&>(in)),
92 Datatype::count(in),
93 Datatype::datatype(),
94 Datatype::address(const_cast<T&>(in)),
95 Datatype::count(in),
96 Datatype::datatype(),
97 root, comm);
98 }
99
100 static void gather(const communicator& comm, const std::vector<T>& in, int root)
101 {
102 Collectives<int,void*>::gather(comm, (int) in.size(), root);
103
104 MPI_Gatherv(Datatype::address(const_cast<T&>(in[0])),
105 in.size(),
106 Datatype::datatype(),
107 0, 0, 0,
108 Datatype::datatype(),
109 root, comm);
110 }
111
112 static void all_gather(const communicator& comm, const T& in, std::vector<T>& out)
113 {
114 size_t s = comm.size();
115 s *= Datatype::count(in);
116 out.resize(s);
117 MPI_Allgather(Datatype::address(const_cast<T&>(in)),
118 Datatype::count(in),
119 Datatype::datatype(),
120 Datatype::address(out[0]),
121 Datatype::count(in),
122 Datatype::datatype(),
123 comm);
124 }
125
126 static void all_gather(const communicator& comm, const std::vector<T>& in, std::vector< std::vector<T> >& out)
127 {
128 std::vector<int> counts(comm.size());
129 Collectives<int,void*>::all_gather(comm, (int) in.size(), counts);
130
131 std::vector<int> offsets(comm.size(), 0);
132 for (unsigned i = 1; i < offsets.size(); ++i)
133 offsets[i] = offsets[i-1] + counts[i-1];
134
135 std::vector<T> buffer(offsets.back() + counts.back());
136 MPI_Allgatherv(Datatype::address(const_cast<T&>(in[0])),
137 in.size(),
138 Datatype::datatype(),
139 Datatype::address(buffer[0]),
140 &counts[0],
141 &offsets[0],
142 Datatype::datatype(),
143 comm);
144
145 out.resize(comm.size());
146 size_t cur = 0;
147 for (int i = 0; i < comm.size(); ++i)
148 {
149 out[i].reserve(counts[i]);
150 for (int j = 0; j < counts[i]; ++j)
151 out[i].push_back(buffer[cur++]);
152 }
153 }
154
155 static void reduce(const communicator& comm, const T& in, T& out, int root, const Op&)
156 {
157 MPI_Reduce(Datatype::address(const_cast<T&>(in)),
158 Datatype::address(out),
159 Datatype::count(in),
160 Datatype::datatype(),
161 detail::mpi_op<Op>::get(),
162 root, comm);
163 }
164
165 static void reduce(const communicator& comm, const T& in, int root, const Op& op)
166 {
167 MPI_Reduce(Datatype::address(const_cast<T&>(in)),
168 Datatype::address(const_cast<T&>(in)),
169 Datatype::count(in),
170 Datatype::datatype(),
171 detail::mpi_op<Op>::get(),
172 root, comm);
173 }
174
175 static void all_reduce(const communicator& comm, const T& in, T& out, const Op&)
176 {
177 MPI_Allreduce(Datatype::address(const_cast<T&>(in)),
178 Datatype::address(out),
179 Datatype::count(in),
180 Datatype::datatype(),
181 detail::mpi_op<Op>::get(),
182 comm);
183 }
184
185 static void all_reduce(const communicator& comm, const std::vector<T>& in, std::vector<T>& out, const Op&)
186 {
187 out.resize(in.size());
188 MPI_Allreduce(Datatype::address(const_cast<T&>(in[0])),
189 Datatype::address(out[0]),
190 in.size(),
191 Datatype::datatype(),
192 detail::mpi_op<Op>::get(),
193 comm);
194 }
195
196 static void scan(const communicator& comm, const T& in, T& out, const Op&)
197 {
198 MPI_Scan(Datatype::address(const_cast<T&>(in)),
199 Datatype::address(out),
200 Datatype::count(in),
201 Datatype::datatype(),
202 detail::mpi_op<Op>::get(),
203 comm);
204 }
205
206 static void all_to_all(const communicator& comm, const std::vector<T>& in, std::vector<T>& out, int n = 1)
207 {
208 // NB: this will fail if T is a vector
209 MPI_Alltoall(Datatype::address(const_cast<T&>(in[0])), n,
210 Datatype::datatype(),
211 Datatype::address(out[0]), n,
212 Datatype::datatype(),
213 comm);
214 }
215 };
216
218 template<class T>
219 void broadcast(const communicator& comm, T& x, int root)
220 {
221 Collectives<T,void*>::broadcast(comm, x, root);
222 }
223
225 template<class T>
226 void broadcast(const communicator& comm, std::vector<T>& x, int root)
227 {
228 Collectives<T,void*>::broadcast(comm, x, root);
229 }
230
232 template<class T>
233 request ibroadcast(const communicator& comm, T& x, int root)
234 {
235 return Collectives<T,void*>::ibroadcast(comm, x, root);
236 }
237
241 template<class T>
242 void gather(const communicator& comm, const T& in, std::vector<T>& out, int root)
243 {
244 Collectives<T,void*>::gather(comm, in, out, root);
245 }
246
248 template<class T>
249 void gather(const communicator& comm, const std::vector<T>& in, std::vector< std::vector<T> >& out, int root)
250 {
251 Collectives<T,void*>::gather(comm, in, out, root);
252 }
253
255 template<class T>
256 void gather(const communicator& comm, const T& in, int root)
257 {
258 Collectives<T,void*>::gather(comm, in, root);
259 }
260
262 template<class T>
263 void gather(const communicator& comm, const std::vector<T>& in, int root)
264 {
265 Collectives<T,void*>::gather(comm, in, root);
266 }
267
271 template<class T>
272 void all_gather(const communicator& comm, const T& in, std::vector<T>& out)
273 {
274 Collectives<T,void*>::all_gather(comm, in, out);
275 }
276
278 template<class T>
279 void all_gather(const communicator& comm, const std::vector<T>& in, std::vector< std::vector<T> >& out)
280 {
281 Collectives<T,void*>::all_gather(comm, in, out);
282 }
283
285 template<class T, class Op>
286 void reduce(const communicator& comm, const T& in, T& out, int root, const Op& op)
287 {
288 Collectives<T, Op>::reduce(comm, in, out, root, op);
289 }
290
292 template<class T, class Op>
293 void reduce(const communicator& comm, const T& in, int root, const Op& op)
294 {
295 Collectives<T, Op>::reduce(comm, in, root, op);
296 }
297
299 template<class T, class Op>
300 void all_reduce(const communicator& comm, const T& in, T& out, const Op& op)
301 {
302 Collectives<T, Op>::all_reduce(comm, in, out, op);
303 }
304
306 template<class T, class Op>
307 void all_reduce(const communicator& comm, const std::vector<T>& in, std::vector<T>& out, const Op& op)
308 {
309 Collectives<T, Op>::all_reduce(comm, in, out, op);
310 }
311
313 template<class T, class Op>
314 void scan(const communicator& comm, const T& in, T& out, const Op& op)
315 {
316 Collectives<T, Op>::scan(comm, in, out, op);
317 }
318
320 template<class T>
321 void all_to_all(const communicator& comm, const std::vector<T>& in, std::vector<T>& out, int n = 1)
322 {
323 Collectives<T, void*>::all_to_all(comm, in, out, n);
324 }
325
327 }
328 }
diy::mpi::ibroadcast
request ibroadcast(const communicator &comm, T &x, int root)
iBroadcast to all processes in comm.
Definition: collectives.hpp:233
diy::mpi::all_reduce
void all_reduce(const communicator &comm, const std::vector< T > &in, std::vector< T > &out, const Op &op)
Same as above, but for vectors.
Definition: collectives.hpp:307
diy::mpi::all_to_all
void all_to_all(const communicator &comm, const std::vector< T > &in, std::vector< T > &out, int n=1)
all_to_all
Definition: collectives.hpp:321
diy::mpi::reduce
void reduce(const communicator &comm, const T &in, T &out, int root, const Op &op)
reduce
Definition: collectives.hpp:286
diy::mpi::broadcast
void broadcast(const communicator &comm, std::vector< T > &x, int root)
Broadcast for vectors.
Definition: collectives.hpp:226
diy::mpi::scan
void scan(const communicator &comm, const T &in, T &out, const Op &op)
scan
Definition: collectives.hpp:314
diy::in
void in(const RegularLink< Bounds > &link, const Point &p, OutIter out, const Bounds &domain)
Finds the neighbor(s) containing the target point.
Definition: pick.hpp:102
diy::mpi::communicator
Simple wrapper around MPI_Comm.
Definition: communicator.hpp:8
diy::mpi::all_gather
void all_gather(const communicator &comm, const T &in, std::vector< T > &out)
all_gather from all processes in comm. out is resized to comm.size() and filled with elements from th...
Definition: collectives.hpp:272
diy::mpi::reduce
void reduce(const communicator &comm, const T &in, int root, const Op &op)
Simplified version (without out) for use on non-root processes.
Definition: collectives.hpp:293
diy::mpi::all_reduce
void all_reduce(const communicator &comm, const T &in, T &out, const Op &op)
all_reduce
Definition: collectives.hpp:300
diy::mpi::request
Definition: request.hpp:5
diy::mpi::all_gather
void all_gather(const communicator &comm, const std::vector< T > &in, std::vector< std::vector< T > > &out)
Same as above, but for vectors.
Definition: collectives.hpp:279
diy::mpi::broadcast
void broadcast(const communicator &comm, T &x, int root)
Broadcast to all processes in comm.
Definition: collectives.hpp:219
diy::mpi::gather
void gather(const communicator &comm, const T &in, std::vector< T > &out, int root)
Gather from all processes in comm. On root process, out is resized to comm.size() and filled with ele...
Definition: collectives.hpp:242
diy::mpi::gather
void gather(const communicator &comm, const std::vector< T > &in, int root)
Simplified version (without out) for use on non-root processes.
Definition: collectives.hpp:263