DIY 3.0 data-parallel out-of-core C++ library

- Main Page
- Modules
- Namespaces
- Classes
- Files
- Examples

- File List

All Classes Namespaces Functions Typedefs Groups Pages
bov.hpp
1 #ifndef DIY_IO_BOV_HPP
2 #define DIY_IO_BOV_HPP
3
4 #include <vector>
5 #include <algorithm>
6 #include <numeric>
7
8 #include "../types.hpp"
9 #include "../mpi.hpp"
10
11 namespace diy
12 {
13 namespace io
14 {
15 // Reads and writes subsets of a block of values into specified block bounds
16 class BOV
17 {
18 public:
19 typedef std::vector<int> Shape;
20 public:
21 BOV(mpi::io::file& f):
22 f_(f), offset_(0) {}
23
24 template<class S>
25 BOV(mpi::io::file& f,
26 const S& shape = S(),
27 mpi::io::offset offset = 0):
28 f_(f), offset_(offset) { set_shape(shape); }
29
30 void set_offset(mpi::io::offset offset) { offset_ = offset; }
31
32 template<class S>
33 void set_shape(const S& shape)
34 {
35 shape_.clear();
36 stride_.clear();
37 for (unsigned i = 0; i < shape.size(); ++i)
38 {
39 shape_.push_back(shape[i]);
40 stride_.push_back(1);
41 }
42 for (int i = shape_.size() - 2; i >= 0; --i)
43 stride_[i] = stride_[i+1] * shape_[i+1];
44 }
45
46 const Shape& shape() const { return shape_; }
47
48 template<class T>
49 void read(const DiscreteBounds& bounds, T* buffer, bool collective = false, int chunk = 1) const;
50
51 template<class T>
52 void write(const DiscreteBounds& bounds, const T* buffer, bool collective = false, int chunk = 1);
53
54 template<class T>
55 void write(const DiscreteBounds& bounds, const T* buffer, const DiscreteBounds& core, bool collective = false, int chunk = 1);
56
57 protected:
58 mpi::io::file& file() { return f_; }
59
60 private:
61 mpi::io::file& f_;
62 Shape shape_;
63 std::vector<size_t> stride_;
64 size_t offset_;
65 };
66 }
67 }
68
69 template<class T>
70 void
71 diy::io::BOV::
72 read(const DiscreteBounds& bounds, T* buffer, bool collective, int chunk) const
73 {
74 #ifndef DIY_NO_MPI
75 int dim = shape_.size();
76 int total = 1;
77 std::vector<int> subsizes;
78 for (int i = 0; i < dim; ++i)
79 {
80 subsizes.push_back(bounds.max[i] - bounds.min[i] + 1);
81 total *= subsizes.back();
82 }
83
84 MPI_Datatype T_type;
85 if (chunk == 1)
86 T_type = mpi::detail::get_mpi_datatype<T>();
87 else
88 {
89 // create an MPI struct of size chunk to read the data in those chunks
90 // (this allows to work around MPI-IO weirdness where crucial quantities
91 // are ints, which are too narrow of a type)
92 int array_of_blocklengths[] = { chunk };
93 MPI_Aint array_of_displacements[] = { 0 };
94 MPI_Datatype array_of_types[] = { mpi::detail::get_mpi_datatype<T>() };
95 MPI_Type_create_struct(1, array_of_blocklengths, array_of_displacements, array_of_types, &T_type);
96 MPI_Type_commit(&T_type);
97 }
98
99 MPI_Datatype fileblk;
100 MPI_Type_create_subarray(dim, (int*) &shape_[0], &subsizes[0], (int*) &bounds.min[0], MPI_ORDER_C, T_type, &fileblk);
101 MPI_Type_commit(&fileblk);
102
103 MPI_File_set_view(f_.handle(), offset_, T_type, fileblk, (char*)"native", MPI_INFO_NULL);
104
105 mpi::status s;
106 if (!collective)
107 MPI_File_read(f_.handle(), buffer, total, T_type, &s.s);
108 else
109 MPI_File_read_all(f_.handle(), buffer, total, T_type, &s.s);
110
111 if (chunk != 1)
112 MPI_Type_free(&T_type);
113 MPI_Type_free(&fileblk);
114 #else
115 (void) bounds; (void) buffer; (void) collective; (void)chunk;
116 DIY_UNSUPPORTED_MPI_CALL(diy::io::BOV::read);
117 #endif
118 }
119
120 template<class T>
121 void
122 diy::io::BOV::
123 write(const DiscreteBounds& bounds, const T* buffer, bool collective, int chunk)
124 {
125 write(bounds, buffer, bounds, collective, chunk);
126 }
127
128 template<class T>
129 void
130 diy::io::BOV::
131 write(const DiscreteBounds& bounds, const T* buffer, const DiscreteBounds& core, bool collective, int chunk)
132 {
133 #ifndef DIY_NO_MPI
134 int dim = shape_.size();
135 std::vector<int> subsizes;
136 std::vector<int> buffer_shape, buffer_start;
137 for (int i = 0; i < dim; ++i)
138 {
139 buffer_shape.push_back(bounds.max[i] - bounds.min[i] + 1);
140 buffer_start.push_back(core.min[i] - bounds.min[i]);
141 subsizes.push_back(core.max[i] - core.min[i] + 1);
142 }
143
144 MPI_Datatype T_type;
145 if (chunk == 1)
146 T_type = mpi::detail::get_mpi_datatype<T>();
147 else
148 {
149 // assume T is a binary block and create an MPI struct of appropriate size
150 int array_of_blocklengths[] = { chunk };
151 MPI_Aint array_of_displacements[] = { 0 };
152 MPI_Datatype array_of_types[] = { mpi::detail::get_mpi_datatype<T>() };
153 MPI_Type_create_struct(1, array_of_blocklengths, array_of_displacements, array_of_types, &T_type);
154 MPI_Type_commit(&T_type);
155 }
156
157 MPI_Datatype fileblk, subbuffer;
158 MPI_Type_create_subarray(dim, (int*) &shape_[0], &subsizes[0], (int*) &bounds.min[0], MPI_ORDER_C, T_type, &fileblk);
159 MPI_Type_create_subarray(dim, (int*) &buffer_shape[0], &subsizes[0], (int*) &buffer_start[0], MPI_ORDER_C, T_type, &subbuffer);
160 MPI_Type_commit(&fileblk);
161 MPI_Type_commit(&subbuffer);
162
163 MPI_File_set_view(f_.handle(), offset_, T_type, fileblk, (char*)"native", MPI_INFO_NULL);
164
165 mpi::status s;
166 if (!collective)
167 MPI_File_write(f_.handle(), (void*)buffer, 1, subbuffer, &s.s);
168 else
169 MPI_File_write_all(f_.handle(), (void*)buffer, 1, subbuffer, &s.s);
170
171 if (chunk != 1)
172 MPI_Type_free(&T_type);
173 MPI_Type_free(&fileblk);
174 MPI_Type_free(&subbuffer);
175 #else
176 (void) bounds; (void) buffer;(void) core; (void) collective; (void) chunk;
177 DIY_UNSUPPORTED_MPI_CALL(diy::io::bov::write);
178 #endif
179 }
180
181 #endif
diy::io::BOV
Definition: bov.hpp:16
diy::mpi::io::file
Wraps MPI file IO.
Definition: io.hpp:18
diy::Bounds
Definition: types.hpp:16