DIY 3.0 data-parallel out-of-core C++ library

- Main Page
- Modules
- Namespaces
- Classes
- Files
- Examples

- File List

All Classes Namespaces Functions Typedefs Groups Pages
serialization.hpp
1 #ifndef DIY_SERIALIZATION_HPP
2 #define DIY_SERIALIZATION_HPP
3
4 #include <vector>
5 #include <valarray>
6 #include <map>
7 #include <set>
8 #include <string>
9 #include <fstream>
10
11 #include <tuple>
12 #include <unordered_map>
13 #include <unordered_set>
14 #include <type_traits> // this is used for a safety check for default serialization
15
16 namespace diy
17 {
19 struct BinaryBuffer
20 {
21 virtual void save_binary(const char* x, size_t count) =0;
22 virtual void load_binary(char* x, size_t count) =0;
23 virtual void load_binary_back(char* x, size_t count) =0;
24 };
25
26 struct MemoryBuffer: public BinaryBuffer
27 {
28 MemoryBuffer(size_t position_ = 0):
29 position(position_) {}
30
31 virtual inline void save_binary(const char* x, size_t count) override;
32 virtual inline void load_binary(char* x, size_t count) override;
33 virtual inline void load_binary_back(char* x, size_t count) override;
34
35 void clear() { buffer.clear(); reset(); }
36 void wipe() { std::vector<char>().swap(buffer); reset(); }
37 void reset() { position = 0; }
38 void skip(size_t s) { position += s; }
39 void swap(MemoryBuffer& o) { std::swap(position, o.position); buffer.swap(o.buffer); }
40 bool empty() const { return buffer.empty(); }
41 size_t size() const { return buffer.size(); }
42 void reserve(size_t s) { buffer.reserve(s); }
43 operator bool() const { return position < buffer.size(); }
44
46 inline static void copy(MemoryBuffer& from, MemoryBuffer& to);
47
49 static float growth_multiplier() { return 1.5; }
50
51 // simple file IO
52 void write(const std::string& fn) const { std::ofstream out(fn.c_str()); out.write(&buffer[0], size()); }
53 void read(const std::string& fn)
54 {
55 std::ifstream in(fn.c_str(), std::ios::binary | std::ios::ate);
56 buffer.resize(in.tellg());
57 in.seekg(0);
58 in.read(&buffer[0], size());
59 position = 0;
60 }
61
62 size_t position;
63 std::vector<char> buffer;
64 };
65
66 namespace detail
67 {
68 struct Default {};
69 }
70
73
89 template<class T>
90 struct Serialization: public detail::Default
91 {
92 #if defined(__clang__) || (defined(__GNUC__) && __GNUC__ >= 5)
93 static_assert(std::is_trivially_copyable<T>::value, "Default serialization works only for trivially copyable types");
94 #endif
95
96 static void save(BinaryBuffer& bb, const T& x) { bb.save_binary((const char*) &x, sizeof(T)); }
97 static void load(BinaryBuffer& bb, T& x) { bb.load_binary((char*) &x, sizeof(T)); }
98 };
99
101 template<class T>
102 void save(BinaryBuffer& bb, const T& x) { Serialization<T>::save(bb, x); }
103
105 template<class T>
106 void load(BinaryBuffer& bb, T& x) { Serialization<T>::load(bb, x); }
107
110 template<class T>
111 void save(BinaryBuffer& bb, const T* x, size_t n);
112
115 template<class T>
116 void load(BinaryBuffer& bb, T* x, size_t n);
117
119 template<class T>
120 void load_back(BinaryBuffer& bb, T& x) { bb.load_binary_back((char*) &x, sizeof(T)); }
121
123
124
125 namespace detail
126 {
127 template<typename T>
128 struct is_default
129 {
130 typedef char yes;
131 typedef int no;
132
133 static yes test(Default*);
134 static no test(...);
135
136 enum { value = (sizeof(test((T*) 0)) == sizeof(yes)) };
137 };
138 }
139
140 template<class T>
141 void save(BinaryBuffer& bb, const T* x, size_t n)
142 {
143 if (!detail::is_default< Serialization<T> >::value)
144 for (size_t i = 0; i < n; ++i)
145 diy::save(bb, x[i]);
146 else // if Serialization is not specialized for U, just save the binary data
147 bb.save_binary((const char*) &x[0], sizeof(T)*n);
148 }
149
150 template<class T>
151 void load(BinaryBuffer& bb, T* x, size_t n)
152 {
153 if (!detail::is_default< Serialization<T> >::value)
154 for (size_t i = 0; i < n; ++i)
155 diy::load(bb, x[i]);
156 else // if Serialization is not specialized for U, just load the binary data
157 bb.load_binary((char*) &x[0], sizeof(T)*n);
158 }
159
160
161 // save/load for MemoryBuffer
162 template<>
163 struct Serialization< MemoryBuffer >
164 {
165 static void save(BinaryBuffer& bb, const MemoryBuffer& x)
166 {
167 diy::save(bb, x.position);
168 diy::save(bb, &x.buffer[0], x.position);
169 }
170
171 static void load(BinaryBuffer& bb, MemoryBuffer& x)
172 {
173 diy::load(bb, x.position);
174 x.buffer.resize(x.position);
175 diy::load(bb, &x.buffer[0], x.position);
176 }
177 };
178
179 // save/load for std::vector<U>
180 template<class U>
181 struct Serialization< std::vector<U> >
182 {
183 typedef std::vector<U> Vector;
184
185 static void save(BinaryBuffer& bb, const Vector& v)
186 {
187 size_t s = v.size();
188 diy::save(bb, s);
189 diy::save(bb, &v[0], v.size());
190 }
191
192 static void load(BinaryBuffer& bb, Vector& v)
193 {
194 size_t s;
195 diy::load(bb, s);
196 v.resize(s);
197 diy::load(bb, &v[0], s);
198 }
199 };
200
201 template<class U>
202 struct Serialization< std::valarray<U> >
203 {
204 typedef std::valarray<U> ValArray;
205
206 static void save(BinaryBuffer& bb, const ValArray& v)
207 {
208 size_t s = v.size();
209 diy::save(bb, s);
210 diy::save(bb, &v[0], v.size());
211 }
212
213 static void load(BinaryBuffer& bb, ValArray& v)
214 {
215 size_t s;
216 diy::load(bb, s);
217 v.resize(s);
218 diy::load(bb, &v[0], s);
219 }
220 };
221
222 // save/load for std::string
223 template<>
224 struct Serialization< std::string >
225 {
226 typedef std::string String;
227
228 static void save(BinaryBuffer& bb, const String& s)
229 {
230 size_t sz = s.size();
231 diy::save(bb, sz);
232 diy::save(bb, s.c_str(), sz);
233 }
234
235 static void load(BinaryBuffer& bb, String& s)
236 {
237 size_t sz;
238 diy::load(bb, sz);
239 s.resize(sz);
240 for (size_t i = 0; i < sz; ++i)
241 {
242 char c;
243 diy::load(bb, c);
244 s[i] = c;
245 }
246 }
247 };
248
249 // save/load for std::pair<X,Y>
250 template<class X, class Y>
251 struct Serialization< std::pair<X,Y> >
252 {
253 typedef std::pair<X,Y> Pair;
254
255 static void save(BinaryBuffer& bb, const Pair& p)
256 {
257 diy::save(bb, p.first);
258 diy::save(bb, p.second);
259 }
260
261 static void load(BinaryBuffer& bb, Pair& p)
262 {
263 diy::load(bb, p.first);
264 diy::load(bb, p.second);
265 }
266 };
267
268 // save/load for std::map<K,V>
269 template<class K, class V>
270 struct Serialization< std::map<K,V> >
271 {
272 typedef std::map<K,V> Map;
273
274 static void save(BinaryBuffer& bb, const Map& m)
275 {
276 size_t s = m.size();
277 diy::save(bb, s);
278 for (typename std::map<K,V>::const_iterator it = m.begin(); it != m.end(); ++it)
279 diy::save(bb, *it);
280 }
281
282 static void load(BinaryBuffer& bb, Map& m)
283 {
284 size_t s;
285 diy::load(bb, s);
286 for (size_t i = 0; i < s; ++i)
287 {
288 K k;
289 diy::load(bb, k);
290 diy::load(bb, m[k]);
291 }
292 }
293 };
294
295 // save/load for std::set<T>
296 template<class T>
297 struct Serialization< std::set<T> >
298 {
299 typedef std::set<T> Set;
300
301 static void save(BinaryBuffer& bb, const Set& m)
302 {
303 size_t s = m.size();
304 diy::save(bb, s);
305 for (typename std::set<T>::const_iterator it = m.begin(); it != m.end(); ++it)
306 diy::save(bb, *it);
307 }
308
309 static void load(BinaryBuffer& bb, Set& m)
310 {
311 size_t s;
312 diy::load(bb, s);
313 for (size_t i = 0; i < s; ++i)
314 {
315 T p;
316 diy::load(bb, p);
317 m.insert(p);
318 }
319 }
320 };
321
322 // save/load for std::unordered_map<K,V,H,E,A>
323 template<class K, class V, class H, class E, class A>
324 struct Serialization< std::unordered_map<K,V,H,E,A> >
325 {
326 typedef std::unordered_map<K,V,H,E,A> Map;
327
328 static void save(BinaryBuffer& bb, const Map& m)
329 {
330 size_t s = m.size();
331 diy::save(bb, s);
332 for (auto& x : m)
333 diy::save(bb, x);
334 }
335
336 static void load(BinaryBuffer& bb, Map& m)
337 {
338 size_t s;
339 diy::load(bb, s);
340 for (size_t i = 0; i < s; ++i)
341 {
342 std::pair<K,V> p;
343 diy::load(bb, p);
344 m.emplace(std::move(p));
345 }
346 }
347 };
348
349 // save/load for std::unordered_set<T,H,E,A>
350 template<class T, class H, class E, class A>
351 struct Serialization< std::unordered_set<T,H,E,A> >
352 {
353 typedef std::unordered_set<T,H,E,A> Set;
354
355 static void save(BinaryBuffer& bb, const Set& m)
356 {
357 size_t s = m.size();
358 diy::save(bb, s);
359 for (auto& x : m)
360 diy::save(bb, x);
361 }
362
363 static void load(BinaryBuffer& bb, Set& m)
364 {
365 size_t s;
366 diy::load(bb, s);
367 for (size_t i = 0; i < s; ++i)
368 {
369 T p;
370 diy::load(bb, p);
371 m.emplace(std::move(p));
372 }
373 }
374 };
375
376 // save/load for std::tuple<...>
377 // TODO: this ought to be default (copying) serialization
378 // if all arguments are default
379 template<class... Args>
380 struct Serialization< std::tuple<Args...> >
381 {
382 typedef std::tuple<Args...> Tuple;
383
384 static void save(BinaryBuffer& bb, const Tuple& t) { save<0>(bb, t); }
385
386 template<std::size_t I = 0>
387 static
388 typename std::enable_if<I == sizeof...(Args), void>::type
389 save(BinaryBuffer&, const Tuple&) {}
390
391 template<std::size_t I = 0>
392 static
393 typename std::enable_if<I < sizeof...(Args), void>::type
394 save(BinaryBuffer& bb, const Tuple& t) { diy::save(bb, std::get<I>(t)); save<I+1>(bb, t); }
395
396 static void load(BinaryBuffer& bb, Tuple& t) { load<0>(bb, t); }
397
398 template<std::size_t I = 0>
399 static
400 typename std::enable_if<I == sizeof...(Args), void>::type
401 load(BinaryBuffer&, Tuple&) {}
402
403 template<std::size_t I = 0>
404 static
405 typename std::enable_if<I < sizeof...(Args), void>::type
406 load(BinaryBuffer& bb, Tuple& t) { diy::load(bb, std::get<I>(t)); load<I+1>(bb, t); }
407
408 };
409 }
410
411 void
412 diy::MemoryBuffer::
413 save_binary(const char* x, size_t count)
414 {
415 if (position + count > buffer.capacity())
416 buffer.reserve((position + count) * growth_multiplier()); // if we have to grow, grow geometrically
417
418 if (position + count > buffer.size())
419 buffer.resize(position + count);
420
421 std::copy(x, x + count, &buffer[position]);
422 position += count;
423 }
424
425 void
426 diy::MemoryBuffer::
427 load_binary(char* x, size_t count)
428 {
429 std::copy(&buffer[position], &buffer[position + count], x);
430 position += count;
431 }
432
433 void
434 diy::MemoryBuffer::
435 load_binary_back(char* x, size_t count)
436 {
437 std::copy(&buffer[buffer.size() - count], &buffer[buffer.size()], x);
438 buffer.resize(buffer.size() - count);
439 }
440
441 void
442 diy::MemoryBuffer::
443 copy(MemoryBuffer& from, MemoryBuffer& to)
444 {
445 size_t sz;
446 diy::load(from, sz);
447 from.position -= sizeof(size_t);
448
449 size_t total = sizeof(size_t) + sz;
450 to.buffer.resize(to.position + total);
451 std::copy(&from.buffer[from.position], &from.buffer[from.position + total], &to.buffer[to.position]);
452 to.position += total;
453 from.position += total;
454 }
455
456 #endif
diy::load
void load(BinaryBuffer &bb, T &x)
Loads x from bb by calling diy::Serialization<T>::load(bb,x).
Definition: serialization.hpp:106
diy::load_back
void load_back(BinaryBuffer &bb, T &x)
Supports only binary data copying (meant for simple footers).
Definition: serialization.hpp:120
diy::BinaryBuffer::save_binary
virtual void save_binary(const char *x, size_t count)=0
copy count bytes from x into the buffer
diy::MemoryBuffer::load_binary
virtual void load_binary(char *x, size_t count) override
copy count bytes into x from the buffer
Definition: serialization.hpp:427
diy::MemoryBuffer::copy
static void copy(MemoryBuffer &from, MemoryBuffer &to)
copy a memory buffer from one buffer to another, bypassing making a temporary copy first ...
Definition: serialization.hpp:443
diy::MemoryBuffer::growth_multiplier
static float growth_multiplier()
multiplier used for the geometric growth of the container
Definition: serialization.hpp:49
diy::save
void save(BinaryBuffer &bb, const T &x)
Saves x to bb by calling diy::Serialization<T>::save(bb,x).
Definition: serialization.hpp:102
diy::MemoryBuffer::load_binary_back
virtual void load_binary_back(char *x, size_t count) override
copy count bytes into x from the back of the buffer
Definition: serialization.hpp:435
diy::in
void in(const RegularLink< Bounds > &link, const Point &p, OutIter out, const Bounds &domain)
Finds the neighbor(s) containing the target point.
Definition: pick.hpp:102
diy::MemoryBuffer
Definition: serialization.hpp:26
diy::BinaryBuffer::load_binary
virtual void load_binary(char *x, size_t count)=0
copy count bytes into x from the buffer
diy::MemoryBuffer::save_binary
virtual void save_binary(const char *x, size_t count) override
copy count bytes from x into the buffer
Definition: serialization.hpp:413
diy::Serialization
Main interface to serialization, meant to be specialized for the types that require special handling...
Definition: serialization.hpp:90
diy::BinaryBuffer::load_binary_back
virtual void load_binary_back(char *x, size_t count)=0
copy count bytes into x from the back of the buffer
diy::BinaryBuffer
A serialization buffer.
Definition: serialization.hpp:19