DIY 3.0 data-parallel out-of-core C++ library

- Main Page
- Modules
- Namespaces
- Classes
- Files
- Examples

- File List

All Classes Namespaces Functions Typedefs Groups Pages
serialization.hpp
1 #ifndef DIY_SERIALIZATION_HPP
2 #define DIY_SERIALIZATION_HPP
3
4 #include <vector>
5 #include <valarray>
6 #include <map>
7 #include <set>
8 #include <string>
9 #include <fstream>
10
11 #include <tuple>
12 #include <unordered_map>
13 #include <unordered_set>
14 #include <type_traits> // this is used for a safety check for default serialization
15
16 namespace diy
17 {
19 struct BinaryBuffer
20 {
21 virtual void save_binary(const char* x, size_t count) =0;
22 virtual void load_binary(char* x, size_t count) =0;
23 virtual void load_binary_back(char* x, size_t count) =0;
24 };
25
26 struct MemoryBuffer: public BinaryBuffer
27 {
28 MemoryBuffer(size_t position_ = 0):
29 position(position_) {}
30
31 virtual inline void save_binary(const char* x, size_t count) override;
32 virtual inline void load_binary(char* x, size_t count) override;
33 virtual inline void load_binary_back(char* x, size_t count) override;
34
35 void clear() { buffer.clear(); reset(); }
36 void wipe() { std::vector<char>().swap(buffer); reset(); }
37 void reset() { position = 0; }
38 void skip(size_t s) { position += s; }
39 void swap(MemoryBuffer& o) { std::swap(position, o.position); buffer.swap(o.buffer); }
40 bool empty() const { return buffer.empty(); }
41 size_t size() const { return buffer.size(); }
42 void reserve(size_t s) { buffer.reserve(s); }
43 operator bool() const { return position < buffer.size(); }
44
46 inline static void copy(MemoryBuffer& from, MemoryBuffer& to);
47
49 static float growth_multiplier() { return 1.5; }
50
51 // simple file IO
52 void write(const std::string& fn) const { std::ofstream out(fn.c_str()); out.write(&buffer[0], size()); }
53 void read(const std::string& fn)
54 {
55 std::ifstream in(fn.c_str(), std::ios::binary | std::ios::ate);
56 buffer.resize(static_cast<size_t>(in.tellg()));
57 in.seekg(0);
58 in.read(&buffer[0], static_cast<std::streamsize>(size()));
59 position = 0;
60 }
61
62 size_t position;
63 std::vector<char> buffer;
64 };
65
66 namespace detail
67 {
68 struct Default {};
69 }
70
73
89 template<class T>
90 struct Serialization: public detail::Default
91 {
92 #if defined(__clang__) || (defined(__GNUC__) && __GNUC__ >= 5)
93 static_assert(std::is_trivially_copyable<T>::value, "Default serialization works only for trivially copyable types");
94 #endif
95
96 static void save(BinaryBuffer& bb, const T& x) { bb.save_binary((const char*) &x, sizeof(T)); }
97 static void load(BinaryBuffer& bb, T& x) { bb.load_binary((char*) &x, sizeof(T)); }
98 };
99
101 template<class T>
102 void save(BinaryBuffer& bb, const T& x) { Serialization<T>::save(bb, x); }
103
105 template<class T>
106 void load(BinaryBuffer& bb, T& x) { Serialization<T>::load(bb, x); }
107
110 template<class T>
111 void save(BinaryBuffer& bb, const T* x, size_t n);
112
115 template<class T>
116 void load(BinaryBuffer& bb, T* x, size_t n);
117
119 template<class T>
120 void load_back(BinaryBuffer& bb, T& x) { bb.load_binary_back((char*) &x, sizeof(T)); }
121
123
124
125 namespace detail
126 {
127 template<typename T>
128 struct is_default
129 {
130 typedef char yes;
131 typedef int no;
132
133 static yes test(Default*);
134 static no test(...);
135
136 enum { value = (sizeof(test((T*) 0)) == sizeof(yes)) };
137 };
138 }
139
140 template<class T>
141 void save(BinaryBuffer& bb, const T* x, size_t n)
142 {
143 if (!detail::is_default< Serialization<T> >::value)
144 for (size_t i = 0; i < n; ++i)
145 diy::save(bb, x[i]);
146 else // if Serialization is not specialized for U, just save the binary data
147 bb.save_binary((const char*) &x[0], sizeof(T)*n);
148 }
149
150 template<class T>
151 void load(BinaryBuffer& bb, T* x, size_t n)
152 {
153 if (!detail::is_default< Serialization<T> >::value)
154 for (size_t i = 0; i < n; ++i)
155 diy::load(bb, x[i]);
156 else // if Serialization is not specialized for U, just load the binary data
157 bb.load_binary((char*) &x[0], sizeof(T)*n);
158 }
159
160
161 // save/load for MemoryBuffer
162 template<>
163 struct Serialization< MemoryBuffer >
164 {
165 static void save(BinaryBuffer& bb, const MemoryBuffer& x)
166 {
167 diy::save(bb, x.position);
168 diy::save(bb, &x.buffer[0], x.position);
169 }
170
171 static void load(BinaryBuffer& bb, MemoryBuffer& x)
172 {
173 diy::load(bb, x.position);
174 x.buffer.resize(x.position);
175 diy::load(bb, &x.buffer[0], x.position);
176 }
177 };
178
179 // save/load for std::vector<U>
180 template<class U>
181 struct Serialization< std::vector<U> >
182 {
183 typedef std::vector<U> Vector;
184
185 static void save(BinaryBuffer& bb, const Vector& v)
186 {
187 size_t s = v.size();
188 diy::save(bb, s);
189 if (s > 0)
190 diy::save(bb, &v[0], v.size());
191 }
192
193 static void load(BinaryBuffer& bb, Vector& v)
194 {
195 size_t s;
196 diy::load(bb, s);
197 v.resize(s);
198 if (s > 0)
199 diy::load(bb, &v[0], s);
200 }
201 };
202
203 template<class U>
204 struct Serialization< std::valarray<U> >
205 {
206 typedef std::valarray<U> ValArray;
207
208 static void save(BinaryBuffer& bb, const ValArray& v)
209 {
210 size_t s = v.size();
211 diy::save(bb, s);
212 if (s > 0)
213 diy::save(bb, &v[0], v.size());
214 }
215
216 static void load(BinaryBuffer& bb, ValArray& v)
217 {
218 size_t s;
219 diy::load(bb, s);
220 v.resize(s);
221 if (s > 0)
222 diy::load(bb, &v[0], s);
223 }
224 };
225
226 // save/load for std::string
227 template<>
228 struct Serialization< std::string >
229 {
230 typedef std::string String;
231
232 static void save(BinaryBuffer& bb, const String& s)
233 {
234 size_t sz = s.size();
235 diy::save(bb, sz);
236 diy::save(bb, s.c_str(), sz);
237 }
238
239 static void load(BinaryBuffer& bb, String& s)
240 {
241 size_t sz;
242 diy::load(bb, sz);
243 s.resize(sz);
244 for (size_t i = 0; i < sz; ++i)
245 {
246 char c;
247 diy::load(bb, c);
248 s[i] = c;
249 }
250 }
251 };
252
253 // save/load for std::pair<X,Y>
254 template<class X, class Y>
255 struct Serialization< std::pair<X,Y> >
256 {
257 typedef std::pair<X,Y> Pair;
258
259 static void save(BinaryBuffer& bb, const Pair& p)
260 {
261 diy::save(bb, p.first);
262 diy::save(bb, p.second);
263 }
264
265 static void load(BinaryBuffer& bb, Pair& p)
266 {
267 diy::load(bb, p.first);
268 diy::load(bb, p.second);
269 }
270 };
271
272 // save/load for std::map<K,V>
273 template<class K, class V>
274 struct Serialization< std::map<K,V> >
275 {
276 typedef std::map<K,V> Map;
277
278 static void save(BinaryBuffer& bb, const Map& m)
279 {
280 size_t s = m.size();
281 diy::save(bb, s);
282 for (typename std::map<K,V>::const_iterator it = m.begin(); it != m.end(); ++it)
283 diy::save(bb, *it);
284 }
285
286 static void load(BinaryBuffer& bb, Map& m)
287 {
288 size_t s;
289 diy::load(bb, s);
290 for (size_t i = 0; i < s; ++i)
291 {
292 K k;
293 diy::load(bb, k);
294 diy::load(bb, m[k]);
295 }
296 }
297 };
298
299 // save/load for std::set<T>
300 template<class T>
301 struct Serialization< std::set<T> >
302 {
303 typedef std::set<T> Set;
304
305 static void save(BinaryBuffer& bb, const Set& m)
306 {
307 size_t s = m.size();
308 diy::save(bb, s);
309 for (typename std::set<T>::const_iterator it = m.begin(); it != m.end(); ++it)
310 diy::save(bb, *it);
311 }
312
313 static void load(BinaryBuffer& bb, Set& m)
314 {
315 size_t s;
316 diy::load(bb, s);
317 for (size_t i = 0; i < s; ++i)
318 {
319 T p;
320 diy::load(bb, p);
321 m.insert(p);
322 }
323 }
324 };
325
326 // save/load for std::unordered_map<K,V,H,E,A>
327 template<class K, class V, class H, class E, class A>
328 struct Serialization< std::unordered_map<K,V,H,E,A> >
329 {
330 typedef std::unordered_map<K,V,H,E,A> Map;
331
332 static void save(BinaryBuffer& bb, const Map& m)
333 {
334 size_t s = m.size();
335 diy::save(bb, s);
336 for (auto& x : m)
337 diy::save(bb, x);
338 }
339
340 static void load(BinaryBuffer& bb, Map& m)
341 {
342 size_t s;
343 diy::load(bb, s);
344 for (size_t i = 0; i < s; ++i)
345 {
346 std::pair<K,V> p;
347 diy::load(bb, p);
348 m.emplace(std::move(p));
349 }
350 }
351 };
352
353 // save/load for std::unordered_set<T,H,E,A>
354 template<class T, class H, class E, class A>
355 struct Serialization< std::unordered_set<T,H,E,A> >
356 {
357 typedef std::unordered_set<T,H,E,A> Set;
358
359 static void save(BinaryBuffer& bb, const Set& m)
360 {
361 size_t s = m.size();
362 diy::save(bb, s);
363 for (auto& x : m)
364 diy::save(bb, x);
365 }
366
367 static void load(BinaryBuffer& bb, Set& m)
368 {
369 size_t s;
370 diy::load(bb, s);
371 for (size_t i = 0; i < s; ++i)
372 {
373 T p;
374 diy::load(bb, p);
375 m.emplace(std::move(p));
376 }
377 }
378 };
379
380 // save/load for std::tuple<...>
381 // TODO: this ought to be default (copying) serialization
382 // if all arguments are default
383 template<class... Args>
384 struct Serialization< std::tuple<Args...> >
385 {
386 typedef std::tuple<Args...> Tuple;
387
388 static void save(BinaryBuffer& bb, const Tuple& t) { save<0>(bb, t); }
389
390 template<std::size_t I = 0>
391 static
392 typename std::enable_if<I == sizeof...(Args), void>::type
393 save(BinaryBuffer&, const Tuple&) {}
394
395 template<std::size_t I = 0>
396 static
397 typename std::enable_if<I < sizeof...(Args), void>::type
398 save(BinaryBuffer& bb, const Tuple& t) { diy::save(bb, std::get<I>(t)); save<I+1>(bb, t); }
399
400 static void load(BinaryBuffer& bb, Tuple& t) { load<0>(bb, t); }
401
402 template<std::size_t I = 0>
403 static
404 typename std::enable_if<I == sizeof...(Args), void>::type
405 load(BinaryBuffer&, Tuple&) {}
406
407 template<std::size_t I = 0>
408 static
409 typename std::enable_if<I < sizeof...(Args), void>::type
410 load(BinaryBuffer& bb, Tuple& t) { diy::load(bb, std::get<I>(t)); load<I+1>(bb, t); }
411
412 };
413 }
414
415 void
416 diy::MemoryBuffer::
417 save_binary(const char* x, size_t count)
418 {
419 if (position + count > buffer.capacity())
420 {
421 double newsize = static_cast<double>(position + count) * growth_multiplier(); // if we have to grow, grow geometrically
422 buffer.reserve(static_cast<size_t>(newsize));
423 }
424
425 if (position + count > buffer.size())
426 buffer.resize(position + count);
427
428 std::copy_n(x, count, &buffer[position]);
429 position += count;
430 }
431
432 void
433 diy::MemoryBuffer::
434 load_binary(char* x, size_t count)
435 {
436 std::copy_n(&buffer[position], count, x);
437 position += count;
438 }
439
440 void
441 diy::MemoryBuffer::
442 load_binary_back(char* x, size_t count)
443 {
444 std::copy_n(&buffer[buffer.size() - count], count, x);
445 buffer.resize(buffer.size() - count);
446 }
447
448 void
449 diy::MemoryBuffer::
450 copy(MemoryBuffer& from, MemoryBuffer& to)
451 {
452 size_t sz;
453 diy::load(from, sz);
454 from.position -= sizeof(size_t);
455
456 size_t total = sizeof(size_t) + sz;
457 to.buffer.resize(to.position + total);
458 std::copy_n(&from.buffer[from.position], total, &to.buffer[to.position]);
459 to.position += total;
460 from.position += total;
461 }
462
463 #endif
diy::load
void load(BinaryBuffer &bb, T &x)
Loads x from bb by calling diy::Serialization<T>::load(bb,x).
Definition: serialization.hpp:106
diy::load_back
void load_back(BinaryBuffer &bb, T &x)
Supports only binary data copying (meant for simple footers).
Definition: serialization.hpp:120
diy::BinaryBuffer::save_binary
virtual void save_binary(const char *x, size_t count)=0
copy count bytes from x into the buffer
diy::MemoryBuffer::load_binary
virtual void load_binary(char *x, size_t count) override
copy count bytes into x from the buffer
Definition: serialization.hpp:434
diy::MemoryBuffer::copy
static void copy(MemoryBuffer &from, MemoryBuffer &to)
copy a memory buffer from one buffer to another, bypassing making a temporary copy first ...
Definition: serialization.hpp:450
diy::MemoryBuffer::growth_multiplier
static float growth_multiplier()
multiplier used for the geometric growth of the container
Definition: serialization.hpp:49
diy::save
void save(BinaryBuffer &bb, const T &x)
Saves x to bb by calling diy::Serialization<T>::save(bb,x).
Definition: serialization.hpp:102
diy::MemoryBuffer::load_binary_back
virtual void load_binary_back(char *x, size_t count) override
copy count bytes into x from the back of the buffer
Definition: serialization.hpp:442
diy::in
void in(const RegularLink< Bounds > &link, const Point &p, OutIter out, const Bounds &domain)
Finds the neighbor(s) containing the target point.
Definition: pick.hpp:102
diy::MemoryBuffer
Definition: serialization.hpp:26
diy::BinaryBuffer::load_binary
virtual void load_binary(char *x, size_t count)=0
copy count bytes into x from the buffer
diy::MemoryBuffer::save_binary
virtual void save_binary(const char *x, size_t count) override
copy count bytes from x into the buffer
Definition: serialization.hpp:417
diy::Serialization
Main interface to serialization, meant to be specialized for the types that require special handling...
Definition: serialization.hpp:90
diy::BinaryBuffer::load_binary_back
virtual void load_binary_back(char *x, size_t count)=0
copy count bytes into x from the back of the buffer
diy::BinaryBuffer
A serialization buffer.
Definition: serialization.hpp:19