Libosmium  2.1.0
Fast and flexible C++ library for working with OpenStreetMap data
buffer.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_MEMORY_BUFFER_HPP
2 #define OSMIUM_MEMORY_BUFFER_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <algorithm>
37 #include <cassert>
38 #include <cstddef>
39 #include <cstring>
40 #include <exception>
41 #include <functional>
42 #include <iterator>
43 #include <stdexcept>
44 #include <utility>
45 #include <vector>
46 
47 #include <osmium/memory/item.hpp>
49 #include <osmium/osm/entity.hpp>
50 
51 namespace osmium {
52 
58  struct buffer_is_full : public std::runtime_error {
59 
61  std::runtime_error("Osmium buffer is full") {
62  }
63 
64  }; // struct buffer_is_full
65 
69  namespace memory {
70 
95  class Buffer {
96 
97  public:
98 
99  enum class auto_grow : bool {
100  yes = true,
101  no = false
102  }; // enum class auto_grow
103 
104  private:
105 
106  std::vector<unsigned char> m_memory;
107  unsigned char* m_data;
108  size_t m_capacity;
109  size_t m_written;
110  size_t m_committed;
112  std::function<void(Buffer&)> m_full;
113 
114  public:
115 
116  typedef Item value_type;
117 
123  Buffer() noexcept :
124  m_memory(),
125  m_data(nullptr),
126  m_capacity(0),
127  m_written(0),
128  m_committed(0) {
129  }
130 
139  explicit Buffer(unsigned char* data, size_t size) :
140  m_memory(),
141  m_data(data),
142  m_capacity(size),
143  m_written(size),
144  m_committed(size) {
145  if (size % align_bytes != 0) {
146  throw std::invalid_argument("buffer size needs to be multiple of alignment");
147  }
148  }
149 
159  explicit Buffer(unsigned char* data, size_t capacity, size_t committed) :
160  m_memory(),
161  m_data(data),
162  m_capacity(capacity),
163  m_written(committed),
164  m_committed(committed) {
165  if (capacity % align_bytes != 0) {
166  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
167  }
168  if (committed % align_bytes != 0) {
169  throw std::invalid_argument("buffer parameter 'committed' needs to be multiple of alignment");
170  }
171  }
172 
180  m_memory(capacity),
181  m_data(m_memory.data()),
182  m_capacity(capacity),
183  m_written(0),
184  m_committed(0),
186  if (capacity % align_bytes != 0) {
187  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
188  }
189  }
190 
191  // buffers can not be copied
192  Buffer(const Buffer&) = delete;
193  Buffer& operator=(const Buffer&) = delete;
194 
195  // buffers can be moved
196  Buffer(Buffer&&) = default;
197  Buffer& operator=(Buffer&&) = default;
198 
199  ~Buffer() = default;
200 
204  unsigned char* data() const noexcept {
205  return m_data;
206  }
207 
211  size_t capacity() const noexcept {
212  return m_capacity;
213  }
214 
218  size_t committed() const noexcept {
219  return m_committed;
220  }
221 
226  size_t written() const noexcept {
227  return m_written;
228  }
229 
234  bool is_aligned() const noexcept {
235  return (m_written % align_bytes == 0) && (m_committed % align_bytes == 0);
236  }
237 
242  void set_full_callback(std::function<void(Buffer&)> full) {
243  m_full = full;
244  }
245 
254  void grow(size_t size) {
255  if (m_memory.empty()) {
256  throw std::logic_error("Can't grow Buffer if it doesn't use internal memory management.");
257  }
258  if (m_capacity < size) {
259  if (size % align_bytes != 0) {
260  throw std::invalid_argument("buffer capacity needs to be multiple of alignment");
261  }
262  m_memory.resize(size);
263  m_data = m_memory.data();
264  m_capacity = size;
265  }
266  }
267 
273  size_t commit() {
274  assert(is_aligned());
275 
276  const size_t offset = m_committed;
277  m_committed = m_written;
278  return offset;
279  }
280 
284  void rollback() {
285  m_written = m_committed;
286  }
287 
293  size_t clear() {
294  const size_t committed = m_committed;
295  m_written = 0;
296  m_committed = 0;
297  return committed;
298  }
299 
306  template <class T>
307  T& get(const size_t offset) const {
308  return *reinterpret_cast<T*>(&m_data[offset]);
309  }
310 
336  unsigned char* reserve_space(const size_t size) {
337  if (m_written + size > m_capacity) {
338  if (m_full) {
339  m_full(*this);
340  } else if (!m_memory.empty() && (m_auto_grow == auto_grow::yes)) {
341  // double buffer size until there is enough space
342  size_t new_capacity = m_capacity * 2;
343  while (m_written + size > new_capacity) {
344  new_capacity *= 2;
345  }
346  grow(new_capacity);
347  } else {
348  throw osmium::buffer_is_full();
349  }
350  }
351  unsigned char* data = &m_data[m_written];
352  m_written += size;
353  return data;
354  }
355 
367  template <class T>
368  T& add_item(const T& item) {
369  unsigned char* target = reserve_space(item.padded_size());
370  std::copy_n(reinterpret_cast<const unsigned char*>(&item), item.padded_size(), target);
371  return *reinterpret_cast<T*>(target);
372  }
373 
380  void add_buffer(const Buffer& buffer) {
381  unsigned char* target = reserve_space(buffer.committed());
382  std::copy_n(reinterpret_cast<const unsigned char*>(buffer.data()), buffer.committed(), target);
383  }
384 
389  void push_back(const osmium::memory::Item& item) {
390  add_item(item);
391  commit();
392  }
393 
398  template <class T>
400 
401  template <class T>
403 
406 
407  template <class T>
409  return t_iterator<T>(m_data, m_data + m_committed);
410  }
411 
412  iterator begin() {
413  return iterator(m_data, m_data + m_committed);
414  }
415 
416  template <class T>
418  return t_iterator<T>(m_data + m_committed, m_data + m_committed);
419  }
420 
421  iterator end() {
422  return iterator(m_data + m_committed, m_data + m_committed);
423  }
424 
425  template <class T>
427  return t_const_iterator<T>(m_data, m_data + m_committed);
428  }
429 
430  const_iterator cbegin() const {
431  return const_iterator(m_data, m_data + m_committed);
432  }
433 
434  template <class T>
436  return t_const_iterator<T>(m_data + m_committed, m_data + m_committed);
437  }
438 
439  const_iterator cend() const {
440  return const_iterator(m_data + m_committed, m_data + m_committed);
441  }
442 
443  template <class T>
445  return cbegin<T>();
446  }
447 
448  const_iterator begin() const {
449  return cbegin();
450  }
451 
452  template <class T>
454  return cend<T>();
455  }
456 
457  const_iterator end() const {
458  return cend();
459  }
460 
464  explicit operator bool() const {
465  return m_data != nullptr;
466  }
467 
468  friend void swap(Buffer& lhs, Buffer& rhs) {
469  using std::swap;
470 
471  swap(lhs.m_memory, rhs.m_memory);
472  swap(lhs.m_data, rhs.m_data);
473  swap(lhs.m_capacity, rhs.m_capacity);
474  swap(lhs.m_written, rhs.m_written);
475  swap(lhs.m_committed, rhs.m_committed);
476  }
477 
492  template <class TCallbackClass>
493  void purge_removed(TCallbackClass* callback) {
494  if (begin() == end()) {
495  return;
496  }
497 
498  iterator it_write = begin();
499 
500  iterator next;
501  for (iterator it_read = begin(); it_read != end(); it_read = next) {
502  next = std::next(it_read);
503  if (!it_read->removed()) {
504  if (it_read != it_write) {
505  assert(it_read.data() >= data());
506  assert(it_write.data() >= data());
507  size_t old_offset = static_cast<size_t>(it_read.data() - data());
508  size_t new_offset = static_cast<size_t>(it_write.data() - data());
509  callback->moving_in_buffer(old_offset, new_offset);
510  std::memmove(it_write.data(), it_read.data(), it_read->padded_size());
511  }
512  it_write.advance_once();
513  }
514  }
515 
516  assert(it_write.data() >= data());
517  m_written = static_cast<size_t>(it_write.data() - data());
518  m_committed = m_written;
519  }
520 
521  }; // class Buffer
522 
523  inline bool operator==(const Buffer& lhs, const Buffer& rhs) noexcept {
524  return lhs.data() == rhs.data() && lhs.capacity() == rhs.capacity() && lhs.committed() == rhs.committed();
525  }
526 
527  inline bool operator!=(const Buffer& lhs, const Buffer& rhs) noexcept {
528  return ! (lhs == rhs);
529  }
530 
531  } // namespace memory
532 
533 } // namespace osmium
534 
535 #endif // OSMIUM_MEMORY_BUFFER_HPP
size_t m_written
Definition: buffer.hpp:109
Item value_type
Definition: buffer.hpp:116
t_const_iterator< T > cend() const
Definition: buffer.hpp:435
const_iterator begin() const
Definition: buffer.hpp:448
size_t clear()
Definition: buffer.hpp:293
bool is_aligned() const noexcept
Definition: buffer.hpp:234
size_t written() const noexcept
Definition: buffer.hpp:226
bool operator!=(const Buffer &lhs, const Buffer &rhs) noexcept
Definition: buffer.hpp:527
void grow(size_t size)
Definition: buffer.hpp:254
const_iterator end() const
Definition: buffer.hpp:457
Definition: item_iterator.hpp:131
unsigned char * m_data
Definition: buffer.hpp:107
t_const_iterator< osmium::OSMEntity > const_iterator
Definition: buffer.hpp:405
constexpr item_size_type align_bytes
Definition: item.hpp:54
Buffer(unsigned char *data, size_t capacity, size_t committed)
Definition: buffer.hpp:159
Definition: reader_iterator.hpp:39
ItemIterator< TMember > & advance_once()
Definition: item_iterator.hpp:180
Buffer(size_t capacity, auto_grow auto_grow=auto_grow::yes)
Definition: buffer.hpp:179
unsigned char * data() const
Definition: item_iterator.hpp:201
bool operator==(const Buffer &lhs, const Buffer &rhs) noexcept
Definition: buffer.hpp:523
t_iterator< T > end()
Definition: buffer.hpp:417
Definition: item.hpp:98
Namespace for everything in the Osmium library.
Definition: assembler.hpp:55
T & add_item(const T &item)
Definition: buffer.hpp:368
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:493
t_iterator< T > begin()
Definition: buffer.hpp:408
void add_buffer(const Buffer &buffer)
Definition: buffer.hpp:380
friend void swap(Buffer &lhs, Buffer &rhs)
Definition: buffer.hpp:468
const_iterator cbegin() const
Definition: buffer.hpp:430
size_t m_committed
Definition: buffer.hpp:110
unsigned char * data() const noexcept
Definition: buffer.hpp:204
Buffer() noexcept
Definition: buffer.hpp:123
size_t capacity() const noexcept
Definition: buffer.hpp:211
unsigned char * reserve_space(const size_t size)
Definition: buffer.hpp:336
auto_grow m_auto_grow
Definition: buffer.hpp:111
void push_back(const osmium::memory::Item &item)
Definition: buffer.hpp:389
iterator end()
Definition: buffer.hpp:421
iterator begin()
Definition: buffer.hpp:412
const_iterator cend() const
Definition: buffer.hpp:439
size_t m_capacity
Definition: buffer.hpp:108
size_t committed() const noexcept
Definition: buffer.hpp:218
Buffer(unsigned char *data, size_t size)
Definition: buffer.hpp:139
t_iterator< osmium::OSMEntity > iterator
Definition: buffer.hpp:404
t_const_iterator< T > end() const
Definition: buffer.hpp:453
Definition: buffer.hpp:95
Definition: buffer.hpp:58
t_const_iterator< T > cbegin() const
Definition: buffer.hpp:426
t_const_iterator< T > begin() const
Definition: buffer.hpp:444
auto_grow
Definition: buffer.hpp:99
std::vector< unsigned char > m_memory
Definition: buffer.hpp:106
buffer_is_full()
Definition: buffer.hpp:60
void rollback()
Definition: buffer.hpp:284
void set_full_callback(std::function< void(Buffer &)> full)
Definition: buffer.hpp:242
std::function< void(Buffer &)> m_full
Definition: buffer.hpp:112
Buffer & operator=(const Buffer &)=delete
size_t commit()
Definition: buffer.hpp:273