Libosmium  2.13.0
Fast and flexible C++ library for working with OpenStreetMap data
buffer.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_MEMORY_BUFFER_HPP
2 #define OSMIUM_MEMORY_BUFFER_HPP
3 
4 #include <iostream>
5 /*
6 
7 This file is part of Osmium (http://osmcode.org/libosmium).
8 
9 Copyright 2013-2017 Jochen Topf <jochen@topf.org> and others (see README).
10 
11 Boost Software License - Version 1.0 - August 17th, 2003
12 
13 Permission is hereby granted, free of charge, to any person or organization
14 obtaining a copy of the software and accompanying documentation covered by
15 this license (the "Software") to use, reproduce, display, distribute,
16 execute, and transmit the Software, and to prepare derivative works of the
17 Software, and to permit third-parties to whom the Software is furnished to
18 do so, all subject to the following:
19 
20 The copyright notices in the Software and this entire statement, including
21 the above license grant, this restriction and the following disclaimer,
22 must be included in all copies of the Software, in whole or in part, and
23 all derivative works of the Software, unless such copies or derivative
24 works are solely in the form of machine-executable object code generated by
25 a source language processor.
26 
27 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
28 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
29 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
30 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
31 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
32 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
33 DEALINGS IN THE SOFTWARE.
34 
35 */
36 
37 #include <algorithm>
38 #include <cassert>
39 #include <cstddef>
40 #include <cstring>
41 #include <functional>
42 #include <iterator>
43 #include <memory>
44 #include <stdexcept>
45 #include <utility>
46 
47 #include <osmium/memory/item.hpp>
49 #include <osmium/osm/entity.hpp>
51 
52 namespace osmium {
53 
59  struct buffer_is_full : public std::runtime_error {
60 
62  std::runtime_error{"Osmium buffer is full"} {
63  }
64 
65  }; // struct buffer_is_full
66 
70  namespace memory {
71 
98  class Buffer {
99 
100  public:
101 
102  // This is needed so we can call std::back_inserter() on a Buffer.
103  using value_type = Item;
104 
105  enum class auto_grow : bool {
106  yes = true,
107  no = false
108  }; // enum class auto_grow
109 
110  private:
111 
112  std::unique_ptr<unsigned char[]> m_memory;
113  unsigned char* m_data;
114  std::size_t m_capacity;
115  std::size_t m_written;
116  std::size_t m_committed;
117 #ifndef NDEBUG
118  uint8_t m_builder_count = 0;
119 #endif
120  auto_grow m_auto_grow{auto_grow::no};
121  std::function<void(Buffer&)> m_full;
122 
123  static std::size_t calculate_capacity(std::size_t capacity) noexcept {
124  // The majority of all Nodes will fit into this size.
125  constexpr static const std::size_t min_capacity = 64;
126  if (capacity < min_capacity) {
127  return min_capacity;
128  }
129  return padded_length(capacity);
130  }
131 
132  public:
133 
142  Buffer() noexcept :
143  m_memory(),
144  m_data(nullptr),
145  m_capacity(0),
146  m_written(0),
147  m_committed(0) {
148  }
149 
160  explicit Buffer(unsigned char* data, std::size_t size) :
161  m_memory(),
162  m_data(data),
163  m_capacity(size),
164  m_written(size),
165  m_committed(size) {
166  if (size % align_bytes != 0) {
167  throw std::invalid_argument{"buffer size needs to be multiple of alignment"};
168  }
169  }
170 
183  explicit Buffer(unsigned char* data, std::size_t capacity, std::size_t committed) :
184  m_memory(),
185  m_data(data),
186  m_capacity(capacity),
187  m_written(committed),
188  m_committed(committed) {
189  if (capacity % align_bytes != 0) {
190  throw std::invalid_argument{"buffer capacity needs to be multiple of alignment"};
191  }
192  if (committed % align_bytes != 0) {
193  throw std::invalid_argument{"buffer parameter 'committed' needs to be multiple of alignment"};
194  }
195  if (committed > capacity) {
196  throw std::invalid_argument{"buffer parameter 'committed' can not be larger than capacity"};
197  }
198  }
199 
212  explicit Buffer(std::size_t capacity, auto_grow auto_grow = auto_grow::yes) :
213  m_memory(new unsigned char[calculate_capacity(capacity)]),
214  m_data(m_memory.get()),
215  m_capacity(calculate_capacity(capacity)),
216  m_written(0),
217  m_committed(0),
218  m_auto_grow(auto_grow) {
219  }
220 
221  // buffers can not be copied
222  Buffer(const Buffer&) = delete;
223  Buffer& operator=(const Buffer&) = delete;
224 
225  // buffers can be moved
226  Buffer(Buffer&&) = default;
227  Buffer& operator=(Buffer&&) = default;
228 
229  ~Buffer() = default;
230 
231 #ifndef NDEBUG
232  void increment_builder_count() noexcept {
233  ++m_builder_count;
234  }
235 
236  void decrement_builder_count() noexcept {
237  assert(m_builder_count > 0);
238  --m_builder_count;
239  }
240 
241  uint8_t builder_count() const noexcept {
242  return m_builder_count;
243  }
244 #endif
245 
251  unsigned char* data() const noexcept {
252  assert(m_data && "This must be a valid buffer");
253  return m_data;
254  }
255 
260  std::size_t capacity() const noexcept {
261  return m_capacity;
262  }
263 
268  std::size_t committed() const noexcept {
269  return m_committed;
270  }
271 
277  std::size_t written() const noexcept {
278  return m_written;
279  }
280 
287  bool is_aligned() const noexcept {
288  assert(m_data && "This must be a valid buffer");
289  return (m_written % align_bytes == 0) && (m_committed % align_bytes == 0);
290  }
291 
307  OSMIUM_DEPRECATED void set_full_callback(const std::function<void(Buffer&)>& full) {
308  assert(m_data && "This must be a valid buffer");
309  m_full = full;
310  }
311 
327  void grow(std::size_t size) {
328  assert(m_data && "This must be a valid buffer");
329  if (!m_memory) {
330  throw std::logic_error{"Can't grow Buffer if it doesn't use internal memory management."};
331  }
332  size = calculate_capacity(size);
333  if (m_capacity < size) {
334  std::unique_ptr<unsigned char[]> memory(new unsigned char[size]);
335  std::copy_n(m_memory.get(), m_capacity, memory.get());
336  using std::swap;
337  swap(m_memory, memory);
338  m_data = m_memory.get();
339  m_capacity = size;
340  }
341  }
342 
355  std::size_t commit() {
356  assert(m_data && "This must be a valid buffer");
357  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
358  assert(is_aligned());
359 
360  const std::size_t offset = m_committed;
361  m_committed = m_written;
362  return offset;
363  }
364 
371  void rollback() {
372  assert(m_data && "This must be a valid buffer");
373  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
374  m_written = m_committed;
375  }
376 
386  std::size_t clear() {
387  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
388  const std::size_t committed = m_committed;
389  m_written = 0;
390  m_committed = 0;
391  return committed;
392  }
393 
404  template <typename T>
405  T& get(const std::size_t offset) const {
406  assert(m_data && "This must be a valid buffer");
407  return *reinterpret_cast<T*>(&m_data[offset]);
408  }
409 
443  unsigned char* reserve_space(const std::size_t size) {
444  assert(m_data && "This must be a valid buffer");
445  // try to flush the buffer empty first.
446  if (m_written + size > m_capacity && m_full) {
447  m_full(*this);
448  }
449  // if there's still not enough space, then try growing the buffer.
450  if (m_written + size > m_capacity) {
451  if (m_memory && (m_auto_grow == auto_grow::yes)) {
452  // double buffer size until there is enough space
453  std::size_t new_capacity = m_capacity * 2;
454  while (m_written + size > new_capacity) {
455  new_capacity *= 2;
456  }
457  grow(new_capacity);
458  } else {
459  throw osmium::buffer_is_full{};
460  }
461  }
462  unsigned char* data = &m_data[m_written];
463  m_written += size;
464  return data;
465  }
466 
482  template <typename T>
483  T& add_item(const T& item) {
484  assert(m_data && "This must be a valid buffer");
485  unsigned char* target = reserve_space(item.padded_size());
486  std::copy_n(reinterpret_cast<const unsigned char*>(&item), item.padded_size(), target);
487  return *reinterpret_cast<T*>(target);
488  }
489 
501  void add_buffer(const Buffer& buffer) {
502  assert(m_data && "This must be a valid buffer");
503  assert(buffer && "Buffer parameter must be a valid buffer");
504  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
505  unsigned char* target = reserve_space(buffer.committed());
506  std::copy_n(buffer.data(), buffer.committed(), target);
507  }
508 
518  void push_back(const osmium::memory::Item& item) {
519  assert(m_data && "This must be a valid buffer");
520  assert(m_builder_count == 0 && "Make sure there are no Builder objects still in scope");
521  add_item(item);
522  commit();
523  }
524 
529  template <typename T>
531 
536  template <typename T>
538 
544 
550 
551  template <typename T>
553  return ItemIteratorRange<T>{m_data, m_data + m_committed};
554  }
555 
556  template <typename T>
558  return ItemIteratorRange<const T>{m_data, m_data + m_committed};
559  }
560 
569  template <typename T>
571  assert(m_data && "This must be a valid buffer");
572  return t_iterator<T>(m_data, m_data + m_committed);
573  }
574 
584  assert(m_data && "This must be a valid buffer");
585  return iterator(m_data, m_data + m_committed);
586  }
587 
597  template <typename T>
598  t_iterator<T> get_iterator(std::size_t offset) {
599  assert(m_data && "This must be a valid buffer");
600  return t_iterator<T>(m_data + offset, m_data + m_committed);
601  }
602 
612  iterator get_iterator(std::size_t offset) {
613  assert(m_data && "This must be a valid buffer");
614  return iterator(m_data + offset, m_data + m_committed);
615  }
616 
625  template <typename T>
627  assert(m_data && "This must be a valid buffer");
628  return t_iterator<T>(m_data + m_committed, m_data + m_committed);
629  }
630 
640  assert(m_data && "This must be a valid buffer");
641  return iterator(m_data + m_committed, m_data + m_committed);
642  }
643 
644  template <typename T>
646  assert(m_data && "This must be a valid buffer");
647  return t_const_iterator<T>(m_data, m_data + m_committed);
648  }
649 
651  assert(m_data && "This must be a valid buffer");
652  return const_iterator(m_data, m_data + m_committed);
653  }
654 
655  template <typename T>
656  t_const_iterator<T> get_iterator(std::size_t offset) const {
657  assert(m_data && "This must be a valid buffer");
658  return t_const_iterator<T>(m_data + offset, m_data + m_committed);
659  }
660 
661  const_iterator get_iterator(std::size_t offset) const {
662  assert(m_data && "This must be a valid buffer");
663  return const_iterator(m_data + offset, m_data + m_committed);
664  }
665 
666  template <typename T>
668  assert(m_data && "This must be a valid buffer");
669  return t_const_iterator<T>(m_data + m_committed, m_data + m_committed);
670  }
671 
673  assert(m_data && "This must be a valid buffer");
674  return const_iterator(m_data + m_committed, m_data + m_committed);
675  }
676 
677  template <typename T>
679  return cbegin<T>();
680  }
681 
683  return cbegin();
684  }
685 
686  template <typename T>
688  return cend<T>();
689  }
690 
691  const_iterator end() const {
692  return cend();
693  }
694 
698  explicit operator bool() const noexcept {
699  return m_data != nullptr;
700  }
701 
702  void swap(Buffer& other) {
703  using std::swap;
704 
705  swap(m_memory, other.m_memory);
706  swap(m_data, other.m_data);
707  swap(m_capacity, other.m_capacity);
708  swap(m_written, other.m_written);
709  swap(m_committed, other.m_committed);
710  swap(m_auto_grow, other.m_auto_grow);
711  swap(m_full, other.m_full);
712  }
713 
730  template <typename TCallbackClass>
731  void purge_removed(TCallbackClass* callback) {
732  assert(m_data && "This must be a valid buffer");
733  if (begin() == end()) {
734  return;
735  }
736 
737  iterator it_write = begin();
738 
739  iterator next;
740  for (iterator it_read = begin(); it_read != end(); it_read = next) {
741  next = std::next(it_read);
742  if (!it_read->removed()) {
743  if (it_read != it_write) {
744  assert(it_read.data() >= data());
745  assert(it_write.data() >= data());
746  const auto old_offset = static_cast<std::size_t>(it_read.data() - data());
747  const auto new_offset = static_cast<std::size_t>(it_write.data() - data());
748  callback->moving_in_buffer(old_offset, new_offset);
749  std::memmove(it_write.data(), it_read.data(), it_read->padded_size());
750  }
751  it_write.advance_once();
752  }
753  }
754 
755  assert(it_write.data() >= data());
756  m_written = static_cast<std::size_t>(it_write.data() - data());
757  m_committed = m_written;
758  }
759 
760  }; // class Buffer
761 
762  inline void swap(Buffer& lhs, Buffer& rhs) {
763  lhs.swap(rhs);
764  }
765 
773  inline bool operator==(const Buffer& lhs, const Buffer& rhs) noexcept {
774  if (!lhs || !rhs) {
775  return !lhs && !rhs;
776  }
777  return lhs.data() == rhs.data() && lhs.capacity() == rhs.capacity() && lhs.committed() == rhs.committed();
778  }
779 
780  inline bool operator!=(const Buffer& lhs, const Buffer& rhs) noexcept {
781  return ! (lhs == rhs);
782  }
783 
784  } // namespace memory
785 
786 } // namespace osmium
787 
788 #endif // OSMIUM_MEMORY_BUFFER_HPP
void swap(Buffer &other)
Definition: buffer.hpp:702
std::size_t capacity() const noexcept
Definition: buffer.hpp:260
t_const_iterator< T > begin() const
Definition: buffer.hpp:678
std::size_t commit()
Definition: buffer.hpp:355
#define OSMIUM_DEPRECATED
Definition: compatibility.hpp:50
bool is_aligned() const noexcept
Definition: buffer.hpp:287
iterator get_iterator(std::size_t offset)
Definition: buffer.hpp:612
Definition: item_iterator.hpp:175
t_const_iterator< T > cend() const
Definition: buffer.hpp:667
const_iterator cend() const
Definition: buffer.hpp:672
Definition: item_iterator.hpp:59
OSMIUM_DEPRECATED void set_full_callback(const std::function< void(Buffer &)> &full)
Definition: buffer.hpp:307
constexpr bool operator==(const Box &lhs, const Box &rhs) noexcept
Definition: box.hpp:221
unsigned char * m_data
Definition: buffer.hpp:113
std::size_t m_committed
Definition: buffer.hpp:116
void increment_builder_count() noexcept
Definition: buffer.hpp:232
Definition: reader_iterator.hpp:39
ItemIteratorRange< const T > select() const
Definition: buffer.hpp:557
void swap(Buffer &lhs, Buffer &rhs)
Definition: buffer.hpp:762
t_const_iterator< T > get_iterator(std::size_t offset) const
Definition: buffer.hpp:656
std::size_t clear()
Definition: buffer.hpp:386
static std::size_t calculate_capacity(std::size_t capacity) noexcept
Definition: buffer.hpp:123
constexpr std::size_t padded_length(std::size_t length) noexcept
Definition: item.hpp:64
std::size_t committed() const noexcept
Definition: buffer.hpp:268
Buffer(std::size_t capacity, auto_grow auto_grow=auto_grow::yes)
Definition: buffer.hpp:212
constexpr const std::size_t align_bytes
Definition: item.hpp:62
const_iterator begin() const
Definition: buffer.hpp:682
std::size_t m_capacity
Definition: buffer.hpp:114
t_iterator< T > end()
Definition: buffer.hpp:626
Definition: item.hpp:105
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
T & add_item(const T &item)
Definition: buffer.hpp:483
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:731
t_iterator< T > begin()
Definition: buffer.hpp:570
void add_buffer(const Buffer &buffer)
Definition: buffer.hpp:501
std::size_t written() const noexcept
Definition: buffer.hpp:277
uint8_t builder_count() const noexcept
Definition: buffer.hpp:241
Buffer(unsigned char *data, std::size_t capacity, std::size_t committed)
Definition: buffer.hpp:183
Buffer(unsigned char *data, std::size_t size)
Definition: buffer.hpp:160
std::size_t m_written
Definition: buffer.hpp:115
ItemIterator< TMember > & advance_once() noexcept
Definition: item_iterator.hpp:114
unsigned char * data() const noexcept
Definition: buffer.hpp:251
Buffer() noexcept
Definition: buffer.hpp:142
osmium::io::InputIterator< osmium::io::Reader > end(osmium::io::Reader &)
Definition: reader_iterator.hpp:45
void grow(std::size_t size)
Definition: buffer.hpp:327
unsigned char * reserve_space(const std::size_t size)
Definition: buffer.hpp:443
auto_grow m_auto_grow
Definition: buffer.hpp:120
void push_back(const osmium::memory::Item &item)
Definition: buffer.hpp:518
iterator end()
Definition: buffer.hpp:639
iterator begin()
Definition: buffer.hpp:583
const_iterator get_iterator(std::size_t offset) const
Definition: buffer.hpp:661
Definition: buffer.hpp:98
t_const_iterator< T > cbegin() const
Definition: buffer.hpp:645
Definition: buffer.hpp:59
void decrement_builder_count() noexcept
Definition: buffer.hpp:236
const_iterator end() const
Definition: buffer.hpp:691
auto_grow
Definition: buffer.hpp:105
t_const_iterator< T > end() const
Definition: buffer.hpp:687
data_type data() noexcept
Definition: item_iterator.hpp:135
t_iterator< T > get_iterator(std::size_t offset)
Definition: buffer.hpp:598
buffer_is_full()
Definition: buffer.hpp:61
ItemIteratorRange< T > select()
Definition: buffer.hpp:552
osmium::io::InputIterator< osmium::io::Reader > begin(osmium::io::Reader &reader)
Definition: reader_iterator.hpp:41
void rollback()
Definition: buffer.hpp:371
std::unique_ptr< unsigned char[]> m_memory
Definition: buffer.hpp:112
bool operator!=(const Changeset &lhs, const Changeset &rhs)
Definition: changeset.hpp:440
const_iterator cbegin() const
Definition: buffer.hpp:650
std::function< void(Buffer &)> m_full
Definition: buffer.hpp:121