Libosmium  2.14.0
Fast and flexible C++ library for working with OpenStreetMap data
collector.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_RELATIONS_COLLECTOR_HPP
2 #define OSMIUM_RELATIONS_COLLECTOR_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2018 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <osmium/handler.hpp>
38 #include <osmium/memory/buffer.hpp>
39 #include <osmium/osm/item_type.hpp>
40 #include <osmium/osm/object.hpp>
41 #include <osmium/osm/relation.hpp>
42 #include <osmium/osm/types.hpp>
43 #include <osmium/relations/detail/member_meta.hpp>
44 #include <osmium/relations/detail/relation_meta.hpp>
45 #include <osmium/util/iterator.hpp>
46 #include <osmium/visitor.hpp>
47 
48 #include <algorithm>
49 #include <cassert>
50 #include <cstddef>
51 #include <cstdint>
52 #include <functional>
53 #include <iomanip>
54 #include <iostream>
55 #include <utility>
56 #include <vector>
57 
58 namespace osmium {
59 
60  class Node;
61  class Way;
62 
66  namespace relations {
67 
96  template <typename TCollector, bool TNodes, bool TWays, bool TRelations>
97  class Collector {
98 
103 
104  TCollector& m_collector;
105 
106  public:
107 
108  explicit HandlerPass1(TCollector& collector) noexcept :
109  m_collector(collector) {
110  }
111 
113  if (m_collector.keep_relation(relation)) {
114  m_collector.add_relation(relation);
115  }
116  }
117 
118  }; // class HandlerPass1
119 
120  public:
121 
126 
128  TCollector& m_collector;
129 
130  public:
131 
132  explicit HandlerPass2(TCollector& collector) noexcept :
133  m_collector(collector) {
134  }
135 
136  void node(const osmium::Node& node) {
137  if (TNodes) {
138  m_check_order.node(node);
139  if (!m_collector.find_and_add_object(node)) {
140  m_collector.node_not_in_any_relation(node);
141  }
142  }
143  }
144 
145  void way(const osmium::Way& way) {
146  if (TWays) {
147  m_check_order.way(way);
148  if (!m_collector.find_and_add_object(way)) {
149  m_collector.way_not_in_any_relation(way);
150  }
151  }
152  }
153 
155  if (TRelations) {
156  m_check_order.relation(relation);
157  if (!m_collector.find_and_add_object(relation)) {
158  m_collector.relation_not_in_any_relation(relation);
159  }
160  }
161  }
162 
163  void flush() {
164  m_collector.flush();
165  }
166 
167  }; // class HandlerPass2
168 
169  private:
170 
171  HandlerPass2 m_handler_pass2;
172 
173  // All relations we are interested in will be kept in this buffer
175 
176  // All members we are interested in will be kept in this buffer
178 
180  std::vector<RelationMeta> m_relations;
181 
186  using mm_vector_type = std::vector<MemberMeta>;
187  using mm_iterator = mm_vector_type::iterator;
189 
191 
192  using callback_func_type = std::function<void(osmium::memory::Buffer&&)>;
194 
195  static constexpr size_t initial_buffer_size = 1024 * 1024;
196 
198  auto& mmv = member_meta(type);
199  return make_range(std::equal_range(mmv.begin(), mmv.end(), MemberMeta(id)));
200  }
201 
202  public:
203 
208  m_handler_pass2(*static_cast<TCollector*>(this)),
209  m_relations_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
210  m_members_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes) {
211  }
212 
213  protected:
214 
215  std::vector<MemberMeta>& member_meta(const item_type type) {
216  return m_member_meta[static_cast<uint16_t>(type) - 1];
217  }
218 
220  return m_callback;
221  }
222 
223  const std::vector<RelationMeta>& relations() const {
224  return m_relations;
225  }
226 
236  bool keep_relation(const osmium::Relation& /*relation*/) const {
237  return true;
238  }
239 
250  bool keep_member(const RelationMeta& /*relation_meta*/, const osmium::RelationMember& /*member*/) const {
251  return true;
252  }
253 
261  void node_not_in_any_relation(const osmium::Node& /*node*/) {
262  }
263 
271  void way_not_in_any_relation(const osmium::Way& /*way*/) {
272  }
273 
281  void relation_not_in_any_relation(const osmium::Relation& /*relation*/) {
282  }
283 
295  void flush() {
296  }
297 
298  const osmium::Relation& get_relation(size_t offset) const {
299  assert(m_relations_buffer.committed() > offset);
300  return m_relations_buffer.get<osmium::Relation>(offset);
301  }
302 
306  const osmium::Relation& get_relation(const RelationMeta& relation_meta) const {
307  return get_relation(relation_meta.relation_offset());
308  }
309 
313  const osmium::Relation& get_relation(const MemberMeta& member_meta) const {
314  return get_relation(m_relations[member_meta.relation_pos()]);
315  }
316 
317  osmium::OSMObject& get_member(size_t offset) const {
318  assert(m_members_buffer.committed() > offset);
319  return m_members_buffer.get<osmium::OSMObject>(offset);
320  }
321 
322  private:
323 
333  const size_t offset = m_relations_buffer.committed();
334  m_relations_buffer.add_item(relation);
335 
336  RelationMeta relation_meta{offset};
337 
338  int n = 0;
339  for (auto& member : m_relations_buffer.get<osmium::Relation>(offset).members()) {
340  if (static_cast<TCollector*>(this)->keep_member(relation_meta, member)) {
341  member_meta(member.type()).emplace_back(member.ref(), m_relations.size(), n);
342  relation_meta.increment_need_members();
343  } else {
344  member.set_ref(0); // set member id to zero to indicate we are not interested
345  }
346  ++n;
347  }
348 
349  assert(offset == m_relations_buffer.committed());
350  if (relation_meta.has_all_members()) {
351  m_relations_buffer.rollback();
352  } else {
353  m_relations_buffer.commit();
354  m_relations.push_back(relation_meta);
355  }
356  }
357 
363  std::sort(m_member_meta[0].begin(), m_member_meta[0].end());
364  std::sort(m_member_meta[1].begin(), m_member_meta[1].end());
365  std::sort(m_member_meta[2].begin(), m_member_meta[2].end());
366  }
367 
369  return std::count_if(range.begin(), range.end(), [](MemberMeta& mm) {
370  return !mm.removed();
371  });
372  }
373 
382  auto range = find_member_meta(object.type(), object.id());
383 
384  if (count_not_removed(range) == 0) {
385  // nothing found
386  return false;
387  }
388 
389  {
390  members_buffer().add_item(object);
391  const size_t member_offset = members_buffer().commit();
392 
393  for (auto& member_meta : range) {
394  member_meta.set_buffer_offset(member_offset);
395  }
396  }
397 
398  for (auto& member_meta : range) {
399  if (member_meta.removed()) {
400  break;
401  }
402  assert(member_meta.member_id() == object.id());
403  assert(member_meta.relation_pos() < m_relations.size());
404  RelationMeta& relation_meta = m_relations[member_meta.relation_pos()];
405  assert(member_meta.member_pos() < get_relation(relation_meta).members().size());
406  relation_meta.got_one_member();
407  if (relation_meta.has_all_members()) {
408  const size_t relation_offset = member_meta.relation_pos();
409  static_cast<TCollector*>(this)->complete_relation(relation_meta);
410  clear_member_metas(relation_meta);
411  m_relations[relation_offset] = RelationMeta{};
413  }
414  }
415 
416  return true;
417  }
418 
419  void clear_member_metas(const RelationMeta& relation_meta) {
420  const osmium::Relation& relation = get_relation(relation_meta);
421  for (const auto& member : relation.members()) {
422  if (member.ref() != 0) {
423  const auto range = find_member_meta(member.type(), member.ref());
424  assert(!range.empty());
425 
426  // if this is the last time this object was needed
427  // then mark it as removed
428  if (count_not_removed(range) == 1) {
429  get_member(range.begin()->buffer_offset()).set_removed(true);
430  }
431 
432  for (auto& member_meta : range) {
433  if (!member_meta.removed() && relation.id() == get_relation(member_meta).id()) {
434  member_meta.remove();
435  break;
436  }
437  }
438  }
439  }
440  }
441 
442  public:
443 
444  uint64_t used_memory() const {
445  const uint64_t nmembers = m_member_meta[0].capacity() + m_member_meta[1].capacity() + m_member_meta[2].capacity();
446  const uint64_t members = nmembers * sizeof(MemberMeta);
447  const uint64_t relations = m_relations.capacity() * sizeof(RelationMeta);
448  const uint64_t relations_buffer_capacity = m_relations_buffer.capacity();
449  const uint64_t members_buffer_capacity = m_members_buffer.capacity();
450 
451  std::cerr << " nR = m_relations.capacity() ........... = " << std::setw(12) << m_relations.capacity() << "\n";
452  std::cerr << " nMN = m_member_meta[NODE].capacity() ... = " << std::setw(12) << m_member_meta[0].capacity() << "\n";
453  std::cerr << " nMW = m_member_meta[WAY].capacity() .... = " << std::setw(12) << m_member_meta[1].capacity() << "\n";
454  std::cerr << " nMR = m_member_meta[RELATION].capacity() = " << std::setw(12) << m_member_meta[2].capacity() << "\n";
455  std::cerr << " nM = m_member_meta[*].capacity() ...... = " << std::setw(12) << nmembers << "\n";
456 
457  std::cerr << " sRM = sizeof(RelationMeta) ............. = " << std::setw(12) << sizeof(RelationMeta) << "\n";
458  std::cerr << " sMM = sizeof(MemberMeta) ............... = " << std::setw(12) << sizeof(MemberMeta) << "\n\n";
459 
460  std::cerr << " nR * sRM ............................... = " << std::setw(12) << relations << "\n";
461  std::cerr << " nM * sMM ............................... = " << std::setw(12) << members << "\n";
462  std::cerr << " relations_buffer_capacity .............. = " << std::setw(12) << relations_buffer_capacity << "\n";
463  std::cerr << " members_buffer_capacity ................ = " << std::setw(12) << members_buffer_capacity << "\n";
464 
465  const uint64_t total = relations + members + relations_buffer_capacity + members_buffer_capacity;
466 
467  std::cerr << " total .................................. = " << std::setw(12) << total << "\n";
468  std::cerr << " =======================================================\n";
469 
470  return relations_buffer_capacity + members_buffer_capacity + relations + members;
471  }
472 
476  HandlerPass2& handler(const callback_func_type& callback = nullptr) {
477  m_callback = callback;
478  return m_handler_pass2;
479  }
480 
482  return m_members_buffer;
483  }
484 
497  const auto range = find_member_meta(type, id);
498  assert(!range.empty());
499  return range.begin()->is_available();
500  }
501 
512  const auto range = find_member_meta(type, id);
513  assert(!range.empty());
514  assert(range.begin()->is_available());
515  return range.begin()->buffer_offset();
516  }
517 
535  const auto range = find_member_meta(type, id);
536  assert(!range.empty());
537  if (range.begin()->is_available()) {
538  return std::make_pair(true, range.begin()->buffer_offset());
539  }
540  return std::make_pair(false, 0);
541  }
542 
543  template <typename TIter>
544  void read_relations(TIter begin, TIter end) {
545  HandlerPass1 handler(*static_cast<TCollector*>(this));
546  osmium::apply(begin, end, handler);
548  }
549 
550  template <typename TSource>
551  void read_relations(TSource& source) {
552  using std::begin;
553  using std::end;
554  read_relations(begin(source), end(source));
555  source.close();
556  }
557 
558  void moving_in_buffer(size_t old_offset, size_t new_offset) {
559  const osmium::OSMObject& object = m_members_buffer.get<osmium::OSMObject>(old_offset);
560  auto range = find_member_meta(object.type(), object.id());
561  for (auto& member_meta : range) {
562  assert(member_meta.buffer_offset() == old_offset);
563  member_meta.set_buffer_offset(new_offset);
564  }
565  }
566 
575  if (m_count_complete > 10000) { // XXX
576 // const size_t size_before = m_members_buffer.committed();
577  m_members_buffer.purge_removed(this);
578 /*
579  const size_t size_after = m_members_buffer.committed();
580  double percent = static_cast<double>(size_before - size_after);
581  percent /= size_before;
582  percent *= 100;
583  std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast<int>(percent) << "%)\n";
584 */
585  m_count_complete = 0;
586  }
587  }
588 
597  std::vector<const osmium::Relation*> get_incomplete_relations() const {
598  std::vector<const osmium::Relation*> relations;
599  for (const auto& relation_meta : m_relations) {
600  if (!relation_meta.has_all_members()) {
601  relations.push_back(&get_relation(relation_meta));
602  }
603  }
604  return relations;
605  }
606 
607  }; // class Collector
608 
609  } // namespace relations
610 
611 } // namespace osmium
612 
613 #endif // OSMIUM_RELATIONS_COLLECTOR_HPP
void relation(const osmium::Relation &relation)
Definition: collector.hpp:154
std::size_t capacity() const noexcept
Definition: buffer.hpp:253
Definition: iterator.hpp:42
bool keep_member(const RelationMeta &, const osmium::RelationMember &) const
Definition: collector.hpp:250
callback_func_type m_callback
Definition: collector.hpp:193
std::size_t commit()
Definition: buffer.hpp:348
osmium::memory::Buffer & members_buffer()
Definition: collector.hpp:481
osmium::OSMObject & get_member(size_t offset) const
Definition: collector.hpp:317
type
Definition: entity_bits.hpp:63
void way(const osmium::Way &) const noexcept
Definition: handler.hpp:81
RelationMemberList & members()
Get a reference to the member list.
Definition: relation.hpp:186
bool is_available(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:496
const std::vector< RelationMeta > & relations() const
Definition: collector.hpp:223
Definition: check_order.hpp:87
mm_vector_type m_member_meta[3]
Definition: collector.hpp:188
void node(const osmium::Node &) const noexcept
Definition: handler.hpp:78
bool keep_relation(const osmium::Relation &) const
Definition: collector.hpp:236
void way(const osmium::Way &way)
Definition: collector.hpp:145
item_type
Definition: item_type.hpp:43
std::vector< const osmium::Relation * > get_incomplete_relations() const
Definition: collector.hpp:597
InputIterator< Reader > end(Reader &)
Definition: reader_iterator.hpp:47
void moving_in_buffer(size_t old_offset, size_t new_offset)
Definition: collector.hpp:558
Definition: relation.hpp:168
static constexpr size_t initial_buffer_size
Definition: collector.hpp:195
void node(const osmium::Node &node)
Definition: check_order.hpp:95
Definition: handler.hpp:71
iterator_range< It > make_range(P &&p) noexcept
Definition: iterator.hpp:68
size_type size() const noexcept
Definition: collection.hpp:152
osmium::memory::Buffer m_members_buffer
Definition: collector.hpp:177
void read_relations(TSource &source)
Definition: collector.hpp:551
size_t get_offset(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:511
void relation_not_in_any_relation(const osmium::Relation &)
Definition: collector.hpp:281
Definition: way.hpp:72
std::vector< MemberMeta > & member_meta(const item_type type)
Definition: collector.hpp:215
bool find_and_add_object(const osmium::OSMObject &object)
Definition: collector.hpp:381
Definition: collector.hpp:125
HandlerPass2(TCollector &collector) noexcept
Definition: collector.hpp:132
std::vector< RelationMeta > m_relations
Vector with all relations we are interested in.
Definition: collector.hpp:180
std::pair< bool, size_t > get_availability_and_offset(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:534
std::size_t committed() const noexcept
Definition: buffer.hpp:261
void sort_member_meta()
Definition: collector.hpp:362
void node_not_in_any_relation(const osmium::Node &)
Definition: collector.hpp:261
void way(const osmium::Way &way)
Definition: check_order.hpp:112
InputIterator< Reader > begin(Reader &reader)
Definition: reader_iterator.hpp:43
std::function< void(osmium::memory::Buffer &&)> callback_func_type
Definition: collector.hpp:192
Definition: relation.hpp:57
TCollector & m_collector
Definition: collector.hpp:104
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
T & add_item(const T &item)
Definition: buffer.hpp:476
Definition: collector.hpp:97
iterator_range< mm_iterator > find_member_meta(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:197
It end() const noexcept
Definition: iterator.hpp:54
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:724
T & get(const std::size_t offset) const
Definition: buffer.hpp:398
HandlerPass2 m_handler_pass2
Definition: collector.hpp:171
Collector()
Definition: collector.hpp:207
void apply(TIterator it, TIterator end, THandlers &&... handlers)
Definition: visitor.hpp:218
const osmium::Relation & get_relation(const RelationMeta &relation_meta) const
Definition: collector.hpp:306
uint64_t used_memory() const
Definition: collector.hpp:444
void relation(const osmium::Relation &relation)
Definition: collector.hpp:112
Definition: collector.hpp:102
int64_t object_id_type
Type for OSM object (node, way, or relation) IDs.
Definition: types.hpp:45
void flush()
Definition: collector.hpp:163
std::vector< MemberMeta > mm_vector_type
Definition: collector.hpp:186
It begin() const noexcept
Definition: iterator.hpp:50
osmium::memory::Buffer m_relations_buffer
Definition: collector.hpp:174
void clear_member_metas(const RelationMeta &relation_meta)
Definition: collector.hpp:419
object_id_type id() const noexcept
Get ID of this object.
Definition: object.hpp:122
void relation(const osmium::Relation &relation)
Definition: check_order.hpp:126
osmium::handler::CheckOrder m_check_order
Definition: collector.hpp:127
callback_func_type callback()
Definition: collector.hpp:219
Definition: buffer.hpp:97
void add_relation(const osmium::Relation &relation)
Definition: collector.hpp:332
HandlerPass2 & handler(const callback_func_type &callback=nullptr)
Definition: collector.hpp:476
void flush()
Definition: collector.hpp:295
Definition: node.hpp:48
void way_not_in_any_relation(const osmium::Way &)
Definition: collector.hpp:271
int m_count_complete
Definition: collector.hpp:190
void node(const osmium::Node &node)
Definition: collector.hpp:136
void possibly_purge_removed_members()
Definition: collector.hpp:573
TCollector & m_collector
Definition: collector.hpp:128
void read_relations(TIter begin, TIter end)
Definition: collector.hpp:544
const osmium::Relation & get_relation(size_t offset) const
Definition: collector.hpp:298
HandlerPass1(TCollector &collector) noexcept
Definition: collector.hpp:108
void rollback()
Definition: buffer.hpp:364
static iterator_range< mm_iterator >::iterator::difference_type count_not_removed(const iterator_range< mm_iterator > &range)
Definition: collector.hpp:368
const osmium::Relation & get_relation(const MemberMeta &member_meta) const
Definition: collector.hpp:313
Definition: object.hpp:64