Libosmium  2.15.1
Fast and flexible C++ library for working with OpenStreetMap data
collector.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_RELATIONS_COLLECTOR_HPP
2 #define OSMIUM_RELATIONS_COLLECTOR_HPP
3 
4 /*
5 
6 This file is part of Osmium (https://osmcode.org/libosmium).
7 
8 Copyright 2013-2019 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <osmium/handler.hpp>
38 #include <osmium/memory/buffer.hpp>
39 #include <osmium/osm/item_type.hpp>
40 #include <osmium/osm/object.hpp>
41 #include <osmium/osm/relation.hpp>
42 #include <osmium/osm/types.hpp>
43 #include <osmium/relations/detail/member_meta.hpp>
44 #include <osmium/relations/detail/relation_meta.hpp>
45 #include <osmium/util/iterator.hpp>
46 #include <osmium/visitor.hpp>
47 
48 #include <algorithm>
49 #include <cassert>
50 #include <cstddef>
51 #include <cstdint>
52 #include <functional>
53 #include <iomanip>
54 #include <iostream>
55 #include <utility>
56 #include <vector>
57 
58 namespace osmium {
59 
60  class Node;
61  class Way;
62 
66  namespace relations {
67 
96  template <typename TCollector, bool TNodes, bool TWays, bool TRelations>
97  class Collector {
98 
103 
104  TCollector& m_collector;
105 
106  public:
107 
108  explicit HandlerPass1(TCollector& collector) noexcept :
109  m_collector(collector) {
110  }
111 
113  if (m_collector.keep_relation(relation)) {
114  m_collector.add_relation(relation);
115  }
116  }
117 
118  }; // class HandlerPass1
119 
120  public:
121 
126 
128  TCollector& m_collector;
129 
130  public:
131 
132  explicit HandlerPass2(TCollector& collector) noexcept :
133  m_collector(collector) {
134  }
135 
136  void node(const osmium::Node& node) {
137  if (TNodes) {
138  m_check_order.node(node);
139  if (!m_collector.find_and_add_object(node)) {
140  m_collector.node_not_in_any_relation(node);
141  }
142  }
143  }
144 
145  void way(const osmium::Way& way) {
146  if (TWays) {
147  m_check_order.way(way);
148  if (!m_collector.find_and_add_object(way)) {
149  m_collector.way_not_in_any_relation(way);
150  }
151  }
152  }
153 
155  if (TRelations) {
156  m_check_order.relation(relation);
157  if (!m_collector.find_and_add_object(relation)) {
158  m_collector.relation_not_in_any_relation(relation);
159  }
160  }
161  }
162 
163  void flush() {
164  m_collector.flush();
165  }
166 
167  }; // class HandlerPass2
168 
169  private:
170 
171  HandlerPass2 m_handler_pass2;
172 
173  // All relations we are interested in will be kept in this buffer
175 
176  // All members we are interested in will be kept in this buffer
178 
180  std::vector<RelationMeta> m_relations;
181 
186  using mm_vector_type = std::vector<MemberMeta>;
187  using mm_iterator = mm_vector_type::iterator;
189 
191 
192  using callback_func_type = std::function<void(osmium::memory::Buffer&&)>;
194 
195  enum {
196  initial_buffer_size = 1024ul * 1024ul
197  };
198 
200  auto& mmv = member_meta(type);
201  return make_range(std::equal_range(mmv.begin(), mmv.end(), MemberMeta(id)));
202  }
203 
204  public:
205 
210  m_handler_pass2(*static_cast<TCollector*>(this)),
211  m_relations_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
212  m_members_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes) {
213  }
214 
215  protected:
216 
217  std::vector<MemberMeta>& member_meta(const item_type type) {
218  return m_member_meta[static_cast<uint16_t>(type) - 1];
219  }
220 
222  return m_callback;
223  }
224 
225  const std::vector<RelationMeta>& relations() const {
226  return m_relations;
227  }
228 
238  bool keep_relation(const osmium::Relation& /*relation*/) const {
239  return true;
240  }
241 
252  bool keep_member(const RelationMeta& /*relation_meta*/, const osmium::RelationMember& /*member*/) const {
253  return true;
254  }
255 
263  void node_not_in_any_relation(const osmium::Node& /*node*/) {
264  }
265 
273  void way_not_in_any_relation(const osmium::Way& /*way*/) {
274  }
275 
283  void relation_not_in_any_relation(const osmium::Relation& /*relation*/) {
284  }
285 
297  void flush() {
298  }
299 
300  const osmium::Relation& get_relation(size_t offset) const {
301  assert(m_relations_buffer.committed() > offset);
302  return m_relations_buffer.get<osmium::Relation>(offset);
303  }
304 
308  const osmium::Relation& get_relation(const RelationMeta& relation_meta) const {
309  return get_relation(relation_meta.relation_offset());
310  }
311 
315  const osmium::Relation& get_relation(const MemberMeta& member_meta) const {
316  return get_relation(m_relations[member_meta.relation_pos()]);
317  }
318 
319  osmium::OSMObject& get_member(size_t offset) const {
320  assert(m_members_buffer.committed() > offset);
321  return m_members_buffer.get<osmium::OSMObject>(offset);
322  }
323 
324  private:
325 
335  const size_t offset = m_relations_buffer.committed();
336  m_relations_buffer.add_item(relation);
337 
338  RelationMeta relation_meta{offset};
339 
340  int n = 0;
341  for (auto& member : m_relations_buffer.get<osmium::Relation>(offset).members()) {
342  if (static_cast<TCollector*>(this)->keep_member(relation_meta, member)) {
343  member_meta(member.type()).emplace_back(member.ref(), m_relations.size(), n);
344  relation_meta.increment_need_members();
345  } else {
346  member.set_ref(0); // set member id to zero to indicate we are not interested
347  }
348  ++n;
349  }
350 
351  assert(offset == m_relations_buffer.committed());
352  if (relation_meta.has_all_members()) {
353  m_relations_buffer.rollback();
354  } else {
355  m_relations_buffer.commit();
356  m_relations.push_back(relation_meta);
357  }
358  }
359 
365  std::sort(m_member_meta[0].begin(), m_member_meta[0].end());
366  std::sort(m_member_meta[1].begin(), m_member_meta[1].end());
367  std::sort(m_member_meta[2].begin(), m_member_meta[2].end());
368  }
369 
371  return std::count_if(range.begin(), range.end(), [](MemberMeta& mm) {
372  return !mm.removed();
373  });
374  }
375 
384  auto range = find_member_meta(object.type(), object.id());
385 
386  if (count_not_removed(range) == 0) {
387  // nothing found
388  return false;
389  }
390 
391  {
392  members_buffer().add_item(object);
393  const size_t member_offset = members_buffer().commit();
394 
395  for (auto& member_meta : range) {
396  member_meta.set_buffer_offset(member_offset);
397  }
398  }
399 
400  for (auto& member_meta : range) {
401  if (member_meta.removed()) {
402  break;
403  }
404  assert(member_meta.member_id() == object.id());
405  assert(member_meta.relation_pos() < m_relations.size());
406  RelationMeta& relation_meta = m_relations[member_meta.relation_pos()];
407  assert(member_meta.member_pos() < get_relation(relation_meta).members().size());
408  relation_meta.got_one_member();
409  if (relation_meta.has_all_members()) {
410  const size_t relation_offset = member_meta.relation_pos();
411  static_cast<TCollector*>(this)->complete_relation(relation_meta);
412  clear_member_metas(relation_meta);
413  m_relations[relation_offset] = RelationMeta{};
415  }
416  }
417 
418  return true;
419  }
420 
421  void clear_member_metas(const RelationMeta& relation_meta) {
422  const osmium::Relation& relation = get_relation(relation_meta);
423  for (const auto& member : relation.members()) {
424  if (member.ref() != 0) {
425  const auto range = find_member_meta(member.type(), member.ref());
426  assert(!range.empty());
427 
428  // if this is the last time this object was needed
429  // then mark it as removed
430  if (count_not_removed(range) == 1) {
431  get_member(range.begin()->buffer_offset()).set_removed(true);
432  }
433 
434  for (auto& member_meta : range) {
435  if (!member_meta.removed() && relation.id() == get_relation(member_meta).id()) {
436  member_meta.remove();
437  break;
438  }
439  }
440  }
441  }
442  }
443 
444  public:
445 
446  uint64_t used_memory() const {
447  const uint64_t nmembers = m_member_meta[0].capacity() + m_member_meta[1].capacity() + m_member_meta[2].capacity();
448  const uint64_t members = nmembers * sizeof(MemberMeta);
449  const uint64_t relations = m_relations.capacity() * sizeof(RelationMeta);
450  const uint64_t relations_buffer_capacity = m_relations_buffer.capacity();
451  const uint64_t members_buffer_capacity = m_members_buffer.capacity();
452 
453  std::cerr << " nR = m_relations.capacity() ........... = " << std::setw(12) << m_relations.capacity() << "\n";
454  std::cerr << " nMN = m_member_meta[NODE].capacity() ... = " << std::setw(12) << m_member_meta[0].capacity() << "\n";
455  std::cerr << " nMW = m_member_meta[WAY].capacity() .... = " << std::setw(12) << m_member_meta[1].capacity() << "\n";
456  std::cerr << " nMR = m_member_meta[RELATION].capacity() = " << std::setw(12) << m_member_meta[2].capacity() << "\n";
457  std::cerr << " nM = m_member_meta[*].capacity() ...... = " << std::setw(12) << nmembers << "\n";
458 
459  std::cerr << " sRM = sizeof(RelationMeta) ............. = " << std::setw(12) << sizeof(RelationMeta) << "\n";
460  std::cerr << " sMM = sizeof(MemberMeta) ............... = " << std::setw(12) << sizeof(MemberMeta) << "\n\n";
461 
462  std::cerr << " nR * sRM ............................... = " << std::setw(12) << relations << "\n";
463  std::cerr << " nM * sMM ............................... = " << std::setw(12) << members << "\n";
464  std::cerr << " relations_buffer_capacity .............. = " << std::setw(12) << relations_buffer_capacity << "\n";
465  std::cerr << " members_buffer_capacity ................ = " << std::setw(12) << members_buffer_capacity << "\n";
466 
467  const uint64_t total = relations + members + relations_buffer_capacity + members_buffer_capacity;
468 
469  std::cerr << " total .................................. = " << std::setw(12) << total << "\n";
470  std::cerr << " =======================================================\n";
471 
472  return relations_buffer_capacity + members_buffer_capacity + relations + members;
473  }
474 
478  HandlerPass2& handler(const callback_func_type& callback = nullptr) {
479  m_callback = callback;
480  return m_handler_pass2;
481  }
482 
484  return m_members_buffer;
485  }
486 
499  const auto range = find_member_meta(type, id);
500  assert(!range.empty());
501  return range.begin()->is_available();
502  }
503 
514  const auto range = find_member_meta(type, id);
515  assert(!range.empty());
516  assert(range.begin()->is_available());
517  return range.begin()->buffer_offset();
518  }
519 
537  const auto range = find_member_meta(type, id);
538  assert(!range.empty());
539  if (range.begin()->is_available()) {
540  return std::make_pair(true, range.begin()->buffer_offset());
541  }
542  return std::make_pair(false, 0);
543  }
544 
545  template <typename TIter>
546  void read_relations(TIter begin, TIter end) {
547  HandlerPass1 handler(*static_cast<TCollector*>(this));
548  osmium::apply(begin, end, handler);
550  }
551 
552  template <typename TSource>
553  void read_relations(TSource& source) {
554  using std::begin;
555  using std::end;
556  read_relations(begin(source), end(source));
557  source.close();
558  }
559 
560  void moving_in_buffer(size_t old_offset, size_t new_offset) {
561  const osmium::OSMObject& object = m_members_buffer.get<osmium::OSMObject>(old_offset);
562  auto range = find_member_meta(object.type(), object.id());
563  for (auto& member_meta : range) {
564  assert(member_meta.buffer_offset() == old_offset);
565  member_meta.set_buffer_offset(new_offset);
566  }
567  }
568 
577  if (m_count_complete > 10000) { // XXX
578 // const size_t size_before = m_members_buffer.committed();
579  m_members_buffer.purge_removed(this);
580 /*
581  const size_t size_after = m_members_buffer.committed();
582  double percent = static_cast<double>(size_before - size_after);
583  percent /= size_before;
584  percent *= 100;
585  std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast<int>(percent) << "%)\n";
586 */
587  m_count_complete = 0;
588  }
589  }
590 
599  std::vector<const osmium::Relation*> get_incomplete_relations() const {
600  std::vector<const osmium::Relation*> relations;
601  for (const auto& relation_meta : m_relations) {
602  if (!relation_meta.has_all_members()) {
603  relations.push_back(&get_relation(relation_meta));
604  }
605  }
606  return relations;
607  }
608 
609  }; // class Collector
610 
611  } // namespace relations
612 
613 } // namespace osmium
614 
615 #endif // OSMIUM_RELATIONS_COLLECTOR_HPP
void relation(const osmium::Relation &relation)
Definition: collector.hpp:154
std::size_t capacity() const noexcept
Definition: buffer.hpp:348
Definition: iterator.hpp:42
bool keep_member(const RelationMeta &, const osmium::RelationMember &) const
Definition: collector.hpp:252
callback_func_type m_callback
Definition: collector.hpp:193
std::size_t commit()
Definition: buffer.hpp:468
osmium::memory::Buffer & members_buffer()
Definition: collector.hpp:483
osmium::OSMObject & get_member(size_t offset) const
Definition: collector.hpp:319
type
Definition: entity_bits.hpp:63
void way(const osmium::Way &) const noexcept
Definition: handler.hpp:81
RelationMemberList & members()
Get a reference to the member list.
Definition: relation.hpp:186
bool is_available(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:498
const std::vector< RelationMeta > & relations() const
Definition: collector.hpp:225
Definition: check_order.hpp:87
mm_vector_type m_member_meta[3]
Definition: collector.hpp:188
void node(const osmium::Node &) const noexcept
Definition: handler.hpp:78
bool keep_relation(const osmium::Relation &) const
Definition: collector.hpp:238
void way(const osmium::Way &way)
Definition: collector.hpp:145
item_type
Definition: item_type.hpp:43
std::vector< const osmium::Relation * > get_incomplete_relations() const
Definition: collector.hpp:599
InputIterator< Reader > end(Reader &)
Definition: reader_iterator.hpp:47
void moving_in_buffer(size_t old_offset, size_t new_offset)
Definition: collector.hpp:560
Definition: relation.hpp:168
void node(const osmium::Node &node)
Definition: check_order.hpp:95
Definition: handler.hpp:71
iterator_range< It > make_range(P &&p) noexcept
Definition: iterator.hpp:68
size_type size() const noexcept
Definition: collection.hpp:152
osmium::memory::Buffer m_members_buffer
Definition: collector.hpp:177
void read_relations(TSource &source)
Definition: collector.hpp:553
size_t get_offset(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:513
void relation_not_in_any_relation(const osmium::Relation &)
Definition: collector.hpp:283
Definition: way.hpp:72
std::vector< MemberMeta > & member_meta(const item_type type)
Definition: collector.hpp:217
bool find_and_add_object(const osmium::OSMObject &object)
Definition: collector.hpp:383
Definition: collector.hpp:125
HandlerPass2(TCollector &collector) noexcept
Definition: collector.hpp:132
std::vector< RelationMeta > m_relations
Vector with all relations we are interested in.
Definition: collector.hpp:180
std::pair< bool, size_t > get_availability_and_offset(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:536
std::size_t committed() const noexcept
Definition: buffer.hpp:356
void sort_member_meta()
Definition: collector.hpp:364
void node_not_in_any_relation(const osmium::Node &)
Definition: collector.hpp:263
void way(const osmium::Way &way)
Definition: check_order.hpp:112
InputIterator< Reader > begin(Reader &reader)
Definition: reader_iterator.hpp:43
std::function< void(osmium::memory::Buffer &&)> callback_func_type
Definition: collector.hpp:192
Definition: relation.hpp:57
TCollector & m_collector
Definition: collector.hpp:104
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
T & add_item(const T &item)
Definition: buffer.hpp:601
Definition: collector.hpp:97
iterator_range< mm_iterator > find_member_meta(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:199
It end() const noexcept
Definition: iterator.hpp:54
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:854
T & get(const std::size_t offset) const
Definition: buffer.hpp:518
HandlerPass2 m_handler_pass2
Definition: collector.hpp:171
Collector()
Definition: collector.hpp:209
void apply(TIterator it, TIterator end, THandlers &&... handlers)
Definition: visitor.hpp:218
const osmium::Relation & get_relation(const RelationMeta &relation_meta) const
Definition: collector.hpp:308
uint64_t used_memory() const
Definition: collector.hpp:446
void relation(const osmium::Relation &relation)
Definition: collector.hpp:112
Definition: collector.hpp:102
int64_t object_id_type
Type for OSM object (node, way, or relation) IDs.
Definition: types.hpp:45
void flush()
Definition: collector.hpp:163
std::vector< MemberMeta > mm_vector_type
Definition: collector.hpp:186
It begin() const noexcept
Definition: iterator.hpp:50
osmium::memory::Buffer m_relations_buffer
Definition: collector.hpp:174
void clear_member_metas(const RelationMeta &relation_meta)
Definition: collector.hpp:421
object_id_type id() const noexcept
Get ID of this object.
Definition: object.hpp:122
void relation(const osmium::Relation &relation)
Definition: check_order.hpp:126
osmium::handler::CheckOrder m_check_order
Definition: collector.hpp:127
callback_func_type callback()
Definition: collector.hpp:221
Definition: buffer.hpp:97
void add_relation(const osmium::Relation &relation)
Definition: collector.hpp:334
HandlerPass2 & handler(const callback_func_type &callback=nullptr)
Definition: collector.hpp:478
void flush()
Definition: collector.hpp:297
Definition: node.hpp:48
void way_not_in_any_relation(const osmium::Way &)
Definition: collector.hpp:273
int m_count_complete
Definition: collector.hpp:190
void node(const osmium::Node &node)
Definition: collector.hpp:136
void possibly_purge_removed_members()
Definition: collector.hpp:575
TCollector & m_collector
Definition: collector.hpp:128
void read_relations(TIter begin, TIter end)
Definition: collector.hpp:546
const osmium::Relation & get_relation(size_t offset) const
Definition: collector.hpp:300
HandlerPass1(TCollector &collector) noexcept
Definition: collector.hpp:108
void rollback()
Definition: buffer.hpp:484
static iterator_range< mm_iterator >::iterator::difference_type count_not_removed(const iterator_range< mm_iterator > &range)
Definition: collector.hpp:370
const osmium::Relation & get_relation(const MemberMeta &member_meta) const
Definition: collector.hpp:315
Definition: object.hpp:64