Libosmium  2.10.3
Fast and flexible C++ library for working with OpenStreetMap data
collector.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_RELATIONS_COLLECTOR_HPP
2 #define OSMIUM_RELATIONS_COLLECTOR_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <algorithm>
37 #include <cassert>
38 #include <cstddef>
39 #include <cstdint>
40 #include <functional>
41 #include <iomanip>
42 #include <iostream>
43 #include <utility>
44 #include <vector>
45 
46 #include <osmium/osm/item_type.hpp>
47 #include <osmium/osm/object.hpp>
48 #include <osmium/osm/relation.hpp>
49 #include <osmium/osm/types.hpp>
50 #include <osmium/handler.hpp>
51 #include <osmium/memory/buffer.hpp>
52 #include <osmium/util/iterator.hpp>
53 #include <osmium/visitor.hpp>
54 
55 #include <osmium/relations/detail/relation_meta.hpp>
56 #include <osmium/relations/detail/member_meta.hpp>
57 
58 namespace osmium {
59 
60  class Node;
61  class Way;
62 
66  namespace relations {
67 
68  namespace detail {
69 
70  } // namespace detail
71 
100  template <typename TCollector, bool TNodes, bool TWays, bool TRelations>
101  class Collector {
102 
107 
108  TCollector& m_collector;
109 
110  public:
111 
112  explicit HandlerPass1(TCollector& collector) noexcept :
113  m_collector(collector) {
114  }
115 
116  void relation(const osmium::Relation& relation) {
117  if (m_collector.keep_relation(relation)) {
118  m_collector.add_relation(relation);
119  }
120  }
121 
122  }; // class HandlerPass1
123 
124  public:
125 
130 
131  TCollector& m_collector;
132 
133  public:
134 
135  explicit HandlerPass2(TCollector& collector) noexcept :
136  m_collector(collector) {
137  }
138 
139  void node(const osmium::Node& node) {
140  if (TNodes) {
141  if (! m_collector.find_and_add_object(node)) {
142  m_collector.node_not_in_any_relation(node);
143  }
144  }
145  }
146 
147  void way(const osmium::Way& way) {
148  if (TWays) {
149  if (! m_collector.find_and_add_object(way)) {
150  m_collector.way_not_in_any_relation(way);
151  }
152  }
153  }
154 
155  void relation(const osmium::Relation& relation) {
156  if (TRelations) {
157  if (! m_collector.find_and_add_object(relation)) {
158  m_collector.relation_not_in_any_relation(relation);
159  }
160  }
161  }
162 
163  void flush() {
164  m_collector.flush();
165  }
166 
167  }; // class HandlerPass2
168 
169  private:
170 
171  HandlerPass2 m_handler_pass2;
172 
173  // All relations we are interested in will be kept in this buffer
175 
176  // All members we are interested in will be kept in this buffer
178 
180  std::vector<RelationMeta> m_relations;
181 
186  using mm_vector_type = std::vector<MemberMeta>;
187  using mm_iterator = mm_vector_type::iterator;
188  mm_vector_type m_member_meta[3];
189 
190  int m_count_complete = 0;
191 
192  using callback_func_type = std::function<void(osmium::memory::Buffer&&)>;
194 
195  static constexpr size_t initial_buffer_size = 1024 * 1024;
196 
198  auto& mmv = member_meta(type);
199  return make_range(std::equal_range(mmv.begin(), mmv.end(), MemberMeta(id)));
200  }
201 
202  public:
203 
208  m_handler_pass2(*static_cast<TCollector*>(this)),
209  m_relations_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
210  m_members_buffer(initial_buffer_size, osmium::memory::Buffer::auto_grow::yes),
211  m_relations(),
212  m_member_meta() {
213  }
214 
215  protected:
216 
217  std::vector<MemberMeta>& member_meta(const item_type type) {
218  return m_member_meta[static_cast<uint16_t>(type) - 1];
219  }
220 
222  return m_callback;
223  }
224 
225  const std::vector<RelationMeta>& relations() const {
226  return m_relations;
227  }
228 
238  bool keep_relation(const osmium::Relation& /*relation*/) const {
239  return true;
240  }
241 
252  bool keep_member(const osmium::relations::RelationMeta& /*relation_meta*/, const osmium::RelationMember& /*member*/) const {
253  return true;
254  }
255 
263  void node_not_in_any_relation(const osmium::Node& /*node*/) {
264  }
265 
273  void way_not_in_any_relation(const osmium::Way& /*way*/) {
274  }
275 
283  void relation_not_in_any_relation(const osmium::Relation& /*relation*/) {
284  }
285 
297  void flush() {
298  }
299 
305  m_relations.erase(
306  std::remove_if(m_relations.begin(), m_relations.end(), has_all_members()),
307  m_relations.end()
308  );
309  }
310 
311  const osmium::Relation& get_relation(size_t offset) const {
312  assert(m_relations_buffer.committed() > offset);
313  return m_relations_buffer.get<osmium::Relation>(offset);
314  }
315 
319  const osmium::Relation& get_relation(const RelationMeta& relation_meta) const {
320  return get_relation(relation_meta.relation_offset());
321  }
322 
326  const osmium::Relation& get_relation(const MemberMeta& member_meta) const {
327  return get_relation(m_relations[member_meta.relation_pos()]);
328  }
329 
330  osmium::OSMObject& get_member(size_t offset) const {
331  assert(m_members_buffer.committed() > offset);
332  return m_members_buffer.get<osmium::OSMObject>(offset);
333  }
334 
335  private:
336 
345  void add_relation(const osmium::Relation& relation) {
346  const size_t offset = m_relations_buffer.committed();
347  m_relations_buffer.add_item(relation);
348 
349  RelationMeta relation_meta(offset);
350 
351  int n = 0;
352  for (auto& member : m_relations_buffer.get<osmium::Relation>(offset).members()) {
353  if (static_cast<TCollector*>(this)->keep_member(relation_meta, member)) {
354  member_meta(member.type()).emplace_back(member.ref(), m_relations.size(), n);
355  relation_meta.increment_need_members();
356  } else {
357  member.set_ref(0); // set member id to zero to indicate we are not interested
358  }
359  ++n;
360  }
361 
362  assert(offset == m_relations_buffer.committed());
363  if (relation_meta.has_all_members()) {
364  m_relations_buffer.rollback();
365  } else {
366  m_relations_buffer.commit();
367  m_relations.push_back(std::move(relation_meta));
368  }
369  }
370 
376  std::sort(m_member_meta[0].begin(), m_member_meta[0].end());
377  std::sort(m_member_meta[1].begin(), m_member_meta[1].end());
378  std::sort(m_member_meta[2].begin(), m_member_meta[2].end());
379  }
380 
382  return std::count_if(range.begin(), range.end(), [](MemberMeta& mm) {
383  return !mm.removed();
384  });
385  }
386 
395  auto range = find_member_meta(object.type(), object.id());
396 
397  if (count_not_removed(range) == 0) {
398  // nothing found
399  return false;
400  }
401 
402  {
403  members_buffer().add_item(object);
404  const size_t member_offset = members_buffer().commit();
405 
406  for (auto& member_meta : range) {
407  member_meta.set_buffer_offset(member_offset);
408  }
409  }
410 
411  for (auto& member_meta : range) {
412  if (member_meta.removed()) {
413  break;
414  }
415  assert(member_meta.member_id() == object.id());
416  assert(member_meta.relation_pos() < m_relations.size());
417  RelationMeta& relation_meta = m_relations[member_meta.relation_pos()];
418  assert(member_meta.member_pos() < get_relation(relation_meta).members().size());
419  relation_meta.got_one_member();
420  if (relation_meta.has_all_members()) {
421  const size_t relation_offset = member_meta.relation_pos();
422  static_cast<TCollector*>(this)->complete_relation(relation_meta);
423  clear_member_metas(relation_meta);
424  m_relations[relation_offset] = RelationMeta();
425  possibly_purge_removed_members();
426  }
427  }
428 
429  return true;
430  }
431 
432  void clear_member_metas(const osmium::relations::RelationMeta& relation_meta) {
433  const osmium::Relation& relation = get_relation(relation_meta);
434  for (const auto& member : relation.members()) {
435  if (member.ref() != 0) {
436  const auto range = find_member_meta(member.type(), member.ref());
437  assert(!range.empty());
438 
439  // if this is the last time this object was needed
440  // then mark it as removed
441  if (count_not_removed(range) == 1) {
442  get_member(range.begin()->buffer_offset()).set_removed(true);
443  }
444 
445  for (auto& member_meta : range) {
446  if (!member_meta.removed() && relation.id() == get_relation(member_meta).id()) {
447  member_meta.remove();
448  break;
449  }
450  }
451  }
452  }
453  }
454 
455  public:
456 
457  uint64_t used_memory() const {
458  const uint64_t nmembers = m_member_meta[0].capacity() + m_member_meta[1].capacity() + m_member_meta[2].capacity();
459  const uint64_t members = nmembers * sizeof(MemberMeta);
460  const uint64_t relations = m_relations.capacity() * sizeof(RelationMeta);
461  const uint64_t relations_buffer_capacity = m_relations_buffer.capacity();
462  const uint64_t members_buffer_capacity = m_members_buffer.capacity();
463 
464  std::cerr << " nR = m_relations.capacity() ........... = " << std::setw(12) << m_relations.capacity() << "\n";
465  std::cerr << " nMN = m_member_meta[NODE].capacity() ... = " << std::setw(12) << m_member_meta[0].capacity() << "\n";
466  std::cerr << " nMW = m_member_meta[WAY].capacity() .... = " << std::setw(12) << m_member_meta[1].capacity() << "\n";
467  std::cerr << " nMR = m_member_meta[RELATION].capacity() = " << std::setw(12) << m_member_meta[2].capacity() << "\n";
468  std::cerr << " nM = m_member_meta[*].capacity() ...... = " << std::setw(12) << nmembers << "\n";
469 
470  std::cerr << " sRM = sizeof(RelationMeta) ............. = " << std::setw(12) << sizeof(RelationMeta) << "\n";
471  std::cerr << " sMM = sizeof(MemberMeta) ............... = " << std::setw(12) << sizeof(MemberMeta) << "\n\n";
472 
473  std::cerr << " nR * sRM ............................... = " << std::setw(12) << relations << "\n";
474  std::cerr << " nM * sMM ............................... = " << std::setw(12) << members << "\n";
475  std::cerr << " relations_buffer_capacity .............. = " << std::setw(12) << relations_buffer_capacity << "\n";
476  std::cerr << " members_buffer_capacity ................ = " << std::setw(12) << members_buffer_capacity << "\n";
477 
478  const uint64_t total = relations + members + relations_buffer_capacity + members_buffer_capacity;
479 
480  std::cerr << " total .................................. = " << std::setw(12) << total << "\n";
481  std::cerr << " =======================================================\n";
482 
483  return relations_buffer_capacity + members_buffer_capacity + relations + members;
484  }
485 
489  HandlerPass2& handler(const callback_func_type& callback = nullptr) {
490  m_callback = callback;
491  return m_handler_pass2;
492  }
493 
495  return m_members_buffer;
496  }
497 
510  const auto range = find_member_meta(type, id);
511  assert(!range.empty());
512  return range.begin()->is_available();
513  }
514 
525  const auto range = find_member_meta(type, id);
526  assert(!range.empty());
527  assert(range.begin()->is_available());
528  return range.begin()->buffer_offset();
529  }
530 
548  const auto range = find_member_meta(type, id);
549  assert(!range.empty());
550  if (range.begin()->is_available()) {
551  return std::make_pair(true, range.begin()->buffer_offset());
552  } else {
553  return std::make_pair(false, 0);
554  }
555  }
556 
557  template <typename TIter>
558  void read_relations(TIter begin, TIter end) {
559  HandlerPass1 handler(*static_cast<TCollector*>(this));
560  osmium::apply(begin, end, handler);
561  sort_member_meta();
562  }
563 
564  template <typename TSource>
565  void read_relations(TSource& source) {
566  read_relations(std::begin(source), std::end(source));
567  source.close();
568  }
569 
570  void moving_in_buffer(size_t old_offset, size_t new_offset) {
571  const osmium::OSMObject& object = m_members_buffer.get<osmium::OSMObject>(old_offset);
572  auto range = find_member_meta(object.type(), object.id());
573  for (auto& member_meta : range) {
574  assert(member_meta.buffer_offset() == old_offset);
575  member_meta.set_buffer_offset(new_offset);
576  }
577  }
578 
586  ++m_count_complete;
587  if (m_count_complete > 10000) { // XXX
588 // const size_t size_before = m_members_buffer.committed();
589  m_members_buffer.purge_removed(this);
590 /*
591  const size_t size_after = m_members_buffer.committed();
592  double percent = static_cast<double>(size_before - size_after);
593  percent /= size_before;
594  percent *= 100;
595  std::cerr << "PURGE (size before=" << size_before << " after=" << size_after << " purged=" << (size_before - size_after) << " / " << static_cast<int>(percent) << "%)\n";
596 */
597  m_count_complete = 0;
598  }
599  }
600 
609  std::vector<const osmium::Relation*> get_incomplete_relations() const {
610  std::vector<const osmium::Relation*> relations;
611  for (const auto& relation_meta : m_relations) {
612  if (!relation_meta.has_all_members()) {
613  relations.push_back(&get_relation(relation_meta));
614  }
615  }
616  return relations;
617  }
618 
619  }; // class Collector
620 
621  } // namespace relations
622 
623 } // namespace osmium
624 
625 #endif // OSMIUM_RELATIONS_COLLECTOR_HPP
void relation(const osmium::Relation &relation)
Definition: collector.hpp:155
void clear_member_metas(const osmium::relations::RelationMeta &relation_meta)
Definition: collector.hpp:432
Definition: iterator.hpp:43
callback_func_type m_callback
Definition: collector.hpp:193
osmium::memory::Buffer & members_buffer()
Definition: collector.hpp:494
osmium::OSMObject & get_member(size_t offset) const
Definition: collector.hpp:330
It begin() const
Definition: iterator.hpp:59
type
Definition: entity_bits.hpp:63
iterator_range< It > make_range(P &&p)
Definition: iterator.hpp:77
RelationMemberList & members()
Definition: relation.hpp:176
bool is_available(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:509
const std::vector< RelationMeta > & relations() const
Definition: collector.hpp:225
bool keep_relation(const osmium::Relation &) const
Definition: collector.hpp:238
void way(const osmium::Way &way)
Definition: collector.hpp:147
item_type
Definition: item_type.hpp:43
std::vector< const osmium::Relation * > get_incomplete_relations() const
Definition: collector.hpp:609
void clean_assembled_relations()
Definition: collector.hpp:304
void moving_in_buffer(size_t old_offset, size_t new_offset)
Definition: collector.hpp:570
Definition: relation.hpp:163
Definition: handler.hpp:71
osmium::memory::Buffer m_members_buffer
Definition: collector.hpp:177
void read_relations(TSource &source)
Definition: collector.hpp:565
bool keep_member(const osmium::relations::RelationMeta &, const osmium::RelationMember &) const
Definition: collector.hpp:252
size_t get_offset(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:524
void relation_not_in_any_relation(const osmium::Relation &)
Definition: collector.hpp:283
Definition: way.hpp:67
std::vector< MemberMeta > & member_meta(const item_type type)
Definition: collector.hpp:217
bool find_and_add_object(const osmium::OSMObject &object)
Definition: collector.hpp:394
Definition: collector.hpp:129
HandlerPass2(TCollector &collector) noexcept
Definition: collector.hpp:135
std::vector< RelationMeta > m_relations
Vector with all relations we are interested in.
Definition: collector.hpp:180
std::pair< bool, size_t > get_availability_and_offset(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:547
void sort_member_meta()
Definition: collector.hpp:375
void node_not_in_any_relation(const osmium::Node &)
Definition: collector.hpp:263
std::function< void(osmium::memory::Buffer &&)> callback_func_type
Definition: collector.hpp:192
Definition: relation.hpp:57
TCollector & m_collector
Definition: collector.hpp:108
Namespace for everything in the Osmium library.
Definition: assembler.hpp:73
T & add_item(const T &item)
Definition: buffer.hpp:482
Definition: collector.hpp:101
iterator_range< mm_iterator > find_member_meta(osmium::item_type type, osmium::object_id_type id)
Definition: collector.hpp:197
void purge_removed(TCallbackClass *callback)
Definition: buffer.hpp:730
Definition: attr.hpp:333
HandlerPass2 m_handler_pass2
Definition: collector.hpp:171
Collector()
Definition: collector.hpp:207
const osmium::Relation & get_relation(const RelationMeta &relation_meta) const
Definition: collector.hpp:319
uint64_t used_memory() const
Definition: collector.hpp:457
size_t capacity() const noexcept
Definition: buffer.hpp:251
void relation(const osmium::Relation &relation)
Definition: collector.hpp:116
osmium::io::InputIterator< osmium::io::Reader > end(osmium::io::Reader &)
Definition: reader_iterator.hpp:45
Definition: collector.hpp:106
int64_t object_id_type
Type for OSM object (node, way, or relation) IDs.
Definition: types.hpp:45
void flush()
Definition: collector.hpp:163
std::vector< MemberMeta > mm_vector_type
Definition: collector.hpp:186
osmium::memory::Buffer m_relations_buffer
Definition: collector.hpp:174
void apply(TIterator it, TIterator end, THandlers &... handlers)
Definition: visitor.hpp:234
object_id_type id() const noexcept
Get ID of this object.
Definition: object.hpp:118
size_t committed() const noexcept
Definition: buffer.hpp:259
callback_func_type callback()
Definition: collector.hpp:221
Definition: buffer.hpp:97
void add_relation(const osmium::Relation &relation)
Definition: collector.hpp:345
HandlerPass2 & handler(const callback_func_type &callback=nullptr)
Definition: collector.hpp:489
void flush()
Definition: collector.hpp:297
Definition: node.hpp:48
void way_not_in_any_relation(const osmium::Way &)
Definition: collector.hpp:273
void node(const osmium::Node &node)
Definition: collector.hpp:139
void possibly_purge_removed_members()
Definition: collector.hpp:585
TCollector & m_collector
Definition: collector.hpp:131
void read_relations(TIter begin, TIter end)
Definition: collector.hpp:558
osmium::io::InputIterator< osmium::io::Reader > begin(osmium::io::Reader &reader)
Definition: reader_iterator.hpp:41
const osmium::Relation & get_relation(size_t offset) const
Definition: collector.hpp:311
HandlerPass1(TCollector &collector) noexcept
Definition: collector.hpp:112
void rollback()
Definition: buffer.hpp:370
T & get(const size_t offset) const
Definition: buffer.hpp:404
static iterator_range< mm_iterator >::iterator::difference_type count_not_removed(const iterator_range< mm_iterator > &range)
Definition: collector.hpp:381
It end() const
Definition: iterator.hpp:63
const osmium::Relation & get_relation(const MemberMeta &member_meta) const
Definition: collector.hpp:326
Definition: object.hpp:63
size_t commit()
Definition: buffer.hpp:354