Libosmium  2.13.0
Fast and flexible C++ library for working with OpenStreetMap data
reader.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_READER_HPP
2 #define OSMIUM_IO_READER_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2017 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <cerrno>
37 #include <cstdlib>
38 #include <fcntl.h>
39 #include <future>
40 #include <memory>
41 #include <string>
42 #include <system_error>
43 #include <thread>
44 #include <utility>
45 
46 #ifndef _WIN32
47 # include <sys/wait.h>
48 #endif
49 
50 #ifndef _MSC_VER
51 # include <unistd.h>
52 #endif
53 
55 #include <osmium/io/detail/input_format.hpp>
56 #include <osmium/io/detail/read_thread.hpp>
57 #include <osmium/io/detail/read_write.hpp>
58 #include <osmium/io/detail/queue_util.hpp>
59 #include <osmium/io/error.hpp>
60 #include <osmium/io/file.hpp>
61 #include <osmium/io/header.hpp>
62 #include <osmium/memory/buffer.hpp>
64 #include <osmium/thread/pool.hpp>
65 #include <osmium/thread/util.hpp>
66 #include <osmium/util/config.hpp>
67 
68 namespace osmium {
69 
70  namespace io {
71 
72  namespace detail {
73 
74  inline std::size_t get_input_queue_size() noexcept {
75  const std::size_t n = osmium::config::get_max_queue_size("INPUT", 20);
76  return n > 2 ? n : 2;
77  }
78 
79  inline std::size_t get_osmdata_queue_size() noexcept {
80  const std::size_t n = osmium::config::get_max_queue_size("OSMDATA", 20);
81  return n > 2 ? n : 2;
82  }
83 
84  } // namespace detail
85 
92  class Reader {
93 
95 
96  osmium::thread::Pool* m_pool = nullptr;
97 
98  detail::ParserFactory::create_parser_type m_creator;
99 
100  enum class status {
101  okay = 0, // normal reading
102  error = 1, // some error occurred while reading
103  closed = 2, // close() called
104  eof = 3 // eof of file was reached without error
105  } m_status;
106 
108 
109  detail::future_string_queue_type m_input_queue;
110 
111  std::unique_ptr<osmium::io::Decompressor> m_decompressor;
112 
113  osmium::io::detail::ReadThreadManager m_read_thread_manager;
114 
115  detail::future_buffer_queue_type m_osmdata_queue;
116  detail::queue_wrapper<osmium::memory::Buffer> m_osmdata_queue_wrapper;
117 
118  std::future<osmium::io::Header> m_header_future;
120 
122 
123  std::size_t m_file_size;
124 
127 
128  void set_option(osmium::thread::Pool& pool) noexcept {
129  m_pool = &pool;
130  }
131 
133  m_read_which_entities = value;
134  }
135 
136  void set_option(osmium::io::read_meta value) noexcept {
137  m_read_metadata = value;
138  }
139 
140  // This function will run in a separate thread.
142  const detail::ParserFactory::create_parser_type& creator,
143  detail::future_string_queue_type& input_queue,
144  detail::future_buffer_queue_type& osmdata_queue,
145  std::promise<osmium::io::Header>&& header_promise,
146  osmium::osm_entity_bits::type read_which_entities,
147  osmium::io::read_meta read_metadata) {
148  std::promise<osmium::io::Header> promise{std::move(header_promise)};
149  osmium::io::detail::parser_arguments args = {
150  pool,
151  input_queue,
152  osmdata_queue,
153  promise,
154  read_which_entities,
155  read_metadata
156  };
157  creator(args)->parse();
158  }
159 
160 #ifndef _WIN32
161 
172  static int execute(const std::string& command, const std::string& filename, int* childpid) {
173  int pipefd[2];
174  if (pipe(pipefd) < 0) {
175  throw std::system_error{errno, std::system_category(), "opening pipe failed"};
176  }
177  const pid_t pid = fork();
178  if (pid < 0) {
179  throw std::system_error{errno, std::system_category(), "fork failed"};
180  }
181  if (pid == 0) { // child
182  // close all file descriptors except one end of the pipe
183  for (int i = 0; i < 32; ++i) {
184  if (i != pipefd[1]) {
185  ::close(i);
186  }
187  }
188  if (dup2(pipefd[1], 1) < 0) { // put end of pipe as stdout/stdin
189  exit(1);
190  }
191 
192  ::open("/dev/null", O_RDONLY); // stdin
193  ::open("/dev/null", O_WRONLY); // stderr
194  // hack: -g switches off globbing in curl which allows [] to be used in file names
195  // this is important for XAPI URLs
196  // in theory this execute() function could be used for other commands, but it is
197  // only used for curl at the moment, so this is okay.
198  if (::execlp(command.c_str(), command.c_str(), "-g", filename.c_str(), nullptr) < 0) {
199  exit(1);
200  }
201  }
202  // parent
203  *childpid = pid;
204  ::close(pipefd[1]);
205  return pipefd[0];
206  }
207 #endif
208 
217  static int open_input_file_or_url(const std::string& filename, int* childpid) {
218  const std::string protocol{filename.substr(0, filename.find_first_of(':'))};
219  if (protocol == "http" || protocol == "https" || protocol == "ftp" || protocol == "file") {
220 #ifndef _WIN32
221  return execute("curl", filename, childpid);
222 #else
223  throw io_error{"Reading OSM files from the network currently not supported on Windows."};
224 #endif
225  }
226  return osmium::io::detail::open_for_reading(filename);
227  }
228 
229  public:
230 
253  template <typename... TArgs>
254  explicit Reader(const osmium::io::File& file, TArgs&&... args) :
255  m_file(file.check()),
256  m_creator(detail::ParserFactory::instance().get_creator_function(m_file)),
257  m_status(status::okay),
258  m_childpid(0),
259  m_input_queue(detail::get_input_queue_size(), "raw_input"),
260  m_decompressor(m_file.buffer() ?
261  osmium::io::CompressionFactory::instance().create_decompressor(file.compression(), m_file.buffer(), m_file.buffer_size()) :
262  osmium::io::CompressionFactory::instance().create_decompressor(file.compression(), open_input_file_or_url(m_file.filename(), &m_childpid))),
263  m_read_thread_manager(*m_decompressor, m_input_queue),
264  m_osmdata_queue(detail::get_osmdata_queue_size(), "parser_results"),
265  m_osmdata_queue_wrapper(m_osmdata_queue),
266  m_header_future(),
267  m_header(),
268  m_thread(),
269  m_file_size(m_decompressor->file_size()) {
270 
271  (void)std::initializer_list<int>{
272  (set_option(args), 0)...
273  };
274 
275  if (!m_pool) {
276  m_pool = &thread::Pool::default_instance();
277  }
278 
279  std::promise<osmium::io::Header> header_promise;
280  m_header_future = header_promise.get_future();
281  m_thread = osmium::thread::thread_handler{parser_thread, std::ref(*m_pool), std::ref(m_creator), std::ref(m_input_queue), std::ref(m_osmdata_queue), std::move(header_promise), m_read_which_entities, m_read_metadata};
282  }
283 
284  template <typename... TArgs>
285  explicit Reader(const std::string& filename, TArgs&&... args) :
286  Reader(osmium::io::File(filename), std::forward<TArgs>(args)...) {
287  }
288 
289  template <typename... TArgs>
290  explicit Reader(const char* filename, TArgs&&... args) :
291  Reader(osmium::io::File(filename), std::forward<TArgs>(args)...) {
292  }
293 
294  Reader(const Reader&) = delete;
295  Reader& operator=(const Reader&) = delete;
296 
297  Reader(Reader&&) = default;
298  Reader& operator=(Reader&&) = default;
299 
300  ~Reader() noexcept {
301  try {
302  close();
303  } catch (...) {
304  // Ignore any exceptions because destructor must not throw.
305  }
306  }
307 
316  void close() {
317  m_status = status::closed;
318 
319  m_read_thread_manager.stop();
320 
321  m_osmdata_queue_wrapper.drain();
322 
323  try {
324  m_read_thread_manager.close();
325  } catch (...) {
326  // Ignore any exceptions.
327  }
328 
329 #ifndef _WIN32
330  if (m_childpid) {
331  int status;
332  const pid_t pid = ::waitpid(m_childpid, &status, 0);
333 #pragma GCC diagnostic push
334 #pragma GCC diagnostic ignored "-Wold-style-cast"
335  if (pid < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) {
336  throw std::system_error{errno, std::system_category(), "subprocess returned error"};
337  }
338 #pragma GCC diagnostic pop
339  m_childpid = 0;
340  }
341 #endif
342  }
343 
351  if (m_status == status::error) {
352  throw io_error{"Can not get header from reader when in status 'error'"};
353  }
354 
355  try {
356  if (m_header_future.valid()) {
357  m_header = m_header_future.get();
358  }
359  } catch (...) {
360  close();
361  m_status = status::error;
362  throw;
363  }
364 
365  return m_header;
366  }
367 
377  osmium::memory::Buffer buffer;
378 
379  if (m_status != status::okay) {
380  throw io_error{"Can not read from reader when in status 'closed', 'eof', or 'error'"};
381  }
382 
383  if (m_read_which_entities == osmium::osm_entity_bits::nothing) {
384  m_status = status::eof;
385  return buffer;
386  }
387 
388  try {
389  // m_input_format.read() can return an invalid buffer to signal EOF,
390  // or a valid buffer with or without data. A valid buffer
391  // without data is not an error, it just means we have to
392  // keep getting the next buffer until there is one with data.
393  while (true) {
394  buffer = m_osmdata_queue_wrapper.pop();
395  if (detail::at_end_of_data(buffer)) {
396  m_status = status::eof;
397  m_read_thread_manager.close();
398  return buffer;
399  }
400  if (buffer.committed() > 0) {
401  return buffer;
402  }
403  }
404  } catch (...) {
405  close();
406  m_status = status::error;
407  throw;
408  }
409  }
410 
415  bool eof() const {
416  return m_status == status::eof || m_status == status::closed;
417  }
418 
423  std::size_t file_size() const noexcept {
424  return m_file_size;
425  }
426 
441  std::size_t offset() const noexcept {
442  return m_decompressor->offset();
443  }
444 
445  }; // class Reader
446 
455  template <typename... TArgs>
458 
459  Reader reader{std::forward<TArgs>(args)...};
460  while (auto read_buffer = reader.read()) {
461  buffer.add_buffer(read_buffer);
462  buffer.commit();
463  }
464 
465  return buffer;
466  }
467 
468  } // namespace io
469 
470 } // namespace osmium
471 
472 #endif // OSMIUM_IO_READER_HPP
detail::queue_wrapper< osmium::memory::Buffer > m_osmdata_queue_wrapper
Definition: reader.hpp:116
status
Definition: reader.hpp:100
osmium::memory::Buffer read()
Definition: reader.hpp:376
Reader(const osmium::io::File &file, TArgs &&... args)
Definition: reader.hpp:254
type
Definition: entity_bits.hpp:63
std::size_t get_max_queue_size(const char *queue_name, std::size_t default_value) noexcept
Definition: config.hpp:71
std::size_t file_size(int fd)
Definition: file.hpp:70
std::size_t file_size() const noexcept
Definition: reader.hpp:423
bool eof() const
Definition: reader.hpp:415
int m_childpid
Definition: reader.hpp:107
static Pool & default_instance()
Definition: pool.hpp:169
Reader(const char *filename, TArgs &&... args)
Definition: reader.hpp:290
std::future< osmium::io::Header > m_header_future
Definition: reader.hpp:118
Definition: reader_iterator.hpp:39
std::unique_ptr< osmium::io::Decompressor > m_decompressor
Definition: reader.hpp:111
std::size_t committed() const noexcept
Definition: buffer.hpp:268
object or changeset
Definition: entity_bits.hpp:76
detail::future_string_queue_type m_input_queue
Definition: reader.hpp:109
osmium::memory::Buffer read_file(TArgs &&... args)
Definition: reader.hpp:456
osmium::io::File m_file
Definition: reader.hpp:94
Definition: file.hpp:72
Namespace for everything in the Osmium library.
Definition: assembler.hpp:53
void set_option(osmium::osm_entity_bits::type value) noexcept
Definition: reader.hpp:132
std::size_t m_file_size
Definition: reader.hpp:123
Definition: attr.hpp:333
osmium::io::detail::ReadThreadManager m_read_thread_manager
Definition: reader.hpp:113
static int execute(const std::string &command, const std::string &filename, int *childpid)
Definition: reader.hpp:172
static void parser_thread(osmium::thread::Pool &pool, const detail::ParserFactory::create_parser_type &creator, detail::future_string_queue_type &input_queue, detail::future_buffer_queue_type &osmdata_queue, std::promise< osmium::io::Header > &&header_promise, osmium::osm_entity_bits::type read_which_entities, osmium::io::read_meta read_metadata)
Definition: reader.hpp:141
Reader(const std::string &filename, TArgs &&... args)
Definition: reader.hpp:285
Definition: reader.hpp:92
~Reader() noexcept
Definition: reader.hpp:300
Definition: error.hpp:44
std::size_t offset() const noexcept
Definition: reader.hpp:441
Definition: pool.hpp:89
osmium::io::Header header()
Definition: reader.hpp:350
void set_option(osmium::io::read_meta value) noexcept
Definition: reader.hpp:136
void set_option(osmium::thread::Pool &pool) noexcept
Definition: reader.hpp:128
Definition: buffer.hpp:98
osmium::thread::thread_handler m_thread
Definition: reader.hpp:121
static int open_input_file_or_url(const std::string &filename, int *childpid)
Definition: reader.hpp:217
detail::future_buffer_queue_type m_osmdata_queue
Definition: reader.hpp:115
osmium::io::Header m_header
Definition: reader.hpp:119
read_meta
Definition: file_format.hpp:52
Definition: entity_bits.hpp:67
Definition: compression.hpp:135
std::string get(const std::string &key, const std::string &default_value="") const noexcept
Definition: options.hpp:124
detail::ParserFactory::create_parser_type m_creator
Definition: reader.hpp:98
Definition: header.hpp:68
void close()
Definition: reader.hpp:316
Definition: util.hpp:85