Libosmium  2.6.0
Fast and flexible C++ library for working with OpenStreetMap data
reader.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_READER_HPP
2 #define OSMIUM_IO_READER_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <cerrno>
37 #include <cstdlib>
38 #include <fcntl.h>
39 #include <future>
40 #include <memory>
41 #include <string>
42 #include <system_error>
43 #include <thread>
44 #include <utility>
45 
46 #ifndef _WIN32
47 # include <sys/wait.h>
48 #endif
49 
50 #ifndef _MSC_VER
51 # include <unistd.h>
52 #endif
53 
55 #include <osmium/io/detail/input_format.hpp>
56 #include <osmium/io/detail/read_thread.hpp>
57 #include <osmium/io/detail/read_write.hpp>
58 #include <osmium/io/detail/queue_util.hpp>
59 #include <osmium/io/error.hpp>
60 #include <osmium/io/file.hpp>
61 #include <osmium/io/header.hpp>
62 #include <osmium/memory/buffer.hpp>
64 #include <osmium/thread/util.hpp>
65 
66 namespace osmium {
67 
68  namespace io {
69 
76  class Reader {
77 
78  static constexpr size_t max_input_queue_size = 20; // XXX
79  static constexpr size_t max_osmdata_queue_size = 20; // XXX
80 
83 
84  enum class status {
85  okay = 0, // normal reading
86  error = 1, // some error occurred while reading
87  closed = 2, // close() called successfully after eof
88  eof = 3 // eof of file was reached without error
89  } m_status;
90 
92 
93  detail::future_string_queue_type m_input_queue;
94 
95  std::unique_ptr<osmium::io::Decompressor> m_decompressor;
96 
97  osmium::io::detail::ReadThreadManager m_read_thread_manager;
98 
99  detail::future_buffer_queue_type m_osmdata_queue;
100  detail::queue_wrapper<osmium::memory::Buffer> m_osmdata_queue_wrapper;
101 
102  std::future<osmium::io::Header> m_header_future;
104 
106 
107  // This function will run in a separate thread.
108  static void parser_thread(const osmium::io::File& file,
109  detail::future_string_queue_type& input_queue,
110  detail::future_buffer_queue_type& osmdata_queue,
111  std::promise<osmium::io::Header>&& header_promise,
112  osmium::osm_entity_bits::type read_which_entities) {
113  std::promise<osmium::io::Header> promise = std::move(header_promise);
114  auto creator = detail::ParserFactory::instance().get_creator_function(file);
115  auto parser = creator(input_queue, osmdata_queue, promise, read_which_entities);
116  parser->parse();
117  }
118 
119 #ifndef _WIN32
120 
131  static int execute(const std::string& command, const std::string& filename, int* childpid) {
132  int pipefd[2];
133  if (pipe(pipefd) < 0) {
134  throw std::system_error(errno, std::system_category(), "opening pipe failed");
135  }
136  pid_t pid = fork();
137  if (pid < 0) {
138  throw std::system_error(errno, std::system_category(), "fork failed");
139  }
140  if (pid == 0) { // child
141  // close all file descriptors except one end of the pipe
142  for (int i = 0; i < 32; ++i) {
143  if (i != pipefd[1]) {
144  ::close(i);
145  }
146  }
147  if (dup2(pipefd[1], 1) < 0) { // put end of pipe as stdout/stdin
148  exit(1);
149  }
150 
151  ::open("/dev/null", O_RDONLY); // stdin
152  ::open("/dev/null", O_WRONLY); // stderr
153  // hack: -g switches off globbing in curl which allows [] to be used in file names
154  // this is important for XAPI URLs
155  // in theory this execute() function could be used for other commands, but it is
156  // only used for curl at the moment, so this is okay.
157  if (::execlp(command.c_str(), command.c_str(), "-g", filename.c_str(), nullptr) < 0) {
158  exit(1);
159  }
160  }
161  // parent
162  *childpid = pid;
163  ::close(pipefd[1]);
164  return pipefd[0];
165  }
166 #endif
167 
176  static int open_input_file_or_url(const std::string& filename, int* childpid) {
177  std::string protocol = filename.substr(0, filename.find_first_of(':'));
178  if (protocol == "http" || protocol == "https" || protocol == "ftp" || protocol == "file") {
179 #ifndef _WIN32
180  return execute("curl", filename, childpid);
181 #else
182  throw io_error("Reading OSM files from the network currently not supported on Windows.");
183 #endif
184  } else {
185  return osmium::io::detail::open_for_reading(filename);
186  }
187  }
188 
189  public:
190 
201  m_file(file.check()),
202  m_read_which_entities(read_which_entities),
203  m_status(status::okay),
204  m_childpid(0),
205  m_input_queue(max_input_queue_size, "raw_input"),
206  m_decompressor(m_file.buffer() ?
207  osmium::io::CompressionFactory::instance().create_decompressor(file.compression(), m_file.buffer(), m_file.buffer_size()) :
208  osmium::io::CompressionFactory::instance().create_decompressor(file.compression(), open_input_file_or_url(m_file.filename(), &m_childpid))),
209  m_read_thread_manager(*m_decompressor, m_input_queue),
210  m_osmdata_queue(max_osmdata_queue_size, "parser_results"),
211  m_osmdata_queue_wrapper(m_osmdata_queue),
212  m_header_future(),
213  m_header(),
214  m_thread() {
215  std::promise<osmium::io::Header> header_promise;
216  m_header_future = header_promise.get_future();
217  m_thread = osmium::thread::thread_handler{parser_thread, std::ref(m_file), std::ref(m_input_queue), std::ref(m_osmdata_queue), std::move(header_promise), read_which_entities};
218  }
219 
220  explicit Reader(const std::string& filename, osmium::osm_entity_bits::type read_types = osmium::osm_entity_bits::all) :
221  Reader(osmium::io::File(filename), read_types) {
222  }
223 
224  explicit Reader(const char* filename, osmium::osm_entity_bits::type read_types = osmium::osm_entity_bits::all) :
225  Reader(osmium::io::File(filename), read_types) {
226  }
227 
228  Reader(const Reader&) = delete;
229  Reader& operator=(const Reader&) = delete;
230 
231  Reader(Reader&&) = default;
232  Reader& operator=(Reader&&) = default;
233 
234  ~Reader() noexcept {
235  try {
236  close();
237  } catch (...) {
238  // Ignore any exceptions because destructor must not throw.
239  }
240  }
241 
250  void close() {
252 
253  m_read_thread_manager.stop();
254 
255  m_osmdata_queue_wrapper.drain();
256 
257  try {
258  m_read_thread_manager.close();
259  } catch (...) {
260  // Ignore any exceptions.
261  }
262 
263 #ifndef _WIN32
264  if (m_childpid) {
265  int status;
266  pid_t pid = ::waitpid(m_childpid, &status, 0);
267 #pragma GCC diagnostic push
268 #pragma GCC diagnostic ignored "-Wold-style-cast"
269  if (pid < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) {
270  throw std::system_error(errno, std::system_category(), "subprocess returned error");
271  }
272 #pragma GCC diagnostic pop
273  m_childpid = 0;
274  }
275 #endif
276  }
277 
285  if (m_status == status::error) {
286  throw io_error("Can not get header from reader when in status 'error'");
287  }
288 
289  try {
290  if (m_header_future.valid()) {
291  m_header = m_header_future.get();
292  if (m_read_which_entities == osmium::osm_entity_bits::nothing) {
294  }
295  }
296  } catch (...) {
297  close();
299  throw;
300  }
301  return m_header;
302  }
303 
315  osmium::memory::Buffer buffer;
316 
317  if (m_status != status::okay ||
318  m_read_which_entities == osmium::osm_entity_bits::nothing) {
319  throw io_error("Can not read from reader when in status 'closed', 'eof', or 'error'");
320  }
321 
322  try {
323  // m_input_format.read() can return an invalid buffer to signal EOF,
324  // or a valid buffer with or without data. A valid buffer
325  // without data is not an error, it just means we have to
326  // keep getting the next buffer until there is one with data.
327  while (true) {
328  buffer = m_osmdata_queue_wrapper.pop();
329  if (detail::at_end_of_data(buffer)) {
331  m_read_thread_manager.close();
332  return buffer;
333  }
334  if (buffer.committed() > 0) {
335  return buffer;
336  }
337  }
338  } catch (...) {
339  close();
341  throw;
342  }
343  }
344 
349  bool eof() const {
351  }
352 
353  }; // class Reader
354 
363  template <typename... TArgs>
366 
367  Reader reader(std::forward<TArgs>(args)...);
368  while (osmium::memory::Buffer read_buffer = reader.read()) {
369  buffer.add_buffer(read_buffer);
370  buffer.commit();
371  }
372 
373  return buffer;
374  }
375 
376  } // namespace io
377 
378 } // namespace osmium
379 
380 #endif // OSMIUM_IO_READER_HPP
detail::queue_wrapper< osmium::memory::Buffer > m_osmdata_queue_wrapper
Definition: reader.hpp:100
status
Definition: reader.hpp:84
osmium::memory::Buffer read()
Definition: reader.hpp:314
Reader(const osmium::io::File &file, osmium::osm_entity_bits::type read_which_entities=osmium::osm_entity_bits::all)
Definition: reader.hpp:200
type
Definition: entity_bits.hpp:60
int m_childpid
Definition: reader.hpp:91
std::future< osmium::io::Header > m_header_future
Definition: reader.hpp:102
osmium::memory::Buffer read_file(TArgs &&...args)
Definition: reader.hpp:364
enum osmium::io::Reader::status m_status
std::unique_ptr< osmium::io::Decompressor > m_decompressor
Definition: reader.hpp:95
Reader & operator=(const Reader &)=delete
static constexpr size_t max_osmdata_queue_size
Definition: reader.hpp:79
bool eof() const
Definition: reader.hpp:349
object or changeset
Definition: entity_bits.hpp:71
detail::future_string_queue_type m_input_queue
Definition: reader.hpp:93
osmium::io::File m_file
Definition: reader.hpp:81
Definition: file.hpp:74
Namespace for everything in the Osmium library.
Definition: assembler.hpp:59
void add_buffer(const Buffer &buffer)
Definition: buffer.hpp:474
osmium::io::detail::ReadThreadManager m_read_thread_manager
Definition: reader.hpp:97
static int execute(const std::string &command, const std::string &filename, int *childpid)
Definition: reader.hpp:131
Definition: reader.hpp:76
~Reader() noexcept
Definition: reader.hpp:234
Definition: error.hpp:44
static void parser_thread(const osmium::io::File &file, detail::future_string_queue_type &input_queue, detail::future_buffer_queue_type &osmdata_queue, std::promise< osmium::io::Header > &&header_promise, osmium::osm_entity_bits::type read_which_entities)
Definition: reader.hpp:108
osmium::io::Header header()
Definition: reader.hpp:284
size_t committed() const noexcept
Definition: buffer.hpp:241
Definition: buffer.hpp:97
osmium::thread::thread_handler m_thread
Definition: reader.hpp:105
static int open_input_file_or_url(const std::string &filename, int *childpid)
Definition: reader.hpp:176
detail::future_buffer_queue_type m_osmdata_queue
Definition: reader.hpp:99
osmium::io::Header m_header
Definition: reader.hpp:103
Definition: entity_bits.hpp:62
Reader(const char *filename, osmium::osm_entity_bits::type read_types=osmium::osm_entity_bits::all)
Definition: reader.hpp:224
Definition: compression.hpp:117
std::string get(const std::string &key, const std::string &default_value="") const noexcept
Definition: options.hpp:124
static constexpr size_t max_input_queue_size
Definition: reader.hpp:78
Definition: header.hpp:49
void close()
Definition: reader.hpp:250
osmium::osm_entity_bits::type m_read_which_entities
Definition: reader.hpp:82
size_t commit()
Definition: buffer.hpp:335
Definition: util.hpp:83
Reader(const std::string &filename, osmium::osm_entity_bits::type read_types=osmium::osm_entity_bits::all)
Definition: reader.hpp:220