Libosmium  2.1.0
Fast and flexible C++ library for working with OpenStreetMap data
reader.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_READER_HPP
2 #define OSMIUM_IO_READER_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <atomic>
37 #include <cerrno>
38 #include <cstdlib>
39 #include <fcntl.h>
40 #include <memory>
41 #include <string>
42 #include <system_error>
43 #include <thread>
44 #include <utility>
45 
46 #ifndef _WIN32
47 # include <sys/wait.h>
48 #endif
49 
50 #ifndef _MSC_VER
51 # include <unistd.h>
52 #endif
53 
55 #include <osmium/io/detail/input_format.hpp>
56 #include <osmium/io/detail/read_thread.hpp>
57 #include <osmium/io/detail/read_write.hpp>
58 #include <osmium/io/file.hpp>
59 #include <osmium/io/header.hpp>
60 #include <osmium/memory/buffer.hpp>
62 #include <osmium/thread/util.hpp>
63 #include <osmium/thread/queue.hpp>
64 
65 namespace osmium {
66 
67  namespace io {
68 
75  class Reader {
76 
79  std::atomic<bool> m_input_done;
81 
83 
84  std::unique_ptr<osmium::io::Decompressor> m_decompressor;
85  std::future<bool> m_read_future;
86 
87  std::unique_ptr<osmium::io::detail::InputFormat> m_input;
88 
89 #ifndef _WIN32
90 
101  static int execute(const std::string& command, const std::string& filename, int* childpid) {
102  int pipefd[2];
103  if (pipe(pipefd) < 0) {
104  throw std::system_error(errno, std::system_category(), "opening pipe failed");
105  }
106  pid_t pid = fork();
107  if (pid < 0) {
108  throw std::system_error(errno, std::system_category(), "fork failed");
109  }
110  if (pid == 0) { // child
111  // close all file descriptors except one end of the pipe
112  for (int i=0; i < 32; ++i) {
113  if (i != pipefd[1]) {
114  ::close(i);
115  }
116  }
117  if (dup2(pipefd[1], 1) < 0) { // put end of pipe as stdout/stdin
118  exit(1);
119  }
120 
121  ::open("/dev/null", O_RDONLY); // stdin
122  ::open("/dev/null", O_WRONLY); // stderr
123  // hack: -g switches off globbing in curl which allows [] to be used in file names
124  // this is important for XAPI URLs
125  // in theory this execute() function could be used for other commands, but it is
126  // only used for curl at the moment, so this is okay.
127  if (::execlp(command.c_str(), command.c_str(), "-g", filename.c_str(), nullptr) < 0) {
128  exit(1);
129  }
130  }
131  // parent
132  *childpid = pid;
133  ::close(pipefd[1]);
134  return pipefd[0];
135  }
136 #endif
137 
146  static int open_input_file_or_url(const std::string& filename, int* childpid) {
147  std::string protocol = filename.substr(0, filename.find_first_of(':'));
148  if (protocol == "http" || protocol == "https" || protocol == "ftp" || protocol == "file") {
149 #ifndef _WIN32
150  return execute("curl", filename, childpid);
151 #else
152  throw std::runtime_error("Reading OSM files from the network currently not supported on Windows.");
153 #endif
154  } else {
155  return osmium::io::detail::open_for_reading(filename);
156  }
157  }
158 
159  public:
160 
171  m_file(file),
172  m_read_which_entities(read_which_entities),
173  m_input_done(false),
174  m_childpid(0),
175  m_input_queue(20, "raw_input"), // XXX
176  m_decompressor(m_file.buffer() ?
177  osmium::io::CompressionFactory::instance().create_decompressor(file.compression(), m_file.buffer(), m_file.buffer_size()) :
178  osmium::io::CompressionFactory::instance().create_decompressor(file.compression(), open_input_file_or_url(m_file.filename(), &m_childpid))),
179  m_read_future(std::async(std::launch::async, detail::ReadThread(m_input_queue, m_decompressor.get(), m_input_done))),
180  m_input(osmium::io::detail::InputFormatFactory::instance().create_input(m_file, m_read_which_entities, m_input_queue)) {
181  }
182 
183  explicit Reader(const std::string& filename, osmium::osm_entity_bits::type read_types = osmium::osm_entity_bits::all) :
184  Reader(osmium::io::File(filename), read_types) {
185  }
186 
187  explicit Reader(const char* filename, osmium::osm_entity_bits::type read_types = osmium::osm_entity_bits::all) :
188  Reader(osmium::io::File(filename), read_types) {
189  }
190 
191  Reader(const Reader&) = delete;
192  Reader& operator=(const Reader&) = delete;
193 
195  try {
196  close();
197  }
198  catch (...) {
199  }
200  }
201 
210  void close() {
211  // Signal to input child process that it should wrap up.
212  m_input_done = true;
213 
214  m_input->close();
215 
216 #ifndef _WIN32
217  if (m_childpid) {
218  int status;
219  pid_t pid = ::waitpid(m_childpid, &status, 0);
220 #pragma GCC diagnostic push
221 #pragma GCC diagnostic ignored "-Wold-style-cast"
222  if (pid < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) {
223  throw std::system_error(errno, std::system_category(), "subprocess returned error");
224  }
225 #pragma GCC diagnostic pop
226  m_childpid = 0;
227  }
228 #endif
229 
230  osmium::thread::wait_until_done(m_read_future);
231  }
232 
237  return m_input->header();
238  }
239 
251  // If an exception happened in the input thread, re-throw
252  // it in this (the main) thread.
254 
255  if (m_read_which_entities == osmium::osm_entity_bits::nothing || m_input_done) {
256  // If the caller didn't want anything but the header, it will
257  // always get an empty buffer here.
258  return osmium::memory::Buffer();
259  }
260 
261  // m_input->read() can return an invalid buffer to signal EOF,
262  // or a valid buffer with or without data. A valid buffer
263  // without data is not an error, it just means we have to
264  // keep getting the next buffer until there is one with data.
265  while (true) {
266  osmium::memory::Buffer buffer = m_input->read();
267  if (!buffer) {
268  m_input_done = true;
269  return buffer;
270  }
271  if (buffer.committed() > 0) {
272  return buffer;
273  }
274  }
275  }
276 
281  bool eof() const {
282  return m_input_done;
283  }
284 
285  }; // class Reader
286 
295  template <class... TArgs>
298 
299  Reader reader(std::forward<TArgs>(args)...);
300  while (osmium::memory::Buffer read_buffer = reader.read()) {
301  buffer.add_buffer(read_buffer);
302  buffer.commit();
303  }
304 
305  return buffer;
306  }
307 
308  } // namespace io
309 
310 } // namespace osmium
311 
312 #endif // OSMIUM_IO_READER_HPP
osmium::memory::Buffer read()
Definition: reader.hpp:250
Reader(const osmium::io::File &file, osmium::osm_entity_bits::type read_which_entities=osmium::osm_entity_bits::all)
Definition: reader.hpp:170
type
Definition: entity_bits.hpp:60
int m_childpid
Definition: reader.hpp:80
osmium::memory::Buffer read_file(TArgs &&...args)
Definition: reader.hpp:296
Definition: reader_iterator.hpp:39
std::unique_ptr< osmium::io::Decompressor > m_decompressor
Definition: reader.hpp:84
Reader & operator=(const Reader &)=delete
bool eof() const
Definition: reader.hpp:281
object or changeset
Definition: entity_bits.hpp:71
osmium::io::File m_file
Definition: reader.hpp:77
Definition: file.hpp:73
Namespace for everything in the Osmium library.
Definition: assembler.hpp:55
void add_buffer(const Buffer &buffer)
Definition: buffer.hpp:380
static int execute(const std::string &command, const std::string &filename, int *childpid)
Definition: reader.hpp:101
Definition: reader.hpp:75
osmium::io::Header header() const
Definition: reader.hpp:236
std::future< bool > m_read_future
Definition: reader.hpp:85
size_t committed() const noexcept
Definition: buffer.hpp:218
Definition: buffer.hpp:95
osmium::thread::Queue< std::string > m_input_queue
Definition: reader.hpp:82
static int open_input_file_or_url(const std::string &filename, int *childpid)
Definition: reader.hpp:146
Definition: entity_bits.hpp:62
~Reader()
Definition: reader.hpp:194
std::unique_ptr< osmium::io::detail::InputFormat > m_input
Definition: reader.hpp:87
Reader(const char *filename, osmium::osm_entity_bits::type read_types=osmium::osm_entity_bits::all)
Definition: reader.hpp:187
void check_for_exception(std::future< T > &future)
Definition: util.hpp:53
Definition: compression.hpp:105
Definition: header.hpp:49
std::atomic< bool > m_input_done
Definition: reader.hpp:79
void close()
Definition: reader.hpp:210
osmium::osm_entity_bits::type m_read_which_entities
Definition: reader.hpp:78
void wait_until_done(std::future< T > &future)
Definition: util.hpp:64
size_t commit()
Definition: buffer.hpp:273
Reader(const std::string &filename, osmium::osm_entity_bits::type read_types=osmium::osm_entity_bits::all)
Definition: reader.hpp:183