Libosmium  2.3.0
Fast and flexible C++ library for working with OpenStreetMap data
compression.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_IO_COMPRESSION_HPP
2 #define OSMIUM_IO_COMPRESSION_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <cerrno>
37 #include <functional>
38 #include <map>
39 #include <memory>
40 #include <stdexcept>
41 #include <string>
42 #include <system_error>
43 #include <utility>
44 
45 #ifndef _MSC_VER
46 # include <unistd.h>
47 #else
48 # include <io.h>
49 #endif
50 
51 #include <osmium/io/detail/read_write.hpp>
54 
55 namespace osmium {
56 
57  namespace io {
58 
59  class Compressor {
60 
61  public:
62 
63  Compressor() = default;
64 
65  virtual ~Compressor() {
66  }
67 
68  virtual void write(const std::string& data) = 0;
69 
70  virtual void close() = 0;
71 
72  }; // class Compressor
73 
74  class Decompressor {
75 
76  public:
77 
78  static constexpr unsigned int input_buffer_size = 1024 * 1024;
79 
80  Decompressor() = default;
81 
82  Decompressor(const Decompressor&) = delete;
83  Decompressor& operator=(const Decompressor&) = delete;
84 
85  Decompressor(Decompressor&&) = delete;
86  Decompressor& operator=(Decompressor&&) = delete;
87 
88  virtual ~Decompressor() {
89  }
90 
91  virtual std::string read() = 0;
92 
93  virtual void close() {
94  }
95 
96  }; // class Decompressor
97 
106 
107  public:
108 
109  typedef std::function<osmium::io::Compressor*(int)> create_compressor_type;
110  typedef std::function<osmium::io::Decompressor*(int)> create_decompressor_type_fd;
111  typedef std::function<osmium::io::Decompressor*(const char*, size_t)> create_decompressor_type_buffer;
112 
113  private:
114 
115  typedef std::map<const osmium::io::file_compression, std::tuple<create_compressor_type, create_decompressor_type_fd, create_decompressor_type_buffer>> compression_map_type;
116 
117  compression_map_type m_callbacks;
118 
119  CompressionFactory() = default;
120 
121  CompressionFactory(const CompressionFactory&) = delete;
123 
126 
128  std::string error_message {"Support for compression '"};
129  error_message += as_string(compression);
130  error_message += "' not compiled into this binary.";
131  throw std::runtime_error(error_message);
132  }
133 
134  public:
135 
137  static CompressionFactory factory;
138  return factory;
139  }
140 
142  osmium::io::file_compression compression,
143  create_compressor_type create_compressor,
144  create_decompressor_type_fd create_decompressor_fd,
145  create_decompressor_type_buffer create_decompressor_buffer) {
146 
147  compression_map_type::value_type cc(compression, std::make_tuple(create_compressor, create_decompressor_fd, create_decompressor_buffer));
148  return m_callbacks.insert(cc).second;
149  }
150 
151  std::unique_ptr<osmium::io::Compressor> create_compressor(osmium::io::file_compression compression, int fd) {
152  auto it = m_callbacks.find(compression);
153 
154  if (it != m_callbacks.end()) {
155  return std::unique_ptr<osmium::io::Compressor>(std::get<0>(it->second)(fd));
156  }
157 
158  error(compression);
159  }
160 
161  std::unique_ptr<osmium::io::Decompressor> create_decompressor(osmium::io::file_compression compression, int fd) {
162  auto it = m_callbacks.find(compression);
163 
164  if (it != m_callbacks.end()) {
165  return std::unique_ptr<osmium::io::Decompressor>(std::get<1>(it->second)(fd));
166  }
167 
168  error(compression);
169  }
170 
171  std::unique_ptr<osmium::io::Decompressor> create_decompressor(osmium::io::file_compression compression, const char* buffer, size_t size) {
172  auto it = m_callbacks.find(compression);
173 
174  if (it != m_callbacks.end()) {
175  return std::unique_ptr<osmium::io::Decompressor>(std::get<2>(it->second)(buffer, size));
176  }
177 
178  error(compression);
179  }
180 
181  }; // class CompressionFactory
182 
183  class NoCompressor : public Compressor {
184 
185  int m_fd;
186 
187  public:
188 
189  NoCompressor(int fd) :
190  Compressor(),
191  m_fd(fd) {
192  }
193 
194  ~NoCompressor() override final {
195  close();
196  }
197 
198  void write(const std::string& data) override final {
199  osmium::io::detail::reliable_write(m_fd, data.data(), data.size());
200  }
201 
202  void close() override final {
203  if (m_fd >= 0) {
204  ::close(m_fd);
205  m_fd = -1;
206  }
207  }
208 
209  }; // class NoCompressor
210 
211  class NoDecompressor : public Decompressor {
212 
213  int m_fd;
214  const char *m_buffer;
216 
217  public:
218 
219  NoDecompressor(int fd) :
220  Decompressor(),
221  m_fd(fd),
222  m_buffer(nullptr),
223  m_buffer_size(0) {
224  }
225 
226  NoDecompressor(const char* buffer, size_t size) :
227  Decompressor(),
228  m_fd(-1),
229  m_buffer(buffer),
230  m_buffer_size(size) {
231  }
232 
233  ~NoDecompressor() override final {
234  close();
235  }
236 
237  std::string read() override final {
238  std::string buffer;
239 
240  if (m_buffer) {
241  if (m_buffer_size != 0) {
242  size_t size = m_buffer_size;
243  m_buffer_size = 0;
244  buffer.append(m_buffer, size);
245  }
246  } else {
248  auto nread = ::read(m_fd, const_cast<char*>(buffer.data()), osmium::io::Decompressor::input_buffer_size);
249  if (nread < 0) {
250  throw std::system_error(errno, std::system_category(), "Read failed");
251  }
252  buffer.resize(nread);
253  }
254 
255  return buffer;
256  }
257 
258  void close() override final {
259  if (m_fd >= 0) {
260  ::close(m_fd);
261  m_fd = -1;
262  }
263  }
264 
265  }; // class NoDecompressor
266 
267  namespace {
268 
269 // we want the register_compression() function to run, setting the variable
270 // is only a side-effect, it will never be used
271 #pragma GCC diagnostic push
272 #pragma GCC diagnostic ignored "-Wunused-variable"
274  [](int fd) { return new osmium::io::NoCompressor(fd); },
275  [](int fd) { return new osmium::io::NoDecompressor(fd); },
276  [](const char* buffer, size_t size) { return new osmium::io::NoDecompressor(buffer, size); }
277  );
278 #pragma GCC diagnostic pop
279 
280  } // anonymous namespace
281 
282  } // namespace io
283 
284 } // namespace osmium
285 
286 #endif // OSMIUM_IO_COMPRESSION_HPP
OSMIUM_NORETURN void error(osmium::io::file_compression compression)
Definition: compression.hpp:127
#define OSMIUM_NORETURN
Definition: compatibility.hpp:44
NoDecompressor(int fd)
Definition: compression.hpp:219
virtual std::string read()=0
Decompressor & operator=(const Decompressor &)=delete
static CompressionFactory & instance()
Definition: compression.hpp:136
virtual ~Decompressor()
Definition: compression.hpp:88
static constexpr unsigned int input_buffer_size
Definition: compression.hpp:78
void close() overridefinal
Definition: compression.hpp:202
Definition: compression.hpp:74
~NoDecompressor() overridefinal
Definition: compression.hpp:233
virtual void write(const std::string &data)=0
void write(const std::string &data) overridefinal
Definition: compression.hpp:198
int m_fd
Definition: compression.hpp:213
const char * as_string(file_compression compression)
Definition: file_compression.hpp:51
size_t m_buffer_size
Definition: compression.hpp:215
Namespace for everything in the Osmium library.
Definition: assembler.hpp:55
~NoCompressor() overridefinal
Definition: compression.hpp:194
std::unique_ptr< osmium::io::Decompressor > create_decompressor(osmium::io::file_compression compression, int fd)
Definition: compression.hpp:161
std::string read() overridefinal
Definition: compression.hpp:237
std::unique_ptr< osmium::io::Decompressor > create_decompressor(osmium::io::file_compression compression, const char *buffer, size_t size)
Definition: compression.hpp:171
virtual void close()=0
std::function< osmium::io::Compressor *(int)> create_compressor_type
Definition: compression.hpp:109
std::map< const osmium::io::file_compression, std::tuple< create_compressor_type, create_decompressor_type_fd, create_decompressor_type_buffer > > compression_map_type
Definition: compression.hpp:115
NoCompressor(int fd)
Definition: compression.hpp:189
virtual ~Compressor()
Definition: compression.hpp:65
std::function< osmium::io::Decompressor *(int)> create_decompressor_type_fd
Definition: compression.hpp:110
file_compression
Definition: file_compression.hpp:42
std::function< osmium::io::Decompressor *(const char *, size_t)> create_decompressor_type_buffer
Definition: compression.hpp:111
Definition: compression.hpp:183
void close() overridefinal
Definition: compression.hpp:258
Definition: compression.hpp:59
int m_fd
Definition: compression.hpp:185
compression_map_type m_callbacks
Definition: compression.hpp:117
CompressionFactory & operator=(const CompressionFactory &)=delete
Definition: compression.hpp:211
virtual void close()
Definition: compression.hpp:93
Definition: compression.hpp:105
NoDecompressor(const char *buffer, size_t size)
Definition: compression.hpp:226
bool register_compression(osmium::io::file_compression compression, create_compressor_type create_compressor, create_decompressor_type_fd create_decompressor_fd, create_decompressor_type_buffer create_decompressor_buffer)
Definition: compression.hpp:141
const char * m_buffer
Definition: compression.hpp:214
std::unique_ptr< osmium::io::Compressor > create_compressor(osmium::io::file_compression compression, int fd)
Definition: compression.hpp:151