Libosmium  2.3.0
Fast and flexible C++ library for working with OpenStreetMap data
memory_mapping.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_UTIL_MEMORY_MAPPING_HPP
2 #define OSMIUM_UTIL_MEMORY_MAPPING_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2015 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <cassert>
37 #include <cerrno>
38 #include <stdexcept>
39 #include <system_error>
40 
41 #include <osmium/util/file.hpp>
42 
43 #ifndef _WIN32
44 # include <sys/mman.h>
45 #else
46 # include <io.h>
47 # include <windows.h>
48 # include <sys/types.h>
49 #endif
50 
51 namespace osmium {
52 
53  namespace util {
54 
89  class MemoryMapping {
90 
91 public:
92  enum class mapping_mode {
93  readonly = 0,
94  write_private = 1,
95  write_shared = 2
96  };
97 
98 private:
99 
101  size_t m_size;
102 
104  off_t m_offset;
105 
107  int m_fd;
108 
111 
112 #ifdef _WIN32
113  HANDLE m_handle;
114 #endif
115 
117  void* m_addr;
118 
119  bool is_valid() const noexcept;
120 
121  void make_invalid() noexcept;
122 
123 #ifdef _WIN32
124  typedef DWORD flag_type;
125 #else
126  typedef int flag_type;
127 #endif
128 
129  flag_type get_protection() const noexcept;
130 
131  flag_type get_flags() const noexcept;
132 
133  // A zero-sized mapping is not allowed by the operating system.
134  // So if the user asks for a mapping of size 0, we map a full
135  // page instead. This way we don't have a special case in the rest
136  // of the code.
137  static size_t initial_size(size_t size) {
138  if (size == 0) {
140  }
141  return size;
142  }
143 
144 #ifdef _WIN32
145  HANDLE get_handle() const noexcept;
146  HANDLE osmium::util::MemoryMapping::create_file_mapping() const noexcept;
147  void* osmium::util::MemoryMapping::map_view_of_file() const noexcept;
148 #endif
149 
150  int resize_fd(int fd) {
151  // Anonymous mapping doesn't need resizing.
152  if (fd == -1) {
153  return -1;
154  }
155 
156  // Make sure the file backing this mapping is large enough.
157  if (osmium::util::file_size(fd) < m_size + m_offset) {
158  osmium::util::resize_file(fd, m_size + m_offset);
159  }
160  return fd;
161  }
162 
163  public:
164 
180  MemoryMapping(size_t size, mapping_mode mode, int fd=-1, off_t offset=0);
181 
183  MemoryMapping(size_t size, bool writable=true, int fd=-1, off_t offset=0) :
184  MemoryMapping(size, writable ? mapping_mode::write_shared : mapping_mode::readonly, fd, offset) {
185  }
186 
188  MemoryMapping(const MemoryMapping&) = delete;
189 
191  MemoryMapping& operator=(const MemoryMapping&) = delete;
192 
197  MemoryMapping(MemoryMapping&& other);
198 
203 
208  ~MemoryMapping() noexcept {
209  try {
210  unmap();
211  } catch (std::system_error&) {
212  // ignore
213  }
214  }
215 
222  void unmap();
223 
234  void resize(size_t new_size);
235 
240  operator bool() const noexcept {
241  return is_valid();
242  }
243 
249  size_t size() const noexcept {
250  return m_size;
251  }
252 
258  int fd() const noexcept {
259  return m_fd;
260  }
261 
265  bool writable() const noexcept {
266  return m_mapping_mode != mapping_mode::readonly;
267  }
268 
274  template <typename T = void>
275  T* get_addr() const {
276  if (is_valid()) {
277  return reinterpret_cast<T*>(m_addr);
278  }
279  throw std::runtime_error("invalid memory mapping");
280  }
281 
282  }; // class MemoryMapping
283 
295 
296  public:
297 
299  MemoryMapping(size, mapping_mode::write_private) {
300  }
301 
302 #ifndef __linux__
303 
307  void resize(size_t) = delete;
308 #endif
309 
310  }; // class AnonymousMemoryMapping
311 
321  template <typename T>
323 
325 
326  public:
327 
335  m_mapping(sizeof(T) * size, MemoryMapping::mapping_mode::write_private) {
336  }
337 
348  TypedMemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset = 0) :
349  m_mapping(sizeof(T) * size, mode, fd, sizeof(T) * offset) {
350  }
351 
353  TypedMemoryMapping(size_t size, bool writable, int fd, off_t offset = 0) :
354  m_mapping(sizeof(T) * size, writable ? MemoryMapping::mapping_mode::write_shared : MemoryMapping::mapping_mode::readonly, fd, sizeof(T) * offset) {
355  }
356 
358  TypedMemoryMapping(const TypedMemoryMapping&) = delete;
359 
362 
367  TypedMemoryMapping(TypedMemoryMapping&& other) = default;
368 
372  TypedMemoryMapping& operator=(TypedMemoryMapping&& other) = default;
373 
378  ~TypedMemoryMapping() = default;
379 
386  void unmap() {
387  m_mapping.unmap();
388  }
389 
400  void resize(size_t new_size) {
401  m_mapping.resize(sizeof(T) * new_size);
402  }
403 
408  operator bool() const noexcept {
409  return !!m_mapping;
410  }
411 
417  size_t size() const noexcept {
418  assert(m_mapping.size() % sizeof(T) == 0);
419  return m_mapping.size() / sizeof(T);
420  }
421 
427  int fd() const noexcept {
428  return m_mapping.fd();
429  }
430 
434  bool writable() const noexcept {
435  return m_mapping.writable();
436  }
437 
443  T* begin() {
444  return m_mapping.get_addr<T>();
445  }
446 
452  T* end() {
453  return m_mapping.get_addr<T>() + size();
454  }
455 
456  const T* cbegin() const {
457  return m_mapping.get_addr<T>();
458  }
459 
460  const T* cend() const {
461  return m_mapping.get_addr<T>() + size();
462  }
463 
464  const T* begin() const {
465  return m_mapping.get_addr<T>();
466  }
467 
468  const T* end() const {
469  return m_mapping.get_addr<T>() + size();
470  }
471 
472  }; // class TypedMemoryMapping
473 
474  template <typename T>
476 
477  public:
478 
480  TypedMemoryMapping<T>(size) {
481  }
482 
483 #ifndef __linux__
484 
488  void resize(size_t) = delete;
489 #endif
490 
491  }; // class AnonymousTypedMemoryMapping
492 
493  } // namespace util
494 
495 } // namespace osmium
496 
497 #ifndef _WIN32
498 
499 // =========== Unix implementation =============
500 
501 // MAP_FAILED is often a macro containing an old style cast
502 #pragma GCC diagnostic push
503 #pragma GCC diagnostic ignored "-Wold-style-cast"
504 
505 inline bool osmium::util::MemoryMapping::is_valid() const noexcept {
506  return m_addr != MAP_FAILED;
507 }
508 
510  m_addr = MAP_FAILED;
511 }
512 
513 #pragma GCC diagnostic pop
514 
515 // for BSD systems
516 #ifndef MAP_ANONYMOUS
517 # define MAP_ANONYMOUS MAP_ANON
518 #endif
519 
520 inline int osmium::util::MemoryMapping::get_protection() const noexcept {
521  if (m_mapping_mode == mapping_mode::readonly) {
522  return PROT_READ;
523  }
524  return PROT_READ | PROT_WRITE;
525 }
526 
527 inline int osmium::util::MemoryMapping::get_flags() const noexcept {
528  if (m_fd == -1) {
529  return MAP_PRIVATE | MAP_ANONYMOUS;
530  }
531  if (m_mapping_mode == mapping_mode::write_shared) {
532  return MAP_SHARED;
533  }
534  return MAP_PRIVATE;
535 }
536 
537 inline osmium::util::MemoryMapping::MemoryMapping(size_t size, mapping_mode mode, int fd, off_t offset) :
538  m_size(initial_size(size)),
539  m_offset(offset),
540  m_fd(resize_fd(fd)),
541  m_mapping_mode(mode),
542  m_addr(::mmap(nullptr, m_size, get_protection(), get_flags(), m_fd, m_offset)) {
543  assert(!(fd == -1 && mode == mapping_mode::readonly));
544  if (!is_valid()) {
545  throw std::system_error(errno, std::system_category(), "mmap failed");
546  }
547 }
548 
550  m_size(other.m_size),
551  m_offset(other.m_offset),
552  m_fd(other.m_fd),
553  m_mapping_mode(other.m_mapping_mode),
554  m_addr(other.m_addr) {
555  other.make_invalid();
556 }
557 
559  unmap();
560  m_size = other.m_size;
561  m_offset = other.m_offset;
562  m_fd = other.m_fd;
563  m_mapping_mode = other.m_mapping_mode;
564  m_addr = other.m_addr;
565  other.make_invalid();
566  return *this;
567 }
568 
570  if (is_valid()) {
571  if (::munmap(m_addr, m_size) != 0) {
572  throw std::system_error(errno, std::system_category(), "munmap failed");
573  }
574  make_invalid();
575  }
576 }
577 
578 inline void osmium::util::MemoryMapping::resize(size_t new_size) {
579  assert(new_size > 0 && "can not resize to zero size");
580  if (m_fd == -1) { // anonymous mapping
581 #ifdef __linux__
582  m_addr = ::mremap(m_addr, m_size, new_size, MREMAP_MAYMOVE);
583  if (!is_valid()) {
584  throw std::system_error(errno, std::system_category(), "mremap failed");
585  }
586  m_size = new_size;
587 #else
588  assert(false && "can't resize anonymous mappings on non-linux systems");
589 #endif
590  } else { // file-based mapping
591  unmap();
592  m_size = new_size;
593  resize_fd(m_fd);
594  m_addr = ::mmap(nullptr, new_size, get_protection(), get_flags(), m_fd, m_offset);
595  if (!is_valid()) {
596  throw std::system_error(errno, std::system_category(), "mmap (remap) failed");
597  }
598  }
599 }
600 
601 #else
602 
603 // =========== Windows implementation =============
604 
605 /* References:
606  * CreateFileMapping: http://msdn.microsoft.com/en-us/library/aa366537(VS.85).aspx
607  * CloseHandle: http://msdn.microsoft.com/en-us/library/ms724211(VS.85).aspx
608  * MapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366761(VS.85).aspx
609  * UnmapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366882(VS.85).aspx
610  */
611 
612 namespace osmium {
613 
614  namespace util {
615 
616  inline DWORD dword_hi(uint64_t x) {
617  return static_cast<DWORD>(x >> 32);
618  }
619 
620  inline DWORD dword_lo(uint64_t x) {
621  return static_cast<DWORD>(x & 0xffffffff);
622  }
623 
624  } // namespace util
625 
626 } // namespace osmium
627 
628 inline DWORD osmium::util::MemoryMapping::get_protection() const noexcept {
629  switch (m_mapping_mode) {
630  case mapping_mode::readonly:
631  return PAGE_READONLY;
632  case mapping_mode::write_private:
633  return PAGE_WRITECOPY;
634  case mapping_mode::write_shared:
635  return PAGE_READWRITE;
636  }
637 }
638 
639 inline DWORD osmium::util::MemoryMapping::get_flags() const noexcept {
640  switch (m_mapping_mode) {
641  case mapping_mode::readonly:
642  return FILE_MAP_READ;
643  case mapping_mode::write_private:
644  return FILE_MAP_COPY;
645  case mapping_mode::write_shared:
646  return FILE_MAP_WRITE;
647  }
648 }
649 
650 inline HANDLE osmium::util::MemoryMapping::get_handle() const noexcept {
651  if (m_fd == -1) {
652  return INVALID_HANDLE_VALUE;
653  }
654  return reinterpret_cast<HANDLE>(_get_osfhandle(m_fd));
655 }
656 
657 inline HANDLE osmium::util::MemoryMapping::create_file_mapping() const noexcept {
658  return CreateFileMapping(get_handle(), nullptr, get_protection(), osmium::util::dword_hi(static_cast<uint64_t>(m_size) + m_offset), osmium::util::dword_lo(static_cast<uint64_t>(m_size) + m_offset), nullptr);
659 }
660 
661 inline void* osmium::util::MemoryMapping::map_view_of_file() const noexcept {
662  return MapViewOfFile(m_handle, get_flags(), osmium::util::dword_hi(m_offset), osmium::util::dword_lo(m_offset), m_size);
663 }
664 
665 inline bool osmium::util::MemoryMapping::is_valid() const noexcept {
666  return m_addr != nullptr;
667 }
668 
669 inline void osmium::util::MemoryMapping::make_invalid() noexcept {
670  m_addr = nullptr;
671 }
672 
673 inline osmium::util::MemoryMapping::MemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset) :
674  m_size(initial_size(size)),
675  m_offset(offset),
676  m_fd(resize_fd(fd)),
677  m_mapping_mode(mode),
678  m_handle(create_file_mapping()),
679  m_addr(nullptr) {
680 
681  if (!m_handle) {
682  throw std::system_error(GetLastError(), std::system_category(), "CreateFileMapping failed");
683  }
684 
685  m_addr = map_view_of_file();
686  if (!is_valid()) {
687  throw std::system_error(GetLastError(), std::system_category(), "MapViewOfFile failed");
688  }
689 }
690 
691 inline osmium::util::MemoryMapping::MemoryMapping(MemoryMapping&& other) :
692  m_size(other.m_size),
693  m_offset(other.m_offset),
694  m_fd(other.m_fd),
695  m_mapping_mode(other.m_mapping_mode),
696  m_handle(std::move(other.m_handle)),
697  m_addr(other.m_addr) {
698  other.make_invalid();
699  other.m_handle = nullptr;
700 }
701 
703  unmap();
704  m_size = other.m_size;
705  m_offset = other.m_offset;
706  m_fd = other.m_fd;
707  m_mapping_mode = other.m_mapping_mode;
708  m_handle = std::move(other.m_handle);
709  m_addr = other.m_addr;
710  other.make_invalid();
711  other.m_handle = nullptr;
712  return *this;
713 }
714 
716  if (is_valid()) {
717  if (! UnmapViewOfFile(m_addr)) {
718  throw std::system_error(GetLastError(), std::system_category(), "UnmapViewOfFile failed");
719  }
720  make_invalid();
721  }
722 
723  if (m_handle) {
724  if (! CloseHandle(m_handle)) {
725  throw std::system_error(GetLastError(), std::system_category(), "CloseHandle failed");
726  }
727  m_handle = nullptr;
728  }
729 }
730 
731 inline void osmium::util::MemoryMapping::resize(size_t new_size) {
732  unmap();
733 
734  m_size = new_size;
735  resize_fd(m_fd);
736 
737  m_handle = create_file_mapping();
738  if (!m_handle) {
739  throw std::system_error(GetLastError(), std::system_category(), "CreateFileMapping failed");
740  }
741 
742  m_addr = map_view_of_file();
743  if (!is_valid()) {
744  throw std::system_error(GetLastError(), std::system_category(), "MapViewOfFile failed");
745  }
746 }
747 
748 #endif
749 
750 #endif // OSMIUM_UTIL_MEMORY_MAPPING_HPP
~MemoryMapping() noexcept
Definition: memory_mapping.hpp:208
const T * end() const
Definition: memory_mapping.hpp:468
bool is_valid() const noexcept
Definition: memory_mapping.hpp:505
flag_type get_protection() const noexcept
Definition: memory_mapping.hpp:520
MemoryMapping m_mapping
Definition: memory_mapping.hpp:324
MemoryMapping(size_t size, mapping_mode mode, int fd=-1, off_t offset=0)
Definition: memory_mapping.hpp:537
size_t file_size(int fd)
Definition: file.hpp:67
flag_type get_flags() const noexcept
Definition: memory_mapping.hpp:527
int fd() const noexcept
Definition: memory_mapping.hpp:258
TypedMemoryMapping(size_t size, bool writable, int fd, off_t offset=0)
DEPRECATED: For backwards compatibility.
Definition: memory_mapping.hpp:353
static size_t initial_size(size_t size)
Definition: memory_mapping.hpp:137
void unmap()
Definition: memory_mapping.hpp:569
Definition: memory_mapping.hpp:89
Definition: reader_iterator.hpp:39
mapping_mode
Definition: memory_mapping.hpp:92
void resize(size_t new_size)
Definition: memory_mapping.hpp:400
int flag_type
Definition: memory_mapping.hpp:126
int resize_fd(int fd)
Definition: memory_mapping.hpp:150
T * end()
Definition: memory_mapping.hpp:452
size_t get_pagesize()
Definition: file.hpp:103
void * m_addr
The address where the memory is mapped.
Definition: memory_mapping.hpp:117
const T * cbegin() const
Definition: memory_mapping.hpp:456
off_t m_offset
Offset into the file.
Definition: memory_mapping.hpp:104
#define MAP_ANONYMOUS
Definition: memory_mapping.hpp:517
int m_fd
File handle we got the mapping from.
Definition: memory_mapping.hpp:107
size_t size() const noexcept
Definition: memory_mapping.hpp:249
mapping_mode m_mapping_mode
Mapping mode.
Definition: memory_mapping.hpp:110
Namespace for everything in the Osmium library.
Definition: assembler.hpp:55
AnonymousMemoryMapping(size_t size)
Definition: memory_mapping.hpp:298
Definition: memory_mapping.hpp:475
TypedMemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset=0)
Definition: memory_mapping.hpp:348
size_t size() const noexcept
Definition: memory_mapping.hpp:417
T * begin()
Definition: memory_mapping.hpp:443
Definition: memory_mapping.hpp:294
void make_invalid() noexcept
Definition: memory_mapping.hpp:509
MemoryMapping(size_t size, bool writable=true, int fd=-1, off_t offset=0)
DEPRECATED: For backwards compatibility.
Definition: memory_mapping.hpp:183
AnonymousTypedMemoryMapping(size_t size)
Definition: memory_mapping.hpp:479
int fd() const noexcept
Definition: memory_mapping.hpp:427
void resize(size_t new_size)
Definition: memory_mapping.hpp:578
MemoryMapping & operator=(const MemoryMapping &)=delete
You can not copy a MemoryMapping.
Definition: memory_mapping.hpp:322
TypedMemoryMapping & operator=(const TypedMemoryMapping &)=delete
You can not copy a MemoryMapping.
void unmap()
Definition: memory_mapping.hpp:386
bool writable() const noexcept
Definition: memory_mapping.hpp:265
void resize_file(int fd, size_t new_size)
Definition: file.hpp:94
const T * begin() const
Definition: memory_mapping.hpp:464
size_t m_size
The size of the mapping.
Definition: memory_mapping.hpp:101
const T * cend() const
Definition: memory_mapping.hpp:460
TypedMemoryMapping(size_t size)
Definition: memory_mapping.hpp:334
T * get_addr() const
Definition: memory_mapping.hpp:275
bool writable() const noexcept
Definition: memory_mapping.hpp:434