Libosmium  2.6.1
Fast and flexible C++ library for working with OpenStreetMap data
memory_mapping.hpp
Go to the documentation of this file.
1 #ifndef OSMIUM_UTIL_MEMORY_MAPPING_HPP
2 #define OSMIUM_UTIL_MEMORY_MAPPING_HPP
3 
4 /*
5 
6 This file is part of Osmium (http://osmcode.org/libosmium).
7 
8 Copyright 2013-2016 Jochen Topf <jochen@topf.org> and others (see README).
9 
10 Boost Software License - Version 1.0 - August 17th, 2003
11 
12 Permission is hereby granted, free of charge, to any person or organization
13 obtaining a copy of the software and accompanying documentation covered by
14 this license (the "Software") to use, reproduce, display, distribute,
15 execute, and transmit the Software, and to prepare derivative works of the
16 Software, and to permit third-parties to whom the Software is furnished to
17 do so, all subject to the following:
18 
19 The copyright notices in the Software and this entire statement, including
20 the above license grant, this restriction and the following disclaimer,
21 must be included in all copies of the Software, in whole or in part, and
22 all derivative works of the Software, unless such copies or derivative
23 works are solely in the form of machine-executable object code generated by
24 a source language processor.
25 
26 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
27 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
29 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
30 FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
31 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
32 DEALINGS IN THE SOFTWARE.
33 
34 */
35 
36 #include <cassert>
37 #include <cerrno>
38 #include <stdexcept>
39 #include <system_error>
40 
42 #include <osmium/util/file.hpp>
43 
44 #ifndef _WIN32
45 # include <sys/mman.h>
46 #else
47 # include <fcntl.h>
48 # include <io.h>
49 # include <windows.h>
50 # include <sys/types.h>
51 #endif
52 
53 namespace osmium {
54 
55  namespace util {
56 
94  class MemoryMapping {
95 
96 public:
97  enum class mapping_mode {
98  readonly = 0,
99  write_private = 1,
100  write_shared = 2
101  };
102 
103 private:
104 
106  size_t m_size;
107 
109  off_t m_offset;
110 
112  int m_fd;
113 
116 
117 #ifdef _WIN32
118  HANDLE m_handle;
119 #endif
120 
122  void* m_addr;
123 
124  bool is_valid() const noexcept;
125 
126  void make_invalid() noexcept;
127 
128 #ifdef _WIN32
129  typedef DWORD flag_type;
130 #else
131  typedef int flag_type;
132 #endif
133 
134  flag_type get_protection() const noexcept;
135 
136  flag_type get_flags() const noexcept;
137 
138  // A zero-sized mapping is not allowed by the operating system.
139  // So if the user asks for a mapping of size 0, we map a full
140  // page instead. This way we don't have a special case in the rest
141  // of the code.
142  static size_t initial_size(size_t size) {
143  if (size == 0) {
145  }
146  return size;
147  }
148 
149 #ifdef _WIN32
150  HANDLE get_handle() const noexcept;
151  HANDLE osmium::util::MemoryMapping::create_file_mapping() const noexcept;
152  void* osmium::util::MemoryMapping::map_view_of_file() const noexcept;
153 #endif
154 
155  int resize_fd(int fd) {
156  // Anonymous mapping doesn't need resizing.
157  if (fd == -1) {
158  return -1;
159  }
160 
161  // Make sure the file backing this mapping is large enough.
162  if (osmium::util::file_size(fd) < m_size + m_offset) {
163  osmium::util::resize_file(fd, m_size + m_offset);
164  }
165  return fd;
166  }
167 
168  public:
169 
186  MemoryMapping(size_t size, mapping_mode mode, int fd=-1, off_t offset=0);
187 
193  OSMIUM_DEPRECATED MemoryMapping(size_t size, bool writable=true, int fd=-1, off_t offset=0) :
194  MemoryMapping(size, writable ? mapping_mode::write_shared : mapping_mode::readonly, fd, offset) {
195  }
196 
198  MemoryMapping(const MemoryMapping&) = delete;
199 
201  MemoryMapping& operator=(const MemoryMapping&) = delete;
202 
207  MemoryMapping(MemoryMapping&& other);
208 
213 
218  ~MemoryMapping() noexcept {
219  try {
220  unmap();
221  } catch (std::system_error&) {
222  // Ignore any exceptions because destructor must not throw.
223  }
224  }
225 
232  void unmap();
233 
245  void resize(size_t new_size);
246 
251  explicit operator bool() const noexcept {
252  return is_valid();
253  }
254 
260  size_t size() const noexcept {
261  return m_size;
262  }
263 
269  int fd() const noexcept {
270  return m_fd;
271  }
272 
276  bool writable() const noexcept {
277  return m_mapping_mode != mapping_mode::readonly;
278  }
279 
285  template <typename T = void>
286  T* get_addr() const {
287  if (is_valid()) {
288  return reinterpret_cast<T*>(m_addr);
289  }
290  throw std::runtime_error("invalid memory mapping");
291  }
292 
293  }; // class MemoryMapping
294 
306 
307  public:
308 
311  }
312 
313 #ifndef __linux__
314 
318  void resize(size_t) = delete;
319 #endif
320 
321  }; // class AnonymousMemoryMapping
322 
332  template <typename T>
334 
336 
337  public:
338 
346  m_mapping(sizeof(T) * size, MemoryMapping::mapping_mode::write_private) {
347  }
348 
359  TypedMemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset = 0) :
360  m_mapping(sizeof(T) * size, mode, fd, sizeof(T) * offset) {
361  }
362 
368  OSMIUM_DEPRECATED TypedMemoryMapping(size_t size, bool writable, int fd, off_t offset = 0) :
369  m_mapping(sizeof(T) * size, writable ? MemoryMapping::mapping_mode::write_shared : MemoryMapping::mapping_mode::readonly, fd, sizeof(T) * offset) {
370  }
371 
373  TypedMemoryMapping(const TypedMemoryMapping&) = delete;
374 
377 
382  TypedMemoryMapping(TypedMemoryMapping&& other) = default;
383 
387  TypedMemoryMapping& operator=(TypedMemoryMapping&& other) = default;
388 
393  ~TypedMemoryMapping() noexcept = default;
394 
401  void unmap() {
402  m_mapping.unmap();
403  }
404 
415  void resize(size_t new_size) {
416  m_mapping.resize(sizeof(T) * new_size);
417  }
418 
423  explicit operator bool() const noexcept {
424  return !!m_mapping;
425  }
426 
432  size_t size() const noexcept {
433  assert(m_mapping.size() % sizeof(T) == 0);
434  return m_mapping.size() / sizeof(T);
435  }
436 
442  int fd() const noexcept {
443  return m_mapping.fd();
444  }
445 
449  bool writable() const noexcept {
450  return m_mapping.writable();
451  }
452 
458  T* begin() {
459  return m_mapping.get_addr<T>();
460  }
461 
467  T* end() {
468  return m_mapping.get_addr<T>() + size();
469  }
470 
471  const T* cbegin() const {
472  return m_mapping.get_addr<T>();
473  }
474 
475  const T* cend() const {
476  return m_mapping.get_addr<T>() + size();
477  }
478 
479  const T* begin() const {
480  return m_mapping.get_addr<T>();
481  }
482 
483  const T* end() const {
484  return m_mapping.get_addr<T>() + size();
485  }
486 
487  }; // class TypedMemoryMapping
488 
489  template <typename T>
491 
492  public:
493 
495  TypedMemoryMapping<T>(size) {
496  }
497 
498 #ifndef __linux__
499 
503  void resize(size_t) = delete;
504 #endif
505 
506  }; // class AnonymousTypedMemoryMapping
507 
508  } // namespace util
509 
510 } // namespace osmium
511 
512 #ifndef _WIN32
513 
514 // =========== Unix implementation =============
515 
516 // MAP_FAILED is often a macro containing an old style cast
517 #pragma GCC diagnostic push
518 #pragma GCC diagnostic ignored "-Wold-style-cast"
519 
520 inline bool osmium::util::MemoryMapping::is_valid() const noexcept {
521  return m_addr != MAP_FAILED;
522 }
523 
525  m_addr = MAP_FAILED;
526 }
527 
528 #pragma GCC diagnostic pop
529 
530 // for BSD systems
531 #ifndef MAP_ANONYMOUS
532 # define MAP_ANONYMOUS MAP_ANON
533 #endif
534 
535 inline int osmium::util::MemoryMapping::get_protection() const noexcept {
537  return PROT_READ;
538  }
539  return PROT_READ | PROT_WRITE;
540 }
541 
542 inline int osmium::util::MemoryMapping::get_flags() const noexcept {
543  if (m_fd == -1) {
544  return MAP_PRIVATE | MAP_ANONYMOUS;
545  }
547  return MAP_SHARED;
548  }
549  return MAP_PRIVATE;
550 }
551 
552 inline osmium::util::MemoryMapping::MemoryMapping(size_t size, mapping_mode mode, int fd, off_t offset) :
553  m_size(initial_size(size)),
554  m_offset(offset),
555  m_fd(resize_fd(fd)),
556  m_mapping_mode(mode),
557  m_addr(::mmap(nullptr, m_size, get_protection(), get_flags(), m_fd, m_offset)) {
558  assert(!(fd == -1 && mode == mapping_mode::readonly));
559  if (!is_valid()) {
560  throw std::system_error(errno, std::system_category(), "mmap failed");
561  }
562 }
563 
565  m_size(other.m_size),
566  m_offset(other.m_offset),
567  m_fd(other.m_fd),
569  m_addr(other.m_addr) {
570  other.make_invalid();
571 }
572 
574  unmap();
575  m_size = other.m_size;
576  m_offset = other.m_offset;
577  m_fd = other.m_fd;
578  m_mapping_mode = other.m_mapping_mode;
579  m_addr = other.m_addr;
580  other.make_invalid();
581  return *this;
582 }
583 
585  if (is_valid()) {
586  if (::munmap(m_addr, m_size) != 0) {
587  throw std::system_error(errno, std::system_category(), "munmap failed");
588  }
589  make_invalid();
590  }
591 }
592 
593 inline void osmium::util::MemoryMapping::resize(size_t new_size) {
594  assert(new_size > 0 && "can not resize to zero size");
595  if (m_fd == -1) { // anonymous mapping
596 #ifdef __linux__
597  m_addr = ::mremap(m_addr, m_size, new_size, MREMAP_MAYMOVE);
598  if (!is_valid()) {
599  throw std::system_error(errno, std::system_category(), "mremap failed");
600  }
601  m_size = new_size;
602 #else
603  assert(false && "can't resize anonymous mappings on non-linux systems");
604 #endif
605  } else { // file-based mapping
606  unmap();
607  m_size = new_size;
608  resize_fd(m_fd);
609  m_addr = ::mmap(nullptr, new_size, get_protection(), get_flags(), m_fd, m_offset);
610  if (!is_valid()) {
611  throw std::system_error(errno, std::system_category(), "mmap (remap) failed");
612  }
613  }
614 }
615 
616 #else
617 
618 // =========== Windows implementation =============
619 
620 /* References:
621  * CreateFileMapping: http://msdn.microsoft.com/en-us/library/aa366537(VS.85).aspx
622  * CloseHandle: http://msdn.microsoft.com/en-us/library/ms724211(VS.85).aspx
623  * MapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366761(VS.85).aspx
624  * UnmapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366882(VS.85).aspx
625  */
626 
627 namespace osmium {
628 
629  namespace util {
630 
631  inline DWORD dword_hi(uint64_t x) {
632  return static_cast<DWORD>(x >> 32);
633  }
634 
635  inline DWORD dword_lo(uint64_t x) {
636  return static_cast<DWORD>(x & 0xffffffff);
637  }
638 
639  } // namespace util
640 
641 } // namespace osmium
642 
643 inline DWORD osmium::util::MemoryMapping::get_protection() const noexcept {
644  switch (m_mapping_mode) {
646  return PAGE_READONLY;
648  return PAGE_WRITECOPY;
650  return PAGE_READWRITE;
651  }
652 }
653 
654 inline DWORD osmium::util::MemoryMapping::get_flags() const noexcept {
655  switch (m_mapping_mode) {
657  return FILE_MAP_READ;
659  return FILE_MAP_COPY;
661  return FILE_MAP_WRITE;
662  }
663 }
664 
665 inline HANDLE osmium::util::MemoryMapping::get_handle() const noexcept {
666  if (m_fd == -1) {
667  return INVALID_HANDLE_VALUE;
668  }
669  return reinterpret_cast<HANDLE>(_get_osfhandle(m_fd));
670 }
671 
672 inline HANDLE osmium::util::MemoryMapping::create_file_mapping() const noexcept {
673  if (m_fd != -1) {
674  _setmode(m_fd, _O_BINARY);
675  }
676  return CreateFileMapping(get_handle(), nullptr, get_protection(), osmium::util::dword_hi(static_cast<uint64_t>(m_size) + m_offset), osmium::util::dword_lo(static_cast<uint64_t>(m_size) + m_offset), nullptr);
677 }
678 
679 inline void* osmium::util::MemoryMapping::map_view_of_file() const noexcept {
680  return MapViewOfFile(m_handle, get_flags(), osmium::util::dword_hi(m_offset), osmium::util::dword_lo(m_offset), m_size);
681 }
682 
683 inline bool osmium::util::MemoryMapping::is_valid() const noexcept {
684  return m_addr != nullptr;
685 }
686 
687 inline void osmium::util::MemoryMapping::make_invalid() noexcept {
688  m_addr = nullptr;
689 }
690 
691 inline osmium::util::MemoryMapping::MemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset) :
692  m_size(initial_size(size)),
693  m_offset(offset),
694  m_fd(resize_fd(fd)),
695  m_mapping_mode(mode),
696  m_handle(create_file_mapping()),
697  m_addr(nullptr) {
698 
699  if (!m_handle) {
700  throw std::system_error(GetLastError(), std::system_category(), "CreateFileMapping failed");
701  }
702 
703  m_addr = map_view_of_file();
704  if (!is_valid()) {
705  throw std::system_error(GetLastError(), std::system_category(), "MapViewOfFile failed");
706  }
707 }
708 
710  m_size(other.m_size),
711  m_offset(other.m_offset),
712  m_fd(other.m_fd),
713  m_mapping_mode(other.m_mapping_mode),
714  m_handle(std::move(other.m_handle)),
715  m_addr(other.m_addr) {
716  other.make_invalid();
717  other.m_handle = nullptr;
718 }
719 
721  unmap();
722  m_size = other.m_size;
723  m_offset = other.m_offset;
724  m_fd = other.m_fd;
725  m_mapping_mode = other.m_mapping_mode;
726  m_handle = std::move(other.m_handle);
727  m_addr = other.m_addr;
728  other.make_invalid();
729  other.m_handle = nullptr;
730  return *this;
731 }
732 
734  if (is_valid()) {
735  if (! UnmapViewOfFile(m_addr)) {
736  throw std::system_error(GetLastError(), std::system_category(), "UnmapViewOfFile failed");
737  }
738  make_invalid();
739  }
740 
741  if (m_handle) {
742  if (! CloseHandle(m_handle)) {
743  throw std::system_error(GetLastError(), std::system_category(), "CloseHandle failed");
744  }
745  m_handle = nullptr;
746  }
747 }
748 
749 inline void osmium::util::MemoryMapping::resize(size_t new_size) {
750  unmap();
751 
752  m_size = new_size;
753  resize_fd(m_fd);
754 
755  m_handle = create_file_mapping();
756  if (!m_handle) {
757  throw std::system_error(GetLastError(), std::system_category(), "CreateFileMapping failed");
758  }
759 
760  m_addr = map_view_of_file();
761  if (!is_valid()) {
762  throw std::system_error(GetLastError(), std::system_category(), "MapViewOfFile failed");
763  }
764 }
765 
766 #endif
767 
768 #endif // OSMIUM_UTIL_MEMORY_MAPPING_HPP
~MemoryMapping() noexcept
Definition: memory_mapping.hpp:218
const T * end() const
Definition: memory_mapping.hpp:483
bool is_valid() const noexcept
Definition: memory_mapping.hpp:520
#define OSMIUM_DEPRECATED
Definition: compatibility.hpp:50
flag_type get_protection() const noexcept
Definition: memory_mapping.hpp:535
MemoryMapping m_mapping
Definition: memory_mapping.hpp:335
MemoryMapping(size_t size, mapping_mode mode, int fd=-1, off_t offset=0)
Definition: memory_mapping.hpp:552
size_t file_size(int fd)
Definition: file.hpp:69
flag_type get_flags() const noexcept
Definition: memory_mapping.hpp:542
int fd() const noexcept
Definition: memory_mapping.hpp:269
static size_t initial_size(size_t size)
Definition: memory_mapping.hpp:142
void unmap()
Definition: memory_mapping.hpp:584
Definition: memory_mapping.hpp:94
mapping_mode
Definition: memory_mapping.hpp:97
void resize(size_t new_size)
Definition: memory_mapping.hpp:415
int flag_type
Definition: memory_mapping.hpp:131
int resize_fd(int fd)
Definition: memory_mapping.hpp:155
T * end()
Definition: memory_mapping.hpp:467
size_t get_pagesize()
Definition: file.hpp:105
void * m_addr
The address where the memory is mapped.
Definition: memory_mapping.hpp:122
const T * cbegin() const
Definition: memory_mapping.hpp:471
off_t m_offset
Offset into the file.
Definition: memory_mapping.hpp:109
#define MAP_ANONYMOUS
Definition: memory_mapping.hpp:532
int m_fd
File handle we got the mapping from.
Definition: memory_mapping.hpp:112
size_t size() const noexcept
Definition: memory_mapping.hpp:260
mapping_mode m_mapping_mode
Mapping mode.
Definition: memory_mapping.hpp:115
Namespace for everything in the Osmium library.
Definition: assembler.hpp:59
AnonymousMemoryMapping(size_t size)
Definition: memory_mapping.hpp:309
Definition: memory_mapping.hpp:490
OSMIUM_DEPRECATED TypedMemoryMapping(size_t size, bool writable, int fd, off_t offset=0)
Definition: memory_mapping.hpp:368
TypedMemoryMapping(size_t size, MemoryMapping::mapping_mode mode, int fd, off_t offset=0)
Definition: memory_mapping.hpp:359
size_t size() const noexcept
Definition: memory_mapping.hpp:432
T * begin()
Definition: memory_mapping.hpp:458
Definition: memory_mapping.hpp:305
void make_invalid() noexcept
Definition: memory_mapping.hpp:524
AnonymousTypedMemoryMapping(size_t size)
Definition: memory_mapping.hpp:494
int fd() const noexcept
Definition: memory_mapping.hpp:442
void resize(size_t new_size)
Definition: memory_mapping.hpp:593
MemoryMapping & operator=(const MemoryMapping &)=delete
You can not copy a MemoryMapping.
Definition: memory_mapping.hpp:333
void unmap()
Definition: memory_mapping.hpp:401
bool writable() const noexcept
Definition: memory_mapping.hpp:276
void resize_file(int fd, size_t new_size)
Definition: file.hpp:96
const T * begin() const
Definition: memory_mapping.hpp:479
size_t m_size
The size of the mapping.
Definition: memory_mapping.hpp:106
const T * cend() const
Definition: memory_mapping.hpp:475
TypedMemoryMapping(size_t size)
Definition: memory_mapping.hpp:345
OSMIUM_DEPRECATED MemoryMapping(size_t size, bool writable=true, int fd=-1, off_t offset=0)
Definition: memory_mapping.hpp:193
T * get_addr() const
Definition: memory_mapping.hpp:286
bool writable() const noexcept
Definition: memory_mapping.hpp:449