DOLFIN-X
DOLFIN-X C++ interface
IndexMap.h
1 // Copyright (C) 2015-2019 Chris Richardson, Garth N. Wells and Igor Baratta
2 //
3 // This file is part of DOLFINX (https://www.fenicsproject.org)
4 //
5 // SPDX-License-Identifier: LGPL-3.0-or-later
6 
7 #pragma once
8 
9 #include <Eigen/Dense>
10 #include <array>
11 #include <cstdint>
12 #include <dolfinx/common/MPI.h>
13 #include <map>
14 #include <tuple>
15 #include <vector>
16 
17 namespace dolfinx::common
18 {
19 // Forward declaration
20 class IndexMap;
21 
31 std::tuple<std::int64_t, std::vector<std::int32_t>,
32  std::vector<std::vector<std::int64_t>>,
33  std::vector<std::vector<int>>>
35  const std::vector<std::reference_wrapper<const common::IndexMap>>& maps);
36 
43 
44 class IndexMap
45 {
46 public:
48  enum class Mode
49  {
50  insert,
51  add
52  };
53 
55  enum class Direction
56  {
57  reverse, // Ghost to owner
58  forward, // Owner to ghost
59  symmetric // Symmetric. NOTE: To be removed
60  };
61 
70  IndexMap(MPI_Comm comm, std::int32_t local_size, int block_size);
71 
86  IndexMap(MPI_Comm mpi_comm, std::int32_t local_size,
87  const std::vector<int>& dest_ranks,
88  const std::vector<std::int64_t>& ghosts,
89  const std::vector<int>& src_ranks, int block_size);
90 
103  IndexMap(
104  MPI_Comm mpi_comm, std::int32_t local_size,
105  const std::vector<int>& dest_ranks,
106  const Eigen::Ref<const Eigen::Array<std::int64_t, Eigen::Dynamic, 1>>&
107  ghosts,
108  const std::vector<int>& src_ranks, int block_size);
109 
111  IndexMap(const IndexMap& map) = delete;
112 
114  IndexMap(IndexMap&& map) = default;
115 
117  ~IndexMap() = default;
118 
120  std::array<std::int64_t, 2> local_range() const noexcept;
121 
123  int block_size() const noexcept;
124 
126  std::int32_t num_ghosts() const;
127 
129  std::int32_t size_local() const;
130 
132  std::int64_t size_global() const;
133 
136  const Eigen::Array<std::int64_t, Eigen::Dynamic, 1>& ghosts() const;
137 
143  MPI_Comm comm(Direction dir = Direction::symmetric) const;
144 
151  Eigen::Array<std::int64_t, Eigen::Dynamic, 1> local_to_global(
152  const Eigen::Ref<const Eigen::Array<std::int32_t, Eigen::Dynamic, 1>>&
153  indices,
154  bool blocked = true) const;
155 
165  std::vector<std::int64_t>
166  local_to_global(const std::vector<std::int32_t>& indices,
167  bool blocked = true) const;
168 
175  std::vector<std::int32_t>
176  global_to_local(const std::vector<std::int64_t>& indices,
177  bool blocked = true) const;
178 
185  std::vector<std::int32_t> global_to_local(
186  const Eigen::Ref<const Eigen::Array<std::int64_t, Eigen::Dynamic, 1>>&
187  indices,
188  bool blocked = true) const;
189 
193  std::vector<std::int64_t> global_indices(bool blocked = true) const;
194 
199  const std::vector<std::int32_t>& shared_indices() const;
200 
202  Eigen::Array<int, Eigen::Dynamic, 1> ghost_owner_rank() const;
203 
206  Eigen::Array<std::int64_t, Eigen::Dynamic, 1>
207  indices(bool unroll_block) const;
208 
215  std::map<std::int32_t, std::set<int>> compute_shared_indices() const;
216 
227  void scatter_fwd(const std::vector<std::int64_t>& local_data,
228  std::vector<std::int64_t>& remote_data, int n) const;
229 
240  void scatter_fwd(const std::vector<std::int32_t>& local_data,
241  std::vector<std::int32_t>& remote_data, int n) const;
242 
253  std::vector<std::int64_t>
254  scatter_fwd(const std::vector<std::int64_t>& local_data, int n) const;
255 
265  std::vector<std::int32_t>
266  scatter_fwd(const std::vector<std::int32_t>& local_data, int n) const;
267 
277  void scatter_rev(std::vector<std::int64_t>& local_data,
278  const std::vector<std::int64_t>& remote_data, int n,
279  IndexMap::Mode op) const;
280 
290  void scatter_rev(std::vector<std::int32_t>& local_data,
291  const std::vector<std::int32_t>& remote_data, int n,
292  IndexMap::Mode op) const;
293 
294 private:
295  int _block_size;
296 
297  // Range of indices (global) owned by this process
298  std::array<std::int64_t, 2> _local_range;
299 
300  // Number indices across communicator
301  std::int64_t _size_global;
302 
303  // MPI neighborhood communicators
304 
305  // Communicator where the source ranks own the indices in the callers
306  // halo, and the destination ranks 'ghost' indices owned by the
307  // caller. I.e.,
308  // - in-edges (src) are from ranks that own my ghosts
309  // - out-edges (dest) go to ranks that 'ghost' my owned indices
310  dolfinx::MPI::Comm _comm_owner_to_ghost;
311 
312  // Communicator where the source ranks have ghost indices that are
313  // owned by the caller, and the destination ranks are the owners of
314  // indices in the callers halo region. I.e.,
315  // - in-edges (src) are from ranks that 'ghost' my owned indicies
316  // - out-edges (dest) are to the owning ranks of my ghost indices
317  dolfinx::MPI::Comm _comm_ghost_to_owner;
318 
319  // TODO: remove
320  dolfinx::MPI::Comm _comm_symmetric;
321 
322  // Local-to-global map for ghost indices
323  Eigen::Array<std::int64_t, Eigen::Dynamic, 1> _ghosts;
324 
325  // Owning neighborhood rank (out edge) on '_comm_owner_to_ghost'
326  // communicator for each ghost index
327  Eigen::Array<std::int32_t, Eigen::Dynamic, 1> _ghost_owners;
328 
329  // TODO: replace _shared_disp and _shared_disp by an AdjacencyList
330 
331  // TODO: _shared_indices are received on _comm_ghost_to_owner, and
332  // _shared_indices is the recv_disp on _comm_ghost_to_owner. Check for
333  // corect use on _comm_owner_to_ghost. Can guarantee that
334  // _comm_owner_to_ghost and _comm_ghost_to_owner are the transpose of
335  // each other?
336 
337  // Owned local indices that are in the halo (ghost) region on other
338  // ranks
339  std::vector<std::int32_t> _shared_indices;
340 
341  // FIXME: explain better the ranks
342  // Displacement vector for _shared_indices. _shared_indices[i] is the
343  // starting postion in _shared_indices for data that is ghosted on
344  // rank i, where i is the ith outgoing edge on _comm_owner_to_ghost.
345  std::vector<std::int32_t> _shared_disp;
346 
347  template <typename T>
348  void scatter_fwd_impl(const std::vector<T>& local_data,
349  std::vector<T>& remote_data, int n) const;
350  template <typename T>
351  void scatter_rev_impl(std::vector<T>& local_data,
352  const std::vector<T>& remote_data, int n,
353  Mode op) const;
354 };
355 
356 } // namespace dolfinx::common
dolfinx::common::IndexMap::indices
Eigen::Array< std::int64_t, Eigen::Dynamic, 1 > indices(bool unroll_block) const
Return array of global indices for all indices on this process, including ghosts.
Definition: IndexMap.cpp:621
dolfinx::common::IndexMap::~IndexMap
~IndexMap()=default
Destructor.
dolfinx::common::IndexMap
This class represents the distribution index arrays across processes. An index array is a contiguous ...
Definition: IndexMap.h:45
dolfinx::common::IndexMap::IndexMap
IndexMap(IndexMap &&map)=default
Move constructor.
dolfinx::common::IndexMap::ghosts
const Eigen::Array< std::int64_t, Eigen::Dynamic, 1 > & ghosts() const
Local-to-global map for ghosts (local indexing beyond end of local range)
Definition: IndexMap.cpp:498
dolfinx::common::IndexMap::local_range
std::array< std::int64_t, 2 > local_range() const noexcept
Range of indices (global) owned by this process.
Definition: IndexMap.cpp:482
dolfinx::common::stack_index_maps
std::tuple< std::int64_t, std::vector< std::int32_t >, std::vector< std::vector< std::int64_t > >, std::vector< std::vector< int > > > stack_index_maps(const std::vector< std::reference_wrapper< const common::IndexMap >> &maps)
Compute layout data and ghost indices for a stacked (concatenated) index map, i.e....
Definition: IndexMap.cpp:226
dolfinx::MPI
This class provides utility functions for easy communication with MPI and handles cases when DOLFINX ...
Definition: MPI.h:31
dolfinx::common::IndexMap::size_global
std::int64_t size_global() const
Number indices across communicator.
Definition: IndexMap.cpp:496
dolfinx::common::IndexMap::Direction
Direction
Edge directions of neighborhood communicator.
Definition: IndexMap.h:56
dolfinx::common::IndexMap::local_to_global
Eigen::Array< std::int64_t, Eigen::Dynamic, 1 > local_to_global(const Eigen::Ref< const Eigen::Array< std::int32_t, Eigen::Dynamic, 1 >> &indices, bool blocked=true) const
Compute global indices for array of local indices.
Definition: IndexMap.cpp:503
dolfinx::common::IndexMap::size_local
std::int32_t size_local() const
Number of indices owned by on this process.
Definition: IndexMap.cpp:491
dolfinx::common::IndexMap::comm
MPI_Comm comm(Direction dir=Direction::symmetric) const
Return a MPI communicator with attached distributed graph topology information.
Definition: IndexMap.cpp:636
dolfinx::common::IndexMap::scatter_fwd
void scatter_fwd(const std::vector< std::int64_t > &local_data, std::vector< std::int64_t > &remote_data, int n) const
Send n values for each index that is owned to processes that have the index as a ghost....
Definition: IndexMap.cpp:767
dolfinx::common::IndexMap::global_to_local
std::vector< std::int32_t > global_to_local(const std::vector< std::int64_t > &indices, bool blocked=true) const
Compute local indices for array of global indices.
Definition: IndexMap.cpp:553
dolfinx::common::IndexMap::compute_shared_indices
std::map< std::int32_t, std::set< int > > compute_shared_indices() const
Definition: IndexMap.cpp:651
dolfinx::common::IndexMap::scatter_rev
void scatter_rev(std::vector< std::int64_t > &local_data, const std::vector< std::int64_t > &remote_data, int n, IndexMap::Mode op) const
Send n values for each ghost index to owning to the process.
Definition: IndexMap.cpp:795
dolfinx::common::IndexMap::block_size
int block_size() const noexcept
Block size.
Definition: IndexMap.cpp:487
dolfinx::common
Miscellaneous classes, functions and types.
dolfinx::common::IndexMap::shared_indices
const std::vector< std::int32_t > & shared_indices() const
Definition: IndexMap.cpp:598
dolfinx::common::IndexMap::num_ghosts
std::int32_t num_ghosts() const
Number of ghost indices on this process.
Definition: IndexMap.cpp:489
dolfinx::common::IndexMap::Mode
Mode
Mode for reverse scatter operation.
Definition: IndexMap.h:49
dolfinx::common::IndexMap::ghost_owner_rank
Eigen::Array< int, Eigen::Dynamic, 1 > ghost_owner_rank() const
Owner rank (on global communicator) of each ghost entry.
Definition: IndexMap.cpp:603
dolfinx::common::IndexMap::IndexMap
IndexMap(const IndexMap &map)=delete
Copy constructor.
dolfinx::common::IndexMap::global_indices
std::vector< std::int64_t > global_indices(bool blocked=true) const
Global indices.
Definition: IndexMap.cpp:533
dolfinx::common::IndexMap::IndexMap
IndexMap(MPI_Comm comm, std::int32_t local_size, int block_size)
Create an non-overlapping index map with local_size owned blocks on this process.
Definition: IndexMap.cpp:344