escript  Revision_
EsysMPI.h
Go to the documentation of this file.
1 
2 /*****************************************************************************
3 *
4 * Copyright (c) 2003-2020 by The University of Queensland
5 * http://www.uq.edu.au
6 *
7 * Primary Business: Queensland, Australia
8 * Licensed under the Apache License, version 2.0
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Development until 2012 by Earth Systems Science Computational Center (ESSCC)
12 * Development 2012-2013 by School of Earth Sciences
13 * Development from 2014-2017 by Centre for Geoscience Computing (GeoComp)
14 * Development from 2019 by School of Earth and Environmental Sciences
15 **
16 *****************************************************************************/
17 
18 #ifndef __ESCRIPT_ESYSMPI_H__
19 #define __ESCRIPT_ESYSMPI_H__
20 
21 #include <escript/DataTypes.h>
22 
23 #include <ctime>
24 #include <sstream>
25 
26 #include <boost/shared_ptr.hpp>
27 
28 #ifdef _OPENMP
29 #include <omp.h>
30 #endif
31 
32 #ifdef ESYS_MPI
33 #include <mpi.h>
34 
35 #ifdef ESYS_INDEXTYPE_LONG
36 #define MPI_DIM_T MPI_LONG
37 #else
38 #define MPI_DIM_T MPI_INT
39 #endif
40 
41 #else
42  typedef int MPI_Comm;
43  typedef int MPI_Request;
44  typedef int MPI_Op;
45  typedef int MPI_Status;
46  #define MPI_INT 6
47  #define MPI_DOUBLE 11
48  #define MPI_COMM_WORLD 91
49  #define MPI_COMM_NULL 0
50 
51  // MPI_Op replacements for non-MPI - these values are arbitrary
52  #define MPI_SUM 100
53  #define MPI_MIN 101
54  #define MPI_MAX 102
55 
56  #define MPI_OP_NULL 17
57  // end MPI_op
58 
59 #endif // ESYS_MPI
60 
61 namespace escript {
62 
67 inline int getSubWorldTag()
68 {
69  return (('S'<< 24) + ('u' << 16) + ('b' << 8) + 'W')%1010201;
70 }
71 
72 class JMPI_;
73 
74 typedef boost::shared_ptr<JMPI_> JMPI;
75 
78 JMPI makeInfo(MPI_Comm comm, bool owncom=false);
79 
80 class JMPI_
81 {
82 public:
83  ~JMPI_();
84 
87  DataTypes::index_t max_id,
88  DataTypes::index_t* distribution);
89 
92  DataTypes::index_t* offset);
93 
96  inline int mod_rank(int k) const
97  {
98  int out=0;
99 #ifdef ESYS_MPI
100  if (size > 1) {
101  const int q = k/size;
102  if (k > 0) {
103  out=k-size*q;
104  } else if (k < 0) {
105  out=k-size*(q-1);
106  }
107  }
108 #endif
109  return out;
110  }
111 
113  inline std::string appendRankToFileName(const std::string& fileName) const
114  {
115 #ifdef ESYS_MPI
116  if (size > 1) {
117  std::stringstream ss;
118  ss << fileName << '.';
119  ss.fill('0');
120  ss.width(4);
121  ss << rank;
122  return ss.str();
123  }
124 #endif
125  return fileName;
126  }
127 
129  inline int counter() const
130  {
131  return msg_tag_counter;
132  }
133 
135  inline void incCounter(int i=1)
136  {
137  msg_tag_counter+=i;
138  // there is no particular significance here other than being 7 digits
139  // and prime (because why not). It just needs to be big.
140  msg_tag_counter %= 1010201;
141  }
142 
144  inline void setCounter(int value)
145  {
146  msg_tag_counter = value%1010201;
147  }
148 
150  inline bool isValid() const
151  {
152  return comm!=MPI_COMM_NULL;
153  }
154 
155  int size;
156  int rank;
158 
159 private:
160  JMPI_(MPI_Comm comm, bool owncomm);
161  friend JMPI makeInfo(MPI_Comm comm, bool owncom);
162 #pragma clang diagnostic push
163 #pragma clang diagnostic ignored "-Wunused-private-field"
164  bool ownscomm;
165 #pragma clang diagnostic pop
167 };
168 
169 // Does not cope with nested calls
171 {
172 public:
173  NoCOMM_WORLD();
174  ~NoCOMM_WORLD();
175  static bool active();
176 };
177 
179 bool checkResult(int input, int& output, const JMPI& comm);
180 
184 bool shipString(const char* src, char** dest, MPI_Comm& comm);
185 
187 inline double gettime()
188 {
189  double out;
190 #ifdef ESYS_MPI
191  out = MPI_Wtime();
192 #else
193 #ifdef _OPENMP
194  out=omp_get_wtime();
195 #else
196  out=((double) clock())/CLOCKS_PER_SEC;
197 #endif
198 #endif
199  return out;
200 }
201 
202 } // namespace escript
203 
204 #endif // __ESCRIPT_ESYSMPI_H__
205 
MPI_Status
int MPI_Status
Definition: EsysMPI.h:45
paso::N
static dim_t N
Definition: SparseMatrix_saveHB.cpp:38
escript::JMPI_
Definition: EsysMPI.h:81
MPI_Op
int MPI_Op
Definition: EsysMPI.h:44
escript::EsysException
The base class for escript exceptions.
Definition: EsysException.h:30
escript::JMPI_::JMPI_
JMPI_(MPI_Comm comm, bool owncomm)
Definition: EsysMPI.cpp:38
escript::getSubWorldTag
int getSubWorldTag()
tag reserved for use by SubWorld code This value should be higher than the modulus used in JMPI_::set...
Definition: EsysMPI.h:67
escript::NoCOMM_WORLD
Definition: EsysMPI.h:171
escript::JMPI_::rank
int rank
Definition: EsysMPI.h:156
escript::checkResult
bool checkResult(int res, int &mres, const JMPI &info)
Everyone puts in their error code and everyone gets the largest one.
Definition: EsysMPI.cpp:110
escript::JMPI_::setDistribution
DataTypes::dim_t setDistribution(DataTypes::index_t min_id, DataTypes::index_t max_id, DataTypes::index_t *distribution)
Definition: EsysMPI.cpp:65
escript::makeInfo
JMPI makeInfo(MPI_Comm comm, bool owncom)
Definition: EsysMPI.cpp:29
MPI_INT
#define MPI_INT
Definition: EsysMPI.h:46
MPI_MAX
#define MPI_MAX
Definition: EsysMPI.h:54
escript::NoCOMM_WORLD::~NoCOMM_WORLD
~NoCOMM_WORLD()
Definition: EsysMPI.cpp:232
MPI_COMM_WORLD
#define MPI_COMM_WORLD
Definition: EsysMPI.h:48
escript::JMPI_::appendRankToFileName
std::string appendRankToFileName(const std::string &fileName) const
appends MPI rank to a file name if MPI size > 1
Definition: EsysMPI.h:113
escript::JMPI_::counter
int counter() const
returns the current value of the message tag counter
Definition: EsysMPI.h:129
escript::DataTypes::dim_t
index_t dim_t
Definition: DataTypes.h:65
escript::JMPI_::makeInfo
friend JMPI makeInfo(MPI_Comm comm, bool owncom)
Definition: EsysMPI.cpp:29
escript::JMPI
boost::shared_ptr< JMPI_ > JMPI
Definition: EsysMPI.h:72
escript::JMPI_::~JMPI_
~JMPI_()
Definition: EsysMPI.cpp:57
escript::JMPI_::mod_rank
int mod_rank(int k) const
Definition: EsysMPI.h:96
escript::JMPI_::msg_tag_counter
int msg_tag_counter
Definition: EsysMPI.h:166
escript::JMPI_::incCounter
void incCounter(int i=1)
increments the message tag counter by i
Definition: EsysMPI.h:135
escript::JMPI_::comm
MPI_Comm comm
Definition: EsysMPI.h:157
escript::JMPI_::setCounter
void setCounter(int value)
sets the message tag counter to value
Definition: EsysMPI.h:144
escript::JMPI_::size
int size
Definition: EsysMPI.h:155
escript::gettime
double gettime()
returns the current ticks for timing
Definition: EsysMPI.h:187
escript::DataTypes::index_t
int index_t
type for array/matrix indices used both globally and on each rank
Definition: DataTypes.h:60
EsysException.h
escript
Definition: AbstractContinuousDomain.cpp:23
MPI_COMM_NULL
#define MPI_COMM_NULL
Definition: EsysMPI.h:49
escript::NoCOMM_WORLD::NoCOMM_WORLD
NoCOMM_WORLD()
Definition: EsysMPI.cpp:224
MPI_Request
int MPI_Request
Definition: EsysMPI.h:43
MPI_Comm
int MPI_Comm
Definition: EsysMPI.h:42
escript::shipString
bool shipString(const char *src, char **dest, MPI_Comm &comm)
Definition: EsysMPI.cpp:164
EsysMPI.h
escript::JMPI_::split
void split(DataTypes::dim_t N, DataTypes::dim_t *local_N, DataTypes::index_t *offset)
Definition: EsysMPI.cpp:92
escript::JMPI_::isValid
bool isValid() const
returns true if this has a valid MPI communicator
Definition: EsysMPI.h:150
escript::JMPI_::ownscomm
bool ownscomm
Definition: EsysMPI.h:164
escript::NoCOMM_WORLD::active
static bool active()
Definition: EsysMPI.cpp:237