escript  Revision_
EsysMPI.h
Go to the documentation of this file.
1 
2 /*****************************************************************************
3 *
4 * Copyright (c) 2003-2016 by The University of Queensland
5 * http://www.uq.edu.au
6 *
7 * Primary Business: Queensland, Australia
8 * Licensed under the Apache License, version 2.0
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Development until 2012 by Earth Systems Science Computational Center (ESSCC)
12 * Development 2012-2013 by School of Earth Sciences
13 * Development from 2014 by Centre for Geoscience Computing (GeoComp)
14 *
15 *****************************************************************************/
16 
17 #ifndef __ESCRIPT_ESYSMPI_H__
18 #define __ESCRIPT_ESYSMPI_H__
19 
20 #include <escript/DataTypes.h>
21 
22 #include <ctime>
23 #include <sstream>
24 
25 #include <boost/shared_ptr.hpp>
26 
27 #ifdef _OPENMP
28 #include <omp.h>
29 #endif
30 
31 #ifdef ESYS_MPI
32 #include <mpi.h>
33 
34 #ifdef ESYS_INDEXTYPE_LONG
35 #define MPI_DIM_T MPI_LONG
36 #else
37 #define MPI_DIM_T MPI_INT
38 #endif
39 
40 #else
41  typedef int MPI_Comm;
42  typedef int MPI_Request;
43  typedef int MPI_Op;
44  typedef int MPI_Status;
45  #define MPI_INT 6
46  #define MPI_DOUBLE 11
47  #define MPI_COMM_WORLD 91
48  #define MPI_COMM_NULL 0
49 
50  // MPI_Op replacements for non-MPI - these values are arbitrary
51  #define MPI_SUM 100
52  #define MPI_MIN 101
53  #define MPI_MAX 102
54 
55  #define MPI_OP_NULL 17
56  // end MPI_op
57 
58 #endif // ESYS_MPI
59 
60 namespace escript {
61 
66 inline int getSubWorldTag()
67 {
68  return (('S'<< 24) + ('u' << 16) + ('b' << 8) + 'W')%1010201;
69 }
70 
71 class JMPI_;
72 
73 typedef boost::shared_ptr<JMPI_> JMPI;
74 
77 JMPI makeInfo(MPI_Comm comm, bool owncom=false);
78 
79 class JMPI_
80 {
81 public:
82  ~JMPI_();
83 
86  DataTypes::index_t max_id,
87  DataTypes::index_t* distribution);
88 
91  DataTypes::index_t* offset);
92 
95  inline int mod_rank(int k) const
96  {
97  int out=0;
98 #ifdef ESYS_MPI
99  if (size > 1) {
100  const int q = k/size;
101  if (k > 0) {
102  out=k-size*q;
103  } else if (k < 0) {
104  out=k-size*(q-1);
105  }
106  }
107 #endif
108  return out;
109  }
110 
112  inline std::string appendRankToFileName(const std::string& fileName) const
113  {
114 #ifdef ESYS_MPI
115  if (size > 1) {
116  std::stringstream ss;
117  ss << fileName << '.';
118  ss.fill('0');
119  ss.width(4);
120  ss << rank;
121  return ss.str();
122  }
123 #endif
124  return fileName;
125  }
126 
128  inline int counter() const
129  {
130  return msg_tag_counter;
131  }
132 
134  inline void incCounter(int i=1)
135  {
136  msg_tag_counter+=i;
137  // there is no particular significance here other than being 7 digits
138  // and prime (because why not). It just needs to be big.
139  msg_tag_counter %= 1010201;
140  }
141 
143  inline void setCounter(int value)
144  {
145  msg_tag_counter = value%1010201;
146  }
147 
149  inline bool isValid() const
150  {
151  return comm!=MPI_COMM_NULL;
152  }
153 
154  int size;
155  int rank;
157 
158 private:
159  JMPI_(MPI_Comm comm, bool owncomm);
160  friend JMPI makeInfo(MPI_Comm comm, bool owncom);
161 
162  bool ownscomm;
164 };
165 
166 // Does not cope with nested calls
168 {
169 public:
170  NoCOMM_WORLD();
171  ~NoCOMM_WORLD();
172  static bool active();
173 };
174 
176 bool checkResult(int input, int& output, const JMPI& comm);
177 
181 bool shipString(const char* src, char** dest, MPI_Comm& comm);
182 
184 inline double gettime()
185 {
186  double out;
187 #ifdef ESYS_MPI
188  out = MPI_Wtime();
189 #else
190 #ifdef _OPENMP
191  out=omp_get_wtime();
192 #else
193  out=((double) clock())/CLOCKS_PER_SEC;
194 #endif
195 #endif
196  return out;
197 }
198 
199 } // namespace escript
200 
201 #endif // __ESCRIPT_ESYSMPI_H__
202 
bool ownscomm
Definition: EsysMPI.h:162
bool shipString(const char *src, char **dest, MPI_Comm &comm)
Definition: EsysMPI.cpp:163
int getSubWorldTag()
tag reserved for use by SubWorld code This value should be higher than the modulus used in JMPI_::set...
Definition: EsysMPI.h:66
Definition: EsysMPI.h:79
int MPI_Op
Definition: EsysMPI.h:43
bool isValid() const
returns true if this has a valid MPI communicator
Definition: EsysMPI.h:149
int MPI_Status
Definition: EsysMPI.h:44
JMPI makeInfo(MPI_Comm comm, bool owncom)
Definition: EsysMPI.cpp:28
Definition: AbstractContinuousDomain.cpp:22
std::string appendRankToFileName(const std::string &fileName) const
appends MPI rank to a file name if MPI size > 1
Definition: EsysMPI.h:112
Definition: EsysMPI.h:167
boost::shared_ptr< JMPI_ > JMPI
Definition: EsysMPI.h:71
JMPI_(MPI_Comm comm, bool owncomm)
Definition: EsysMPI.cpp:37
int counter() const
returns the current value of the message tag counter
Definition: EsysMPI.h:128
int mod_rank(int k) const
Definition: EsysMPI.h:95
friend JMPI makeInfo(MPI_Comm comm, bool owncom)
Definition: EsysMPI.cpp:28
int rank
Definition: EsysMPI.h:155
MPI_Comm comm
Definition: EsysMPI.h:156
bool checkResult(int res, int &mres, const JMPI &info)
Everyone puts in their error code and everyone gets the largest one.
Definition: EsysMPI.cpp:109
DataTypes::dim_t setDistribution(DataTypes::index_t min_id, DataTypes::index_t max_id, DataTypes::index_t *distribution)
Definition: EsysMPI.cpp:64
int index_t
type for array/matrix indices used both globally and on each rank
Definition: DataTypes.h:59
int size
Definition: EsysMPI.h:154
~JMPI_()
Definition: EsysMPI.cpp:56
void setCounter(int value)
sets the message tag counter to value
Definition: EsysMPI.h:143
#define MPI_COMM_NULL
Definition: EsysMPI.h:48
static dim_t N
Definition: SparseMatrix_saveHB.cpp:37
int MPI_Request
Definition: EsysMPI.h:42
int msg_tag_counter
Definition: EsysMPI.h:163
int MPI_Comm
Definition: EsysMPI.h:41
void incCounter(int i=1)
increments the message tag counter by i
Definition: EsysMPI.h:134
void split(DataTypes::dim_t N, DataTypes::dim_t *local_N, DataTypes::index_t *offset)
Definition: EsysMPI.cpp:91
double gettime()
returns the current ticks for timing
Definition: EsysMPI.h:184
index_t dim_t
Definition: DataTypes.h:64