escript  Revision_
blocktools.h
Go to the documentation of this file.
1 
2 /*****************************************************************************
3 *
4 * Copyright (c) 2014-2016 by The University of Queensland
5 * http://www.uq.edu.au
6 *
7 * Primary Business: Queensland, Australia
8 * Licensed under the Apache License, version 2.0
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Development until 2012 by Earth Systems Science Computational Center (ESSCC)
12 * Development 2012-2013 by School of Earth Sciences
13 * Development from 2014 by Centre for Geoscience Computing (GeoComp)
14 *
15 *****************************************************************************/
16 
17 #ifndef __RIPLEY_BLOCKTOOLS_H__
18 #define __RIPLEY_BLOCKTOOLS_H__
19 
20 /* This file contains two main classes for dealing with a large 3D region which
21  * has been divided into a 3D Grid of Blocks (usually to be distributed).
22  * Each block is divided into 27 subblocks. The first and last subblocks in
23  * each dimension are cubes.
24  *
25  * class Block manages a single block. It has methods for copying between a
26  * flat array (storing all the values in the 3D block) and buffers storing
27  * individual subblocks. These buffers can be used to transfer individual
28  * subblocks to other blocks (using some external means).
29  *
30  * class BlockGrid deals with the position of a given block in relation to the
31  * rest of the blocks in the grid. (In an MPI setting, there would be one block
32  * per rank.)
33  * It also requires transfers of subblocks into and out of the block in order
34  * to make the whole (global) array consistent.
35  *
36  * Each block has a region "inset" wide in from each edge which is shared with
37  * neighbouring blocks. Where these regions overlap with another block, the
38  * block closest to block 0,0,0 is used.
39  * Or more precisely, values move left->right then lowy->highy and finally
40  * lowz -> highz.
41  *
42  * Please don't mix external calls into this file, it may be useful to separate
43  * it for debugging purposes.
44  *
45  * Types required:
46  * neighbourID_t - Stores the label of a neighbouring block.
47  * In an MPI setting, this will be the type used refer to
48  * ranks
49  * coord_t - Stores a position of a block in the grid of blocks
50  * (it could be within one dimension or overall in a flat
51  * structure. It is not (necessarily) the same as
52  * neighbourID_t because coord_t should be _unsigned_ and
53  * there is no guarantee that neighbourID_t will even be
54  * an integral type.
55  */
56 
57 #include <escript/EsysMPI.h>
58 
59 #include <vector>
60 
61 typedef int neighbourID_t; // This should be the MPI_rank type
62 typedef unsigned coord_t; // if we ever get more than 2^32 ranks, we have other problems
63 
64 typedef std::pair<neighbourID_t, int> neighpair;
65 typedef std::vector<neighpair> neighbourvector;
66 
67 
68 typedef struct
69 {
70 public:
71  neighbourID_t sourceID; // ranks involved in communication
73  int tag;
74  unsigned char srcbuffid; // number of buffer to use for coms
75  unsigned char destbuffid;
76 } message;
77 
78 typedef std::vector<message> messvec;
79 
80 
81 class BlockGrid
82 {
83 public:
84  BlockGrid(coord_t maxx, coord_t maxy, coord_t maxz);
85 
87 
88 
89  // generate all incoming com messages for this block.
90  // for each subblock (27 of them), there may be an x, y, z direction to search in
91  void generateInNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec& v);
92 
93 
94  // generate all outgoing com messages for this block
95  void generateOutNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec& v);
96 
97 private:
101 };
102 
103 
104 
105 /* Do not ask about buffers for sub-block 1,1,1 (also known as #13)
106  They do not exist, such buffers would be:
107  1) big
108  2) unnecessary since the centre sub-block is not sent anywhere
109 
110 Note that this class does not deal with data transfer between blocks
111 Sub-blocks are copied to and from buffers. Other code is required to
112 actually move the data.
113 
114 "dpp" == doubles per point and gives the number of doubles that make up each "point"
115 This is required to calculate offsets and buffer sizes.
116 */
117 
118 class Block
119 {
120 public:
121  // s? specifiy the [local] size (in points) of each dimension
122  Block(size_t sx, size_t sy, size_t sz, size_t inset, size_t xmidlen,
123  size_t ymidlen, size_t zmidlen, unsigned int dpp=1);
124 
125  ~Block();
126 
127  // Out buffers are loaded with the contents of the flat array and are
128  // to be sent to other blocks
129  double* getOutBuffer(unsigned char subx, unsigned char suby, unsigned char subz);
130  double* getOutBuffer(unsigned char bid);
131 
132  // In buffers are populated from external communications
133  // and copied back to the flat array
134  double* getInBuffer(unsigned char subx, unsigned char suby, unsigned char subz);
135  double* getInBuffer(unsigned char bid);
136 
137  // return number of doubles in the given block
138  size_t getBuffSize(unsigned char subx, unsigned char suby, unsigned char subz);
139  size_t getBuffSize(unsigned char bid);
140 
141  // where does the subblock specified start in a source array
142  size_t startOffset(unsigned char subx, unsigned char suby, unsigned char subz);
143 
144  // debug only
145  void displayBlock(unsigned char subx, unsigned char suby, unsigned char subz, bool out);
146 
147  // Copy a 3d region from a flat array into a buffer
148  void copyToBuffer(unsigned char buffid, double* src);
149 
150  // Copy a 3d region from a buffer into a flat array
151  void copyFromBuffer(unsigned char buffid, double* dest);
152 
153 
154  void copyAllToBuffer(double* src);
155 
156  void copyUsedFromBuffer(double* dest);
157 
158  void setUsed(unsigned char buffid);
159 
160 private:
161 
162  // determines the dimensions of each subblock
163  void populateDimsTable();
164  void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen);
165  void createBuffArrays(double* startaddress, double* buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen);
166 
167 
168  double* inbuff;
169  double* outbuff;
170  size_t buffoffsets[27]; // offsets of the various blocks within the buffer arrays
171  size_t flatoffsets[27]; // starting point of each block within a flat array
172  bool used[27];
173  size_t dims[27][3]; // dimension of each subblock
174  size_t sx;
175  size_t sy;
176  size_t sz;
177  size_t inset;
178  size_t xmidlen;
179  size_t ymidlen;
180  size_t zmidlen;
181  double* inbuffptr[27];
182  double* outbuffptr[27];
183  const unsigned int dpsize; // number of doubles which make up a point
184 };
185 
186 // Returns the MPI message tag to use for a transfer between the two subblocks
187 int getTag(unsigned char sourcex, unsigned char sourcey, unsigned char sourcez,
188  unsigned char targetx, unsigned char targety, unsigned char targetz);
189 
190 // computes the tag based on the destination and the direction it comes from
191 // the booleans indicate whether a negative shift in that direction is required
192 int getTag(unsigned char destx, unsigned char desty, unsigned char destz,
193  bool deltax, bool deltay, bool deltaz);
194 
195 
196 // the booleans indicate whether a negative shift in that direction is required
197 unsigned char getSrcBuffID(unsigned char destx, unsigned char desty,
198  unsigned char destz, bool deltax, bool deltay,
199  bool deltaz);
200 
201 
202 /* Now the 2D versions */
203 
204 // 2D version
206 {
207 public:
208  BlockGrid2(coord_t maxx, coord_t maxy);
209 
210  neighbourID_t getNID(coord_t x, coord_t y) const;
211 
212  // generate all incoming com messages for this block.
213  // for each subblock (9 of them), there may be an x, y direction to search in
214  void generateInNeighbours(coord_t blockx, coord_t blocky, messvec& v);
215 
216  // generate all outgoing com messages for this block
217  void generateOutNeighbours(coord_t blockx, coord_t blocky, messvec& v);
218 
219 private:
222 };
223 
224 // The 2D version - there is no block 4
225 class Block2
226 {
227 public:
228  // s? specifiy the [local] size (in points) of each dimension
229  Block2(size_t sx, size_t sy, size_t inset, size_t xmidlen,
230  size_t ymidlen, unsigned int dpp=1);
231 
232  ~Block2();
233 
234  // Out buffers are loaded with the contents of the flat array and are
235  // to be sent to other blocks
236  double* getOutBuffer(unsigned char subx, unsigned char suby);
237  double* getOutBuffer(unsigned char bid);
238 
239  // In buffers are populated from external communications
240  // and copied back to the flat array
241  double* getInBuffer(unsigned char subx, unsigned char suby);
242  double* getInBuffer(unsigned char bid);
243 
244  // return number of doubles in the given block
245  size_t getBuffSize(unsigned char subx, unsigned char suby);
246  size_t getBuffSize(unsigned char bid);
247 
248  // where does the subblock specified start in a source array
249  size_t startOffset(unsigned char subx, unsigned char suby);
250 
251  // debug only
252  void displayBlock(unsigned char subx, unsigned char suby, bool out);
253 
254  // Copy a 3d region from a flat array into a buffer
255  void copyToBuffer(unsigned char buffid, double* src);
256 
257  // Copy a 3d region from a buffer into a flat array
258  void copyFromBuffer(unsigned char buffid, double* dest);
259 
260 
261  void copyAllToBuffer(double* src);
262 
263  void copyUsedFromBuffer(double* dest);
264 
265  void setUsed(unsigned char buffid);
266 
267 private:
268 
269  // determines the dimensions of each subblock
270  void populateDimsTable();
271  void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen);
272  void createBuffArrays(double* startaddress, double* buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen);
273 
274 
275  double* inbuff;
276  double* outbuff;
277  size_t buffoffsets[9]; // offsets of the various blocks within the buffer arrays
278  size_t flatoffsets[9]; // starting point of each block within a flat array
279  bool used[9];
280  size_t dims[9][2]; // dimension of each subblock
281  size_t sx;
282  size_t sy;
283  size_t inset;
284  size_t xmidlen;
285  size_t ymidlen;
286  double* inbuffptr[9];
287  double* outbuffptr[9];
288  const unsigned int dpsize; // number of doubles which make up a point
289 };
290 
291 
292 // Returns the MPI message tag to use for a transfer between the two subblocks
293 int getTag2(unsigned char sourcex, unsigned char sourcey, unsigned char targetx, unsigned char targety);
294 
295 // computes the tag based on the destination and the direction it comes from
296 // the booleans indicate whether a negative shift in that direction is required
297 int getTag2(unsigned char destx, unsigned char desty, bool deltax, bool deltay);
298 
299 
300 // the booleans indicate whether a negative shift in that direction is required
301 unsigned char getSrcBuffID2(unsigned char destx, unsigned char desty, bool deltax, bool deltay);
302 
303 #endif // __RIPLEY_BLOCKTOOLS_H__
304 
void generateInNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec &v)
Definition: blocktools.cpp:43
Definition: blocktools.h:68
int tag
Definition: blocktools.h:73
unsigned coord_t
Definition: blocktools.h:62
size_t sx
Definition: blocktools.h:174
double * inbuff
Definition: blocktools.h:168
unsigned char destbuffid
Definition: blocktools.h:75
std::pair< neighbourID_t, int > neighpair
Definition: blocktools.h:64
size_t sx
Definition: blocktools.h:281
neighbourID_t getNID(coord_t x, coord_t y, coord_t z) const
Definition: blocktools.cpp:28
const unsigned int dpsize
Definition: blocktools.h:183
size_t inset
Definition: blocktools.h:177
Definition: blocktools.h:225
double * outbuff
Definition: blocktools.h:276
size_t ymidlen
Definition: blocktools.h:285
Definition: blocktools.h:205
std::vector< neighpair > neighbourvector
Definition: blocktools.h:65
unsigned char getSrcBuffID(unsigned char destx, unsigned char desty, unsigned char destz, bool deltax, bool deltay, bool deltaz)
Definition: blocktools.cpp:430
int getTag(unsigned char sourcex, unsigned char sourcey, unsigned char sourcez, unsigned char targetx, unsigned char targety, unsigned char targetz)
Definition: blocktools.cpp:412
void generateOutNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec &v)
Definition: blocktools.cpp:82
Definition: blocktools.h:81
double * outbuff
Definition: blocktools.h:169
int getTag2(unsigned char sourcex, unsigned char sourcey, unsigned char targetx, unsigned char targety)
Definition: blocktools2.cpp:364
coord_t zmax
Definition: blocktools.h:100
neighbourID_t sourceID
Definition: blocktools.h:71
const unsigned int dpsize
Definition: blocktools.h:288
double * inbuff
Definition: blocktools.h:275
size_t ymidlen
Definition: blocktools.h:179
neighbourID_t destID
Definition: blocktools.h:72
BlockGrid(coord_t maxx, coord_t maxy, coord_t maxz)
Definition: blocktools.cpp:24
coord_t xmax
Definition: blocktools.h:220
unsigned char srcbuffid
Definition: blocktools.h:74
std::vector< message > messvec
Definition: blocktools.h:78
size_t xmidlen
Definition: blocktools.h:178
size_t sy
Definition: blocktools.h:282
size_t xmidlen
Definition: blocktools.h:284
unsigned char getSrcBuffID2(unsigned char destx, unsigned char desty, bool deltax, bool deltay)
Definition: blocktools2.cpp:380
size_t sy
Definition: blocktools.h:175
size_t inset
Definition: blocktools.h:283
coord_t ymax
Definition: blocktools.h:99
coord_t xmax
Definition: blocktools.h:98
size_t zmidlen
Definition: blocktools.h:180
Definition: blocktools.h:118
coord_t ymax
Definition: blocktools.h:221
size_t sz
Definition: blocktools.h:176
int neighbourID_t
Definition: blocktools.h:61