3 #ifndef DUNE_GRID_YASPGRID_TORUS_HH
4 #define DUNE_GRID_YASPGRID_TORUS_HH
15 #include <dune/common/array.hh>
16 #include <dune/common/binaryfunctions.hh>
41 template<
class CollectiveCommunication,
int d>
74 : _comm(comm), _tag(tag)
81 for (
int i=0; i<d; i++)
88 if (inc != _comm.size())
89 DUNE_THROW(Dune::Exception,
"Communicator size and result of the given load balancer do not match!");
126 CollectiveCommunication
comm ()
const
140 for (
int i=d-1; i>=0; i--)
141 if (c[i]<0 || c[i]>=_dims[i])
return false;
149 rank = rank%_comm.size();
150 for (
int i=d-1; i>=0; i--)
152 coord[i] = rank/_increment[i];
153 rank = rank%_increment[i];
161 for (
int i=0; i<d; i++) coord[i] = coord[i]%_dims[i];
163 for (
int i=0; i<d; i++) rank += coord[i]*_increment[i];
171 coord[dir] = (coord[dir]+_dims[dir]+cnt)%_dims[dir];
182 for (
int i=0; i<d; i++)
184 if (coord[i]%2==1) c += power;
189 for (
int i=0; i<d; i++)
191 if (_dims[i]>1 && coord[i]==_dims[i]-1) c += power;
208 for (
int i=0; i<d; ++i)
218 for (
int i=0; i<d; i++)
223 if (coord[i]==0 && periodic[i]==
false)
return false;
228 if (coord[i]==_dims[i]-1 && periodic[i]==
false)
return false;
241 double partition (
int rank, iTupel origin_in, iTupel size_in, iTupel& origin_out, iTupel& size_out)
const
248 for (
int i=0; i<d; i++)
251 int m = size_in[i]/_dims[i];
252 int r = size_in[i]%_dims[i];
256 if (coord[i]<_dims[i]-r)
258 origin_out[i] = origin_in[i] + coord[i]*m;
264 origin_out[i] = origin_in[i] + (_dims[i]-r)*m + (coord[i]-(_dims[i]-r))*(m+1);
269 return maxsize/(sz/_comm.size());
308 iTupel
delta=i->delta;
309 for (
int j=0; j<d; ++j)
335 typename std::deque<CommPartner>::const_iterator i;
341 return ProcListIterator(_sendlist.begin());
347 return ProcListIterator(_sendlist.end());
353 return ProcListIterator(_recvlist.begin());
359 return ProcListIterator(_recvlist.end());
367 task.buffer = buffer;
369 if (rank!=_comm.rank())
370 _sendrequests.push_back(task);
372 _localsendrequests.push_back(task);
380 task.buffer = buffer;
382 if (rank!=_comm.rank())
383 _recvrequests.push_back(task);
385 _localrecvrequests.push_back(task);
392 if (_localsendrequests.size()!=_localrecvrequests.size())
394 std::cout <<
"[" <<
rank() <<
"]: ERROR: local sends/receives do not match in exchange!" << std::endl;
397 for (
unsigned int i=0; i<_localsendrequests.size(); i++)
399 if (_localsendrequests[i].size!=_localrecvrequests[i].size)
401 std::cout <<
"[" <<
rank() <<
"]: ERROR: size in local sends/receive does not match in exchange!" << std::endl;
404 memcpy(_localrecvrequests[i].buffer,_localsendrequests[i].buffer,_localsendrequests[i].size);
406 _localsendrequests.clear();
407 _localrecvrequests.clear();
415 for (
unsigned int i=0; i<_sendrequests.size(); i++)
420 MPI_Isend(_sendrequests[i].buffer, _sendrequests[i].size, MPI_BYTE,
421 _sendrequests[i].
rank, _tag, _comm, &(_sendrequests[i].request));
422 _sendrequests[i].flag =
false;
427 for (
unsigned int i=0; i<_recvrequests.size(); i++)
432 MPI_Irecv(_recvrequests[i].buffer, _recvrequests[i].size, MPI_BYTE,
433 _recvrequests[i].
rank, _tag, _comm, &(_recvrequests[i].request));
434 _recvrequests[i].flag =
false;
441 for (
unsigned int i=0; i<_sendrequests.size(); i++)
442 if (!_sendrequests[i].flag)
445 MPI_Test( &(_sendrequests[i].request), &(_sendrequests[i].flag), &status);
446 if (_sendrequests[i].flag)
457 for (
unsigned int i=0; i<_recvrequests.size(); i++)
458 if (!_recvrequests[i].flag)
461 MPI_Test( &(_recvrequests[i].request), &(_recvrequests[i].flag), &status);
462 if (_recvrequests[i].flag)
472 _sendrequests.clear();
473 _recvrequests.clear();
481 _comm.template allreduce<Dune::Max<double>,
double>(&x, &res, 1);
488 s <<
"[" <<
rank() <<
"]: Torus " <<
procs() <<
" processor(s) arranged as " <<
dims() << std::endl;
491 s <<
"[" <<
rank() <<
"]: send to "
492 <<
"rank=" << i.rank()
493 <<
" index=" << i.index()
494 <<
" delta=" << i.delta() <<
" dist=" << i.distance() << std::endl;
498 s <<
"[" <<
rank() <<
"]: recv from "
499 <<
"rank=" << i.rank()
500 <<
" index=" << i.index()
501 <<
" delta=" << i.delta() <<
" dist=" << i.distance() << std::endl;
513 std::fill(delta.begin(), delta.end(), -1);
522 for (
int i=0; i<d; i++)
523 nb[i] = ( me[i]+_dims[i]+delta[i] ) % _dims[i];
529 for (
int i=0; i<d; i++)
535 _recvlist.push_back(cp);
536 cp.index = last-index;
537 _sendlist.push_front(cp);
544 for (
int i=0; i<d; i++)
559 CollectiveCommunication _comm;
564 std::deque<CommPartner> _sendlist;
565 std::deque<CommPartner> _recvlist;
567 mutable std::vector<CommTask> _sendrequests;
568 mutable std::vector<CommTask> _recvrequests;
569 mutable std::vector<CommTask> _localsendrequests;
570 mutable std::vector<CommTask> _localrecvrequests;
575 template <
class CollectiveCommunication,
int d>
576 inline std::ostream& operator<< (std::ostream& s, const Torus<CollectiveCommunication, d> & t)
iTupel rank_to_coord(int rank) const
map rank to coordinate in torus using lexicographic ordering
Definition: torus.hh:146
a base class for the yaspgrid partitioning strategy The name might be irritating. It will probably ch...
Definition: partitioning.hh:23
int rank() const
return own rank
Definition: torus.hh:96
int neighbors() const
return the number of neighbors, which is
Definition: torus.hh:205
static double dist(const double *x, const double *y)
Definition: ghmesh.cc:149
virtual void loadbalance(const iTupel &, int, iTupel &) const =0
void exchange() const
exchange messages stored in request buffers; clear request buffers afterwards
Definition: torus.hh:389
bool inside(iTupel c) const
return true if coordinate is inside torus
Definition: torus.hh:138
iTupel delta() const
return distance vector
Definition: torus.hh:293
int dims(int i) const
return dimensions of torus in direction i
Definition: torus.hh:120
Include standard header files.
Definition: agrid.hh:59
bool operator!=(const ProcListIterator &iter)
Return true when two iterators do not point to same member.
Definition: torus.hh:322
int color(const iTupel &coord) const
assign color to given coordinate
Definition: torus.hh:176
Torus()
constructor making uninitialized object
Definition: torus.hh:69
int tag() const
return tag used by torus
Definition: torus.hh:132
ProcListIterator sendend() const
end of send list
Definition: torus.hh:345
bool operator==(const ProcListIterator &iter)
Return true when two iterators point to same member.
Definition: torus.hh:315
int distance() const
return 1-norm of distance vector
Definition: torus.hh:305
ProcListIterator sendbegin() const
first process in send list
Definition: torus.hh:339
int index() const
return index in proclist
Definition: torus.hh:299
ProcListIterator recvbegin() const
first process in receive list
Definition: torus.hh:351
bool is_neighbor(iTupel delta, std::bitset< d > periodic) const
return true if neighbor with given delta is a neighbor under the given periodicity ...
Definition: torus.hh:214
This file provides tools to partition YaspGrids. If you want to write your own partitioner, inherit from YLoadBalance and implement the loadbalance() method. You can also browse this file for already available useful partitioners, like YaspFixedSizePartitioner.
void recv(int rank, void *buffer, int size) const
store a receive request; buffers are received in order; handles also local requests with memcpy ...
Definition: torus.hh:376
ProcListIterator & operator++()
Increment iterator to next cell.
Definition: torus.hh:328
double partition(int rank, iTupel origin_in, iTupel size_in, iTupel &origin_out, iTupel &size_out) const
partition the given grid onto the torus and return the piece of the process with given rank; returns ...
Definition: torus.hh:241
int procs() const
return number of processes
Definition: torus.hh:108
ProcListIterator(typename std::deque< CommPartner >::const_iterator iter)
make an iterator
Definition: torus.hh:281
const iTupel & dims() const
return dimensions of torus
Definition: torus.hh:114
double global_max(double x) const
global max
Definition: torus.hh:478
void abs(const DofVectorPointer< int > &dofVector)
Definition: dofvector.hh:326
Dune::array< int, d > iTupel
type used to pass tupels in and out
Definition: torus.hh:45
ProcListIterator recvend() const
last process in receive list
Definition: torus.hh:357
Torus(CollectiveCommunication comm, int tag, iTupel size, const YLoadBalance< d > *lb)
make partitioner from communicator and coarse mesh size
Definition: torus.hh:73
int rank() const
return rank of neighboring process
Definition: torus.hh:287
void print(std::ostream &s) const
print contents of torus object
Definition: torus.hh:486
iTupel coord() const
return own coordinates
Definition: torus.hh:102
int rank_relative(int rank, int dir, int cnt) const
return rank of process where its coordinate in direction dir has offset cnt (handles periodic case) ...
Definition: torus.hh:168
int coord_to_rank(iTupel coord) const
map coordinate in torus to rank using lexicographic ordering
Definition: torus.hh:159
CollectiveCommunication comm() const
return communicator
Definition: torus.hh:126
int color(int rank) const
assign color to given rank
Definition: torus.hh:199
void send(int rank, void *buffer, int size) const
store a send request; buffers are sent in order; handles also local requests with memcpy ...
Definition: torus.hh:363