dune-common  2.4
communicator.hh
Go to the documentation of this file.
1 // -*- tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 // vi: set et ts=4 sw=2 sts=2:
3 #ifndef DUNE_COMMUNICATOR
4 #define DUNE_COMMUNICATOR
5 
6 #include "remoteindices.hh"
7 #include "interface.hh"
11 
12 #if HAVE_MPI
13 // MPI header
14 #include <mpi.h>
15 
16 namespace Dune
17 {
101  struct SizeOne
102  {};
103 
110  {};
111 
112 
118  template<class V>
119  struct CommPolicy
120  {
132  typedef V Type;
133 
139  typedef typename V::value_type IndexedType;
140 
146 
155  static const void* getAddress(const V& v, int index);
156 
162  static int getSize(const V&, int index);
163  };
164 
165  template<class K, int n> class FieldVector;
166 
167  template<class B, class A> class VariableBlockVector;
168 
169  template<class K, class A, int n>
171  {
173 
174  typedef typename Type::B IndexedType;
175 
177 
178  static const void* getAddress(const Type& v, int i);
179 
180  static int getSize(const Type& v, int i);
181  };
182 
187  {};
188 
192  template<class T>
194  {
196 
197  static const IndexedType& gather(const T& vec, std::size_t i);
198 
199  static void scatter(T& vec, const IndexedType& v, std::size_t i);
200 
201  };
202 
214  template<typename T>
215  class DatatypeCommunicator : public InterfaceBuilder
216  {
217  public:
218 
222  typedef T ParallelIndexSet;
223 
228 
232  typedef typename RemoteIndices::GlobalIndex GlobalIndex;
233 
237  typedef typename RemoteIndices::Attribute Attribute;
238 
242  typedef typename RemoteIndices::LocalIndex LocalIndex;
243 
247  DatatypeCommunicator();
248 
252  ~DatatypeCommunicator();
253 
280  template<class T1, class T2, class V>
281  void build(const RemoteIndices& remoteIndices, const T1& sourceFlags, V& sendData, const T2& destFlags, V& receiveData);
282 
286  void forward();
287 
291  void backward();
292 
296  void free();
297  private:
298  enum {
302  commTag_ = 234
303  };
304 
308  const RemoteIndices* remoteIndices_;
309 
310  typedef std::map<int,std::pair<MPI_Datatype,MPI_Datatype> >
311  MessageTypeMap;
312 
316  MessageTypeMap messageTypes;
317 
321  void* data_;
322 
323  MPI_Request* requests_[2];
324 
328  bool created_;
329 
333  template<class V, bool FORWARD>
334  void createRequests(V& sendData, V& receiveData);
335 
339  template<class T1, class T2, class V, bool send>
340  void createDataTypes(const T1& source, const T2& destination, V& data);
341 
345  void sendRecv(MPI_Request* req);
346 
350  struct IndexedTypeInformation
351  {
357  void build(int i)
358  {
359  length = new int[i];
360  displ = new MPI_Aint[i];
361  size = i;
362  }
363 
367  void free()
368  {
369  delete[] length;
370  delete[] displ;
371  }
373  int* length;
375  MPI_Aint* displ;
381  int elements;
385  int size;
386  };
387 
393  template<class V>
394  struct MPIDatatypeInformation
395  {
400  MPIDatatypeInformation(const V& data) : data_(data)
401  {}
402 
408  void reserve(int proc, int size)
409  {
410  information_[proc].build(size);
411  }
418  void add(int proc, int local)
419  {
420  IndexedTypeInformation& info=information_[proc];
421  assert((info.elements)<info.size);
422  MPI_Address( const_cast<void*>(CommPolicy<V>::getAddress(data_, local)),
423  info.displ+info.elements);
424  info.length[info.elements]=CommPolicy<V>::getSize(data_, local);
425  info.elements++;
426  }
427 
432  std::map<int,IndexedTypeInformation> information_;
436  const V& data_;
437 
438  };
439 
440  };
441 
452  {
453 
454  public:
459 
466  template<class Data, class Interface>
467  typename enable_if<is_same<SizeOne,typename CommPolicy<Data>::IndexedTypeFlag>::value, void>::type
468  build(const Interface& interface);
469 
477  template<class Data, class Interface>
478  void build(const Data& source, const Data& target, const Interface& interface);
479 
508  template<class GatherScatter, class Data>
509  void forward(const Data& source, Data& dest);
510 
539  template<class GatherScatter, class Data>
540  void backward(Data& source, const Data& dest);
541 
567  template<class GatherScatter, class Data>
568  void forward(Data& data);
569 
595  template<class GatherScatter, class Data>
596  void backward(Data& data);
597 
601  void free();
602 
607 
608  private:
609 
613  typedef std::map<int,std::pair<InterfaceInformation,InterfaceInformation> >
614  InterfaceMap;
615 
616 
620  template<class Data, typename IndexedTypeFlag>
621  struct MessageSizeCalculator
622  {};
623 
628  template<class Data>
629  struct MessageSizeCalculator<Data,SizeOne>
630  {
637  inline int operator()(const InterfaceInformation& info) const;
646  inline int operator()(const Data& data, const InterfaceInformation& info) const;
647  };
648 
653  template<class Data>
654  struct MessageSizeCalculator<Data,VariableSize>
655  {
664  inline int operator()(const Data& data, const InterfaceInformation& info) const;
665  };
666 
670  template<class Data, class GatherScatter, bool send, typename IndexedTypeFlag>
671  struct MessageGatherer
672  {};
673 
678  template<class Data, class GatherScatter, bool send>
679  struct MessageGatherer<Data,GatherScatter,send,SizeOne>
680  {
682  typedef typename CommPolicy<Data>::IndexedType Type;
683 
688  typedef GatherScatter Gatherer;
689 
690  enum {
696  forward=send
697  };
698 
706  inline void operator()(const InterfaceMap& interface, const Data& data, Type* buffer, size_t bufferSize) const;
707  };
708 
713  template<class Data, class GatherScatter, bool send>
714  struct MessageGatherer<Data,GatherScatter,send,VariableSize>
715  {
717  typedef typename CommPolicy<Data>::IndexedType Type;
718 
723  typedef GatherScatter Gatherer;
724 
725  enum {
731  forward=send
732  };
733 
741  inline void operator()(const InterfaceMap& interface, const Data& data, Type* buffer, size_t bufferSize) const;
742  };
743 
747  template<class Data, class GatherScatter, bool send, typename IndexedTypeFlag>
748  struct MessageScatterer
749  {};
750 
755  template<class Data, class GatherScatter, bool send>
756  struct MessageScatterer<Data,GatherScatter,send,SizeOne>
757  {
759  typedef typename CommPolicy<Data>::IndexedType Type;
760 
765  typedef GatherScatter Scatterer;
766 
767  enum {
773  forward=send
774  };
775 
783  inline void operator()(const InterfaceMap& interface, Data& data, Type* buffer, const int& proc) const;
784  };
789  template<class Data, class GatherScatter, bool send>
790  struct MessageScatterer<Data,GatherScatter,send,VariableSize>
791  {
793  typedef typename CommPolicy<Data>::IndexedType Type;
794 
799  typedef GatherScatter Scatterer;
800 
801  enum {
807  forward=send
808  };
809 
817  inline void operator()(const InterfaceMap& interface, Data& data, Type* buffer, const int& proc) const;
818  };
819 
823  struct MessageInformation
824  {
826  MessageInformation()
827  : start_(0), size_(0)
828  {}
829 
837  MessageInformation(size_t start, size_t size)
838  : start_(start), size_(size)
839  {}
843  size_t start_;
847  size_t size_;
848  };
849 
856  typedef std::map<int,std::pair<MessageInformation,MessageInformation> >
857  InformationMap;
861  InformationMap messageInformation_;
865  char* buffers_[2];
869  size_t bufferSize_[2];
870 
871  enum {
875  commTag_
876  };
877 
881  std::map<int,std::pair<InterfaceInformation,InterfaceInformation> > interfaces_;
882 
883  MPI_Comm communicator_;
884 
888  template<class GatherScatter, bool FORWARD, class Data>
889  void sendRecv(const Data& source, Data& target);
890 
891  };
892 
893 #ifndef DOXYGEN
894 
895  template<class V>
896  inline const void* CommPolicy<V>::getAddress(const V& v, int index)
897  {
898  return &(v[index]);
899  }
900 
901  template<class V>
902  inline int CommPolicy<V>::getSize(const V& v, int index)
903  {
905  DUNE_UNUSED_PARAMETER(index);
906  return 1;
907  }
908 
909  template<class K, class A, int n>
910  inline const void* CommPolicy<VariableBlockVector<FieldVector<K, n>, A> >::getAddress(const Type& v, int index)
911  {
912  return &(v[index][0]);
913  }
914 
915  template<class K, class A, int n>
916  inline int CommPolicy<VariableBlockVector<FieldVector<K, n>, A> >::getSize(const Type& v, int index)
917  {
918  return v[index].getsize();
919  }
920 
921  template<class T>
922  inline const typename CopyGatherScatter<T>::IndexedType& CopyGatherScatter<T>::gather(const T & vec, std::size_t i)
923  {
924  return vec[i];
925  }
926 
927  template<class T>
928  inline void CopyGatherScatter<T>::scatter(T& vec, const IndexedType& v, std::size_t i)
929  {
930  vec[i]=v;
931  }
932 
933  template<typename T>
934  DatatypeCommunicator<T>::DatatypeCommunicator()
935  : remoteIndices_(0), created_(false)
936  {
937  requests_[0]=0;
938  requests_[1]=0;
939  }
940 
941 
942 
943  template<typename T>
944  DatatypeCommunicator<T>::~DatatypeCommunicator()
945  {
946  free();
947  }
948 
949  template<typename T>
950  template<class T1, class T2, class V>
951  inline void DatatypeCommunicator<T>::build(const RemoteIndices& remoteIndices,
952  const T1& source, V& sendData,
953  const T2& destination, V& receiveData)
954  {
955  remoteIndices_ = &remoteIndices;
956  free();
957  createDataTypes<T1,T2,V,false>(source,destination, receiveData);
958  createDataTypes<T1,T2,V,true>(source,destination, sendData);
959  createRequests<V,true>(sendData, receiveData);
960  createRequests<V,false>(receiveData, sendData);
961  created_=true;
962  }
963 
964  template<typename T>
965  void DatatypeCommunicator<T>::free()
966  {
967  if(created_) {
968  delete[] requests_[0];
969  delete[] requests_[1];
970  typedef MessageTypeMap::iterator iterator;
971  typedef MessageTypeMap::const_iterator const_iterator;
972 
973  const const_iterator end=messageTypes.end();
974 
975  for(iterator process = messageTypes.begin(); process != end; ++process) {
976  MPI_Datatype *type = &(process->second.first);
977  int finalized=0;
978 #if MPI_2
979  MPI_Finalized(&finalized);
980 #endif
981  if(*type!=MPI_DATATYPE_NULL && !finalized)
982  MPI_Type_free(type);
983  type = &(process->second.second);
984  if(*type!=MPI_DATATYPE_NULL && !finalized)
985  MPI_Type_free(type);
986  }
987  messageTypes.clear();
988  created_=false;
989  }
990 
991  }
992 
993  template<typename T>
994  template<class T1, class T2, class V, bool send>
995  void DatatypeCommunicator<T>::createDataTypes(const T1& sourceFlags, const T2& destFlags, V& data)
996  {
997 
998  MPIDatatypeInformation<V> dataInfo(data);
999  this->template buildInterface<RemoteIndices,T1,T2,MPIDatatypeInformation<V>,send>(*remoteIndices_,sourceFlags, destFlags, dataInfo);
1000 
1001  typedef typename RemoteIndices::RemoteIndexMap::const_iterator const_iterator;
1002  const const_iterator end=this->remoteIndices_->end();
1003 
1004  // Allocate MPI_Datatypes and deallocate memory for the type construction.
1005  for(const_iterator process=this->remoteIndices_->begin(); process != end; ++process) {
1006  IndexedTypeInformation& info=dataInfo.information_[process->first];
1007  // Shift the displacement
1008  MPI_Aint base;
1009  MPI_Address(const_cast<void *>(CommPolicy<V>::getAddress(data, 0)), &base);
1010 
1011  for(int i=0; i< info.elements; i++) {
1012  info.displ[i]-=base;
1013  }
1014 
1015  // Create data type
1016  MPI_Datatype* type = &( send ? messageTypes[process->first].first : messageTypes[process->first].second);
1017  MPI_Type_hindexed(info.elements, info.length, info.displ,
1018  MPITraits<typename CommPolicy<V>::IndexedType>::getType(),
1019  type);
1020  MPI_Type_commit(type);
1021  // Deallocate memory
1022  info.free();
1023  }
1024  }
1025 
1026  template<typename T>
1027  template<class V, bool createForward>
1028  void DatatypeCommunicator<T>::createRequests(V& sendData, V& receiveData)
1029  {
1030  typedef std::map<int,std::pair<MPI_Datatype,MPI_Datatype> >::const_iterator MapIterator;
1031  int rank;
1032  static int index = createForward ? 1 : 0;
1033  int noMessages = messageTypes.size();
1034  // allocate request handles
1035  requests_[index] = new MPI_Request[2*noMessages];
1036  const MapIterator end = messageTypes.end();
1037  int request=0;
1038  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
1039 
1040  // Set up the requests for receiving first
1041  for(MapIterator process = messageTypes.begin(); process != end;
1042  ++process, ++request) {
1043  MPI_Datatype type = createForward ? process->second.second : process->second.first;
1044  void* address = const_cast<void*>(CommPolicy<V>::getAddress(receiveData,0));
1045  MPI_Recv_init(address, 1, type, process->first, commTag_, this->remoteIndices_->communicator(), requests_[index]+request);
1046  }
1047 
1048  // And now the send requests
1049 
1050  for(MapIterator process = messageTypes.begin(); process != end;
1051  ++process, ++request) {
1052  MPI_Datatype type = createForward ? process->second.first : process->second.second;
1053  void* address = const_cast<void*>(CommPolicy<V>::getAddress(sendData, 0));
1054  MPI_Ssend_init(address, 1, type, process->first, commTag_, this->remoteIndices_->communicator(), requests_[index]+request);
1055  }
1056  }
1057 
1058  template<typename T>
1059  void DatatypeCommunicator<T>::forward()
1060  {
1061  sendRecv(requests_[1]);
1062  }
1063 
1064  template<typename T>
1065  void DatatypeCommunicator<T>::backward()
1066  {
1067  sendRecv(requests_[0]);
1068  }
1069 
1070  template<typename T>
1071  void DatatypeCommunicator<T>::sendRecv(MPI_Request* requests)
1072  {
1073  int noMessages = messageTypes.size();
1074  // Start the receive calls first
1075  MPI_Startall(noMessages, requests);
1076  // Now the send calls
1077  MPI_Startall(noMessages, requests+noMessages);
1078 
1079  // Wait for completion of the communication send first then receive
1080  MPI_Status* status=new MPI_Status[2*noMessages];
1081  for(int i=0; i<2*noMessages; i++)
1082  status[i].MPI_ERROR=MPI_SUCCESS;
1083 
1084  int send = MPI_Waitall(noMessages, requests+noMessages, status+noMessages);
1085  int receive = MPI_Waitall(noMessages, requests, status);
1086 
1087  // Error checks
1088  int success=1, globalSuccess=0;
1089  if(send==MPI_ERR_IN_STATUS) {
1090  int rank;
1091  MPI_Comm_rank(this->remoteIndices_->communicator(), &rank);
1092  std::cerr<<rank<<": Error in sending :"<<std::endl;
1093  // Search for the error
1094  for(int i=noMessages; i< 2*noMessages; i++)
1095  if(status[i].MPI_ERROR!=MPI_SUCCESS) {
1096  char message[300];
1097  int messageLength;
1098  MPI_Error_string(status[i].MPI_ERROR, message, &messageLength);
1099  std::cerr<<" source="<<status[i].MPI_SOURCE<<" message: ";
1100  for(int i=0; i< messageLength; i++)
1101  std::cout<<message[i];
1102  }
1103  std::cerr<<std::endl;
1104  success=0;
1105  }
1106 
1107  if(receive==MPI_ERR_IN_STATUS) {
1108  int rank;
1109  MPI_Comm_rank(this->remoteIndices_->communicator(), &rank);
1110  std::cerr<<rank<<": Error in receiving!"<<std::endl;
1111  // Search for the error
1112  for(int i=0; i< noMessages; i++)
1113  if(status[i].MPI_ERROR!=MPI_SUCCESS) {
1114  char message[300];
1115  int messageLength;
1116  MPI_Error_string(status[i].MPI_ERROR, message, &messageLength);
1117  std::cerr<<" source="<<status[i].MPI_SOURCE<<" message: ";
1118  for(int i=0; i< messageLength; i++)
1119  std::cerr<<message[i];
1120  }
1121  std::cerr<<std::endl;
1122  success=0;
1123  }
1124 
1125  MPI_Allreduce(&success, &globalSuccess, 1, MPI_INT, MPI_MIN, this->remoteIndices_->communicator());
1126 
1127  delete[] status;
1128 
1129  if(!globalSuccess)
1130  DUNE_THROW(CommunicationError, "A communication error occurred!");
1131 
1132  }
1133 
1135  {
1136  buffers_[0]=0;
1137  buffers_[1]=0;
1138  bufferSize_[0]=0;
1139  bufferSize_[1]=0;
1140  }
1141 
1142  template<class Data, class Interface>
1143  typename enable_if<is_same<SizeOne, typename CommPolicy<Data>::IndexedTypeFlag>::value, void>::type
1144  BufferedCommunicator::build(const Interface& interface)
1145  {
1146  interfaces_=interface.interfaces();
1147  communicator_=interface.communicator();
1148  typedef typename std::map<int,std::pair<InterfaceInformation,InterfaceInformation> >
1149  ::const_iterator const_iterator;
1150  typedef typename CommPolicy<Data>::IndexedTypeFlag Flag;
1151  const const_iterator end = interfaces_.end();
1152  int lrank;
1153  MPI_Comm_rank(communicator_, &lrank);
1154 
1155  bufferSize_[0]=0;
1156  bufferSize_[1]=0;
1157 
1158  for(const_iterator interfacePair = interfaces_.begin();
1159  interfacePair != end; ++interfacePair) {
1160  int noSend = MessageSizeCalculator<Data,Flag>() (interfacePair->second.first);
1161  int noRecv = MessageSizeCalculator<Data,Flag>() (interfacePair->second.second);
1162  if (noSend + noRecv > 0)
1163  messageInformation_.insert(std::make_pair(interfacePair->first,
1164  std::make_pair(MessageInformation(bufferSize_[0],
1165  noSend*sizeof(typename CommPolicy<Data>::IndexedType)),
1166  MessageInformation(bufferSize_[1],
1167  noRecv*sizeof(typename CommPolicy<Data>::IndexedType)))));
1168  bufferSize_[0] += noSend;
1169  bufferSize_[1] += noRecv;
1170  }
1171 
1172  // allocate the buffers
1173  bufferSize_[0] *= sizeof(typename CommPolicy<Data>::IndexedType);
1174  bufferSize_[1] *= sizeof(typename CommPolicy<Data>::IndexedType);
1175 
1176  buffers_[0] = new char[bufferSize_[0]];
1177  buffers_[1] = new char[bufferSize_[1]];
1178  }
1179 
1180  template<class Data, class Interface>
1181  void BufferedCommunicator::build(const Data& source, const Data& dest, const Interface& interface)
1182  {
1183 
1184  interfaces_=interface.interfaces();
1185  communicator_=interface.communicator();
1186  typedef typename std::map<int,std::pair<InterfaceInformation,InterfaceInformation> >
1187  ::const_iterator const_iterator;
1188  typedef typename CommPolicy<Data>::IndexedTypeFlag Flag;
1189  const const_iterator end = interfaces_.end();
1190 
1191  bufferSize_[0]=0;
1192  bufferSize_[1]=0;
1193 
1194  for(const_iterator interfacePair = interfaces_.begin();
1195  interfacePair != end; ++interfacePair) {
1196  int noSend = MessageSizeCalculator<Data,Flag>() (source, interfacePair->second.first);
1197  int noRecv = MessageSizeCalculator<Data,Flag>() (dest, interfacePair->second.second);
1198  if (noSend + noRecv > 0)
1199  messageInformation_.insert(std::make_pair(interfacePair->first,
1200  std::make_pair(MessageInformation(bufferSize_[0],
1201  noSend*sizeof(typename CommPolicy<Data>::IndexedType)),
1202  MessageInformation(bufferSize_[1],
1203  noRecv*sizeof(typename CommPolicy<Data>::IndexedType)))));
1204  bufferSize_[0] += noSend;
1205  bufferSize_[1] += noRecv;
1206  }
1207 
1208  bufferSize_[0] *= sizeof(typename CommPolicy<Data>::IndexedType);
1209  bufferSize_[1] *= sizeof(typename CommPolicy<Data>::IndexedType);
1210  // allocate the buffers
1211  buffers_[0] = new char[bufferSize_[0]];
1212  buffers_[1] = new char[bufferSize_[1]];
1213  }
1214 
1215  inline void BufferedCommunicator::free()
1216  {
1217  messageInformation_.clear();
1218  if(buffers_[0])
1219  delete[] buffers_[0];
1220 
1221  if(buffers_[1])
1222  delete[] buffers_[1];
1223  buffers_[0]=buffers_[1]=0;
1224  }
1225 
1227  {
1228  free();
1229  }
1230 
1231  template<class Data>
1232  inline int BufferedCommunicator::MessageSizeCalculator<Data,SizeOne>::operator()
1233  (const InterfaceInformation& info) const
1234  {
1235  return info.size();
1236  }
1237 
1238 
1239  template<class Data>
1240  inline int BufferedCommunicator::MessageSizeCalculator<Data,SizeOne>::operator()
1241  (const Data&, const InterfaceInformation& info) const
1242  {
1243  return operator()(info);
1244  }
1245 
1246 
1247  template<class Data>
1248  inline int BufferedCommunicator::MessageSizeCalculator<Data, VariableSize>::operator()
1249  (const Data& data, const InterfaceInformation& info) const
1250  {
1251  int entries=0;
1252 
1253  for(size_t i=0; i < info.size(); i++)
1254  entries += CommPolicy<Data>::getSize(data,info[i]);
1255 
1256  return entries;
1257  }
1258 
1259 
1260  template<class Data, class GatherScatter, bool FORWARD>
1261  inline void BufferedCommunicator::MessageGatherer<Data,GatherScatter,FORWARD,VariableSize>::operator()(const InterfaceMap& interfaces,const Data& data, Type* buffer, size_t bufferSize) const
1262  {
1263  DUNE_UNUSED_PARAMETER(bufferSize);
1264  typedef typename InterfaceMap::const_iterator
1265  const_iterator;
1266 
1267  int rank;
1268  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
1269  const const_iterator end = interfaces.end();
1270  size_t index=0;
1271 
1272  for(const_iterator interfacePair = interfaces.begin();
1273  interfacePair != end; ++interfacePair) {
1274  int size = forward ? interfacePair->second.first.size() :
1275  interfacePair->second.second.size();
1276 
1277  for(int i=0; i < size; i++) {
1278  int local = forward ? interfacePair->second.first[i] :
1279  interfacePair->second.second[i];
1280  for(std::size_t j=0; j < CommPolicy<Data>::getSize(data, local); j++, index++) {
1281 
1282 #ifdef DUNE_ISTL_WITH_CHECKING
1283  assert(bufferSize>=(index+1)*sizeof(typename CommPolicy<Data>::IndexedType));
1284 #endif
1285  buffer[index]=GatherScatter::gather(data, local, j);
1286  }
1287 
1288  }
1289  }
1290 
1291  }
1292 
1293 
1294  template<class Data, class GatherScatter, bool FORWARD>
1295  inline void BufferedCommunicator::MessageGatherer<Data,GatherScatter,FORWARD,SizeOne>::operator()(const InterfaceMap& interfaces, const Data& data, Type* buffer, size_t bufferSize) const
1296  {
1297  DUNE_UNUSED_PARAMETER(bufferSize);
1298  typedef typename InterfaceMap::const_iterator
1299  const_iterator;
1300  const const_iterator end = interfaces.end();
1301  size_t index = 0;
1302 
1303  int rank;
1304  MPI_Comm_rank(MPI_COMM_WORLD, &rank);
1305 
1306  for(const_iterator interfacePair = interfaces.begin();
1307  interfacePair != end; ++interfacePair) {
1308  size_t size = FORWARD ? interfacePair->second.first.size() :
1309  interfacePair->second.second.size();
1310 
1311  for(size_t i=0; i < size; i++) {
1312 
1313 #ifdef DUNE_ISTL_WITH_CHECKING
1314  assert(bufferSize>=(index+1)*sizeof(typename CommPolicy<Data>::IndexedType));
1315 #endif
1316 
1317  buffer[index++] = GatherScatter::gather(data, FORWARD ? interfacePair->second.first[i] :
1318  interfacePair->second.second[i]);
1319  }
1320  }
1321 
1322  }
1323 
1324 
1325  template<class Data, class GatherScatter, bool FORWARD>
1326  inline void BufferedCommunicator::MessageScatterer<Data,GatherScatter,FORWARD,VariableSize>::operator()(const InterfaceMap& interfaces, Data& data, Type* buffer, const int& proc) const
1327  {
1328  typedef typename InterfaceMap::value_type::second_type::first_type Information;
1329  const typename InterfaceMap::const_iterator infoPair = interfaces.find(proc);
1330 
1331  assert(infoPair!=interfaces.end());
1332 
1333  const Information& info = FORWARD ? infoPair->second.second :
1334  infoPair->second.first;
1335 
1336  for(size_t i=0, index=0; i < info.size(); i++) {
1337  for(size_t j=0; j < CommPolicy<Data>::getSize(data, info[i]); j++)
1338  GatherScatter::scatter(data, buffer[index++], info[i], j);
1339  }
1340  }
1341 
1342 
1343  template<class Data, class GatherScatter, bool FORWARD>
1344  inline void BufferedCommunicator::MessageScatterer<Data,GatherScatter,FORWARD,SizeOne>::operator()(const InterfaceMap& interfaces, Data& data, Type* buffer, const int& proc) const
1345  {
1346  typedef typename InterfaceMap::value_type::second_type::first_type Information;
1347  const typename InterfaceMap::const_iterator infoPair = interfaces.find(proc);
1348 
1349  assert(infoPair!=interfaces.end());
1350 
1351  const Information& info = FORWARD ? infoPair->second.second :
1352  infoPair->second.first;
1353 
1354  for(size_t i=0; i < info.size(); i++) {
1355  GatherScatter::scatter(data, buffer[i], info[i]);
1356  }
1357  }
1358 
1359 
1360  template<class GatherScatter,class Data>
1361  void BufferedCommunicator::forward(Data& data)
1362  {
1363  this->template sendRecv<GatherScatter,true>(data, data);
1364  }
1365 
1366 
1367  template<class GatherScatter, class Data>
1368  void BufferedCommunicator::backward(Data& data)
1369  {
1370  this->template sendRecv<GatherScatter,false>(data, data);
1371  }
1372 
1373 
1374  template<class GatherScatter, class Data>
1375  void BufferedCommunicator::forward(const Data& source, Data& dest)
1376  {
1377  this->template sendRecv<GatherScatter,true>(source, dest);
1378  }
1379 
1380 
1381  template<class GatherScatter, class Data>
1382  void BufferedCommunicator::backward(Data& source, const Data& dest)
1383  {
1384  this->template sendRecv<GatherScatter,false>(dest, source);
1385  }
1386 
1387 
1388  template<class GatherScatter, bool FORWARD, class Data>
1389  void BufferedCommunicator::sendRecv(const Data& source, Data& dest)
1390  {
1391  int rank, lrank;
1392 
1393  MPI_Comm_rank(MPI_COMM_WORLD,&rank);
1394  MPI_Comm_rank(MPI_COMM_WORLD,&lrank);
1395 
1396  typedef typename CommPolicy<Data>::IndexedType Type;
1397  Type *sendBuffer, *recvBuffer;
1398  size_t sendBufferSize;
1399 #ifndef NDEBUG
1400  size_t recvBufferSize;
1401 #endif
1402 
1403  if(FORWARD) {
1404  sendBuffer = reinterpret_cast<Type*>(buffers_[0]);
1405  sendBufferSize = bufferSize_[0];
1406  recvBuffer = reinterpret_cast<Type*>(buffers_[1]);
1407 #ifndef NDEBUG
1408  recvBufferSize = bufferSize_[1];
1409 #endif
1410  }else{
1411  sendBuffer = reinterpret_cast<Type*>(buffers_[1]);
1412  sendBufferSize = bufferSize_[1];
1413  recvBuffer = reinterpret_cast<Type*>(buffers_[0]);
1414 #ifndef NDEBUG
1415  recvBufferSize = bufferSize_[0];
1416 #endif
1417  }
1418  typedef typename CommPolicy<Data>::IndexedTypeFlag Flag;
1419 
1420  MessageGatherer<Data,GatherScatter,FORWARD,Flag>() (interfaces_, source, sendBuffer, sendBufferSize);
1421 
1422  MPI_Request* sendRequests = new MPI_Request[messageInformation_.size()];
1423  MPI_Request* recvRequests = new MPI_Request[messageInformation_.size()];
1424  /* Number of recvRequests that are not MPI_REQUEST_NULL */
1425  size_t numberOfRealRecvRequests = 0;
1426 
1427  // Setup receive first
1428  typedef typename InformationMap::const_iterator const_iterator;
1429 
1430  const const_iterator end = messageInformation_.end();
1431  size_t i=0;
1432  int* processMap = new int[messageInformation_.size()];
1433 
1434  for(const_iterator info = messageInformation_.begin(); info != end; ++info, ++i) {
1435  processMap[i]=info->first;
1436  if(FORWARD) {
1437  assert(info->second.second.start_*sizeof(typename CommPolicy<Data>::IndexedType)+info->second.second.size_ <= recvBufferSize );
1438  Dune::dvverb<<rank<<": receiving "<<info->second.second.size_<<" from "<<info->first<<std::endl;
1439  if(info->second.second.size_) {
1440  MPI_Irecv(recvBuffer+info->second.second.start_, info->second.second.size_,
1441  MPI_BYTE, info->first, commTag_, communicator_,
1442  recvRequests+i);
1443  numberOfRealRecvRequests += 1;
1444  } else {
1445  // Nothing to receive -> set request to inactive
1446  recvRequests[i]=MPI_REQUEST_NULL;
1447  }
1448  }else{
1449  assert(info->second.first.start_*sizeof(typename CommPolicy<Data>::IndexedType)+info->second.first.size_ <= recvBufferSize );
1450  Dune::dvverb<<rank<<": receiving "<<info->second.first.size_<<" to "<<info->first<<std::endl;
1451  if(info->second.first.size_) {
1452  MPI_Irecv(recvBuffer+info->second.first.start_, info->second.first.size_,
1453  MPI_BYTE, info->first, commTag_, communicator_,
1454  recvRequests+i);
1455  numberOfRealRecvRequests += 1;
1456  } else {
1457  // Nothing to receive -> set request to inactive
1458  recvRequests[i]=MPI_REQUEST_NULL;
1459  }
1460  }
1461  }
1462 
1463  // now the send requests
1464  i=0;
1465  for(const_iterator info = messageInformation_.begin(); info != end; ++info, ++i)
1466  if(FORWARD) {
1467  assert(info->second.second.start_*sizeof(typename CommPolicy<Data>::IndexedType)+info->second.second.size_ <= recvBufferSize );
1468  Dune::dvverb<<rank<<": sending "<<info->second.first.size_<<" to "<<info->first<<std::endl;
1469  assert(info->second.first.start_*sizeof(typename CommPolicy<Data>::IndexedType)+info->second.first.size_ <= sendBufferSize );
1470  if(info->second.first.size_)
1471  MPI_Issend(sendBuffer+info->second.first.start_, info->second.first.size_,
1472  MPI_BYTE, info->first, commTag_, communicator_,
1473  sendRequests+i);
1474  else
1475  // Nothing to send -> set request to inactive
1476  sendRequests[i]=MPI_REQUEST_NULL;
1477  }else{
1478  assert(info->second.second.start_*sizeof(typename CommPolicy<Data>::IndexedType)+info->second.second.size_ <= sendBufferSize );
1479  Dune::dvverb<<rank<<": sending "<<info->second.second.size_<<" to "<<info->first<<std::endl;
1480  if(info->second.second.size_)
1481  MPI_Issend(sendBuffer+info->second.second.start_, info->second.second.size_,
1482  MPI_BYTE, info->first, commTag_, communicator_,
1483  sendRequests+i);
1484  else
1485  // Nothing to send -> set request to inactive
1486  sendRequests[i]=MPI_REQUEST_NULL;
1487  }
1488 
1489  // Wait for completion of receive and immediately start scatter
1490  i=0;
1491  //int success = 1;
1492  int finished = MPI_UNDEFINED;
1493  MPI_Status status; //[messageInformation_.size()];
1494  //MPI_Waitall(messageInformation_.size(), recvRequests, status);
1495 
1496  for(i=0; i< numberOfRealRecvRequests; i++) {
1497  status.MPI_ERROR=MPI_SUCCESS;
1498  MPI_Waitany(messageInformation_.size(), recvRequests, &finished, &status);
1499  assert(finished != MPI_UNDEFINED);
1500 
1501  if(status.MPI_ERROR==MPI_SUCCESS) {
1502  int& proc = processMap[finished];
1503  typename InformationMap::const_iterator infoIter = messageInformation_.find(proc);
1504  assert(infoIter != messageInformation_.end());
1505 
1506  MessageInformation info = (FORWARD) ? infoIter->second.second : infoIter->second.first;
1507  assert(info.start_+info.size_ <= recvBufferSize);
1508 
1509  MessageScatterer<Data,GatherScatter,FORWARD,Flag>() (interfaces_, dest, recvBuffer+info.start_, proc);
1510  }else{
1511  std::cerr<<rank<<": MPI_Error occurred while receiving message from "<<processMap[finished]<<std::endl;
1512  //success=0;
1513  }
1514  }
1515 
1516  MPI_Status recvStatus;
1517 
1518  // Wait for completion of sends
1519  for(i=0; i< messageInformation_.size(); i++)
1520  if(MPI_SUCCESS!=MPI_Wait(sendRequests+i, &recvStatus)) {
1521  std::cerr<<rank<<": MPI_Error occurred while sending message to "<<processMap[finished]<<std::endl;
1522  //success=0;
1523  }
1524  /*
1525  int globalSuccess;
1526  MPI_Allreduce(&success, &globalSuccess, 1, MPI_INT, MPI_MIN, interface_->communicator());
1527 
1528  if(!globalSuccess)
1529  DUNE_THROW(CommunicationError, "A communication error occurred!");
1530  */
1531  delete[] processMap;
1532  delete[] sendRequests;
1533  delete[] recvRequests;
1534 
1535  }
1536 
1537 #endif // DOXYGEN
1538 
1540 }
1541 
1542 #endif
1543 
1544 #endif
void backward(Data &source, const Data &dest)
Communicate in the reverse direction, i.e. send from target to source.
V::value_type IndexedType
The type we get at each index with operator[].
Definition: communicator.hh:139
Definition: communicator.hh:167
A communicator that uses buffers to gather and scatter the data to be send or received.
Definition: communicator.hh:451
SizeOne IndexedTypeFlag
Whether the indexed type has variable size or there is always one value at each index.
Definition: communicator.hh:145
~BufferedCommunicator()
Destructor.
The indices present on remote processes.
Definition: remoteindices.hh:45
Communication interface between remote and local indices.
Definition: interface.hh:206
static int getSize(const V &, int index)
Get the number of primitve elements at that index.
A few common exception classes.
Default exception class for I/O errors.
Definition: exceptions.hh:256
Provides classes for building the communication interface between remote indices. ...
DVVerbType dvverb(std::cout)
stream for very verbose output.
Definition: stdstreams.hh:93
V Type
The type the policy is for.
Definition: communicator.hh:132
static const void * getAddress(const V &v, int index)
Get the address of entry at an index.
std::size_t size_
The size of the buffer.
Definition: variablesizecommunicator.hh:132
Definition: communicator.hh:165
Dune namespace.
Definition: alignment.hh:9
Information describing an interface.
Definition: interface.hh:98
Default policy used for communicating an indexed type.
Definition: communicator.hh:119
void free()
Free the allocated memory (i.e. buffers and message information.
void forward(const Data &source, Data &dest)
Send from source to target.
ParallelIndexSet::LocalIndex LocalIndex
The type of the local index.
Definition: remoteindices.hh:213
ParallelIndexSet::GlobalIndex GlobalIndex
The type of the global index.
Definition: remoteindices.hh:207
Classes describing a distributed indexset.
LocalIndex::Attribute Attribute
The type of the attribute.
Definition: remoteindices.hh:218
DataHandle & data_
Definition: variablesizecommunicator.hh:521
VariableBlockVector< FieldVector< K, n >, A > Type
Definition: communicator.hh:172
#define DUNE_THROW(E, m)
Definition: exceptions.hh:243
#define DUNE_UNUSED_PARAMETER(parm)
A macro to mark intentional unused function parameters with.
Definition: unused.hh:18
GatherScatter default implementation that just copies data.
Definition: communicator.hh:193
Standard Dune debug streams.
enable_if< is_same< SizeOne, typename CommPolicy< Data >::IndexedTypeFlag >::value, void >::type build(const Interface &interface)
Build the buffers and information for the communication process.
BufferedCommunicator()
Constructor.
Base class of all classes representing a communication interface.
Definition: interface.hh:32
Flag for marking indexed data structures where data at each index is of the same size.
Definition: communicator.hh:101
CommPolicy< T >::IndexedType IndexedType
Definition: communicator.hh:195
Traits for type conversions and type information.
Flag for marking indexed data structures where the data at each index may be a variable multiple of a...
Definition: communicator.hh:109
Error thrown if there was a problem with the communication.
Definition: communicator.hh:186
An index present on the local process.
Definition: localindex.hh:32
Manager class for the mapping between local indices and globally unique indices.
Definition: indexset.hh:216