@@ -178,40 +178,40 @@ void Mpi::Init(Grid *grid, std::vector<int> inputMap,
178178 MPI_Send_init (BufferSendX1[faceLeft].data (), bufferSizeX1[faceLeft], realMPI,
179179 procSendX1[faceLeft],thisInstance*1000 +1 ,mygrid->CartComm , &sendRequestX1[faceLeft]);
180180
181- MPI_Recv_init (BufferRecvX1[faceRight].data (), bufferSizeX1[faceLeft], realMPI, procRecvX1[faceRight],
182- thisInstance*1000 +1 ,mygrid->CartComm , &recvRequestX1[faceRight]);
181+ MPI_Recv_init (BufferRecvX1[faceRight].data (), bufferSizeX1[faceLeft], realMPI,
182+ procRecvX1[faceRight], thisInstance*1000 +1 ,mygrid->CartComm , &recvRequestX1[faceRight]);
183183
184184 #if DIMENSIONS >= 2
185- MPI_Send_init (BufferSendX2[faceRight].data (), bufferSizeX2[faceRight], realMPI, procSendX2[faceRight],
186- thisInstance*1000 +10 , mygrid->CartComm , &sendRequestX2[faceRight]);
185+ MPI_Send_init (BufferSendX2[faceRight].data (), bufferSizeX2[faceRight], realMPI,
186+ procSendX2[faceRight], thisInstance*1000 +10 , mygrid->CartComm , &sendRequestX2[faceRight]);
187187
188- MPI_Recv_init (BufferRecvX2[faceLeft].data (), bufferSizeX2[faceRight], realMPI, procRecvX2[faceLeft],
189- thisInstance*1000 +10 , mygrid->CartComm , &recvRequestX2[faceLeft]);
188+ MPI_Recv_init (BufferRecvX2[faceLeft].data (), bufferSizeX2[faceRight], realMPI,
189+ procRecvX2[faceLeft], thisInstance*1000 +10 , mygrid->CartComm , &recvRequestX2[faceLeft]);
190190
191191 // Send to the left
192192 // We receive from procRecv, and we send to procSend
193- MPI_Send_init (BufferSendX2[faceLeft].data (), bufferSizeX2[faceLeft], realMPI, procSendX2[faceLeft],
194- thisInstance*1000 +11 , mygrid->CartComm , &sendRequestX2[faceLeft]);
193+ MPI_Send_init (BufferSendX2[faceLeft].data (), bufferSizeX2[faceLeft], realMPI,
194+ procSendX2[faceLeft], thisInstance*1000 +11 , mygrid->CartComm , &sendRequestX2[faceLeft]);
195195
196- MPI_Recv_init (BufferRecvX2[faceRight].data (), bufferSizeX2[faceLeft], realMPI, procRecvX2[faceRight],
197- thisInstance*1000 +11 , mygrid->CartComm , &recvRequestX2[faceRight]);
196+ MPI_Recv_init (BufferRecvX2[faceRight].data (), bufferSizeX2[faceLeft], realMPI,
197+ procRecvX2[faceRight], thisInstance*1000 +11 , mygrid->CartComm , &recvRequestX2[faceRight]);
198198 #endif
199199
200200 #if DIMENSIONS == 3
201201 // We receive from procRecv, and we send to procSend
202- MPI_Send_init (BufferSendX3[faceRight].data (), bufferSizeX3[faceRight], realMPI, procSendX3[faceRight],
203- thisInstance*1000 +20 , mygrid->CartComm , &sendRequestX3[faceRight]);
202+ MPI_Send_init (BufferSendX3[faceRight].data (), bufferSizeX3[faceRight], realMPI,
203+ procSendX3[faceRight], thisInstance*1000 +20 , mygrid->CartComm , &sendRequestX3[faceRight]);
204204
205- MPI_Recv_init (BufferRecvX3[faceLeft].data (), bufferSizeX3[faceRight], realMPI, procRecvX3[faceLeft],
206- thisInstance*1000 +20 , mygrid->CartComm , &recvRequestX3[faceLeft]);
205+ MPI_Recv_init (BufferRecvX3[faceLeft].data (), bufferSizeX3[faceRight], realMPI,
206+ procRecvX3[faceLeft], thisInstance*1000 +20 , mygrid->CartComm , &recvRequestX3[faceLeft]);
207207
208208 // Send to the left
209209 // We receive from procRecv, and we send to procSend
210- MPI_Send_init (BufferSendX3[faceLeft].data (), bufferSizeX3[faceLeft], realMPI, procSendX3[faceLeft],
211- thisInstance*1000 +21 , mygrid->CartComm , &sendRequestX3[faceLeft]);
210+ MPI_Send_init (BufferSendX3[faceLeft].data (), bufferSizeX3[faceLeft], realMPI,
211+ procSendX3[faceLeft], thisInstance*1000 +21 , mygrid->CartComm , &sendRequestX3[faceLeft]);
212212
213- MPI_Recv_init (BufferRecvX3[faceRight].data (), bufferSizeX3[faceLeft], realMPI, procRecvX3[faceRight],
214- thisInstance*1000 +21 , mygrid->CartComm , &recvRequestX3[faceRight]);
213+ MPI_Recv_init (BufferRecvX3[faceRight].data (), bufferSizeX3[faceLeft], realMPI,
214+ procRecvX3[faceRight], thisInstance*1000 +21 , mygrid->CartComm , &recvRequestX3[faceRight]);
215215 #endif
216216
217217#endif // MPI_Persistent
@@ -362,20 +362,20 @@ void Mpi::ExchangeX1(IdefixArray4D<real> Vc, IdefixArray4D<real> Vs) {
362362
363363 // We receive from procRecv, and we send to procSend
364364
365- MPI_Isend (BufferSendX1[faceRight].data (), bufferSizeX1[faceRight], realMPI, procSendX1[faceRight], 100 ,
366- mygrid->CartComm , &sendRequest[0 ]);
365+ MPI_Isend (BufferSendX1[faceRight].data (), bufferSizeX1[faceRight], realMPI,
366+ procSendX1[faceRight], 100 , mygrid->CartComm , &sendRequest[0 ]);
367367
368- MPI_Irecv (BufferRecvX1[faceLeft].data (), bufferSizeX1[faceRight], realMPI, procRecvX1[faceLeft], 100 ,
369- mygrid->CartComm , &recvRequest[0 ]);
368+ MPI_Irecv (BufferRecvX1[faceLeft].data (), bufferSizeX1[faceRight], realMPI,
369+ procRecvX1[faceLeft], 100 , mygrid->CartComm , &recvRequest[0 ]);
370370
371371 // Send to the left
372372 // We receive from procRecv, and we send to procSend
373373
374- MPI_Isend (BufferSendX1[faceLeft].data (), bufferSizeX1[faceLeft], realMPI, procSendX1[faceLeft], 101 ,
375- mygrid->CartComm , &sendRequest[1 ]);
374+ MPI_Isend (BufferSendX1[faceLeft].data (), bufferSizeX1[faceLeft], realMPI,
375+ procSendX1[faceLeft], 101 , mygrid->CartComm , &sendRequest[1 ]);
376376
377- MPI_Irecv (BufferRecvX1[faceRight].data (), bufferSizeX1[faceLeft], realMPI, procRecvX1[faceRight], 101 ,
378- mygrid->CartComm , &recvRequest[1 ]);
377+ MPI_Irecv (BufferRecvX1[faceRight].data (), bufferSizeX1[faceLeft], realMPI,
378+ procRecvX1[faceRight], 101 , mygrid->CartComm , &recvRequest[1 ]);
379379
380380 // Wait for recv to complete (we don't care about the sends)
381381 MPI_Waitall (2 , recvRequest, recvStatus);
@@ -385,15 +385,19 @@ void Mpi::ExchangeX1(IdefixArray4D<real> Vc, IdefixArray4D<real> Vs) {
385385 // Send to the right
386386 // We receive from procRecv, and we send to procSend
387387
388- MPI_Sendrecv (BufferSendX1[faceRight].data (), bufferSizeX1[faceRight], realMPI, procSendX1[faceRight], 100 ,
389- BufferRecvX1[faceLeft].data (), bufferSizeX1[faceRight], realMPI, procRecvX1[faceLeft], 100 ,
388+ MPI_Sendrecv (BufferSendX1[faceRight].data (), bufferSizeX1[faceRight], realMPI,
389+ procSendX1[faceRight], 100 ,
390+ BufferRecvX1[faceLeft].data (), bufferSizeX1[faceRight], realMPI,
391+ procRecvX1[faceLeft], 100 ,
390392 mygrid->CartComm , &status);
391393
392394 // Send to the left
393395 // We receive from procRecv, and we send to procSend
394396
395- MPI_Sendrecv (BufferSendX1[faceLeft].data (), bufferSizeX1[faceLeft], realMPI, procSendX1[faceLeft], 101 ,
396- BufferRecvX1[faceRight].data (), bufferSizeX1[faceLeft], realMPI, procRecvX1[faceRight], 101 ,
397+ MPI_Sendrecv (BufferSendX1[faceLeft].data (), bufferSizeX1[faceLeft], realMPI,
398+ procSendX1[faceLeft], 101 ,
399+ BufferRecvX1[faceRight].data (), bufferSizeX1[faceLeft], realMPI,
400+ procRecvX1[faceRight], 101 ,
397401 mygrid->CartComm , &status);
398402 #endif
399403#endif
@@ -451,7 +455,7 @@ if(recvRight) {
451455 #endif
452456 }
453457}
454-
458+
455459myTimer -= MPI_Wtime ();
456460#ifdef MPI_NON_BLOCKING
457461 // Wait for the sends if they have not yet completed
@@ -563,35 +567,39 @@ void Mpi::ExchangeX2(IdefixArray4D<real> Vc, IdefixArray4D<real> Vs) {
563567 MPI_Request recvRequest[2 ];
564568
565569 // We receive from procRecv, and we send to procSend
566- MPI_Isend (BufferSendX2[faceRight].data (), bufferSizeX2[faceRight], realMPI, procSendX2[faceRight], 100 ,
567- mygrid->CartComm , &sendRequest[0 ]);
570+ MPI_Isend (BufferSendX2[faceRight].data (), bufferSizeX2[faceRight], realMPI, procSendX2[faceRight],
571+ 100 , mygrid->CartComm , &sendRequest[0 ]);
568572
569- MPI_Irecv (BufferRecvX2[faceLeft].data (), bufferSizeX2[faceRight], realMPI, procRecvX2[faceLeft], 100 ,
570- mygrid->CartComm , &recvRequest[0 ]);
573+ MPI_Irecv (BufferRecvX2[faceLeft].data (), bufferSizeX2[faceRight], realMPI, procRecvX2[faceLeft],
574+ 100 , mygrid->CartComm , &recvRequest[0 ]);
571575
572576 // Send to the left
573577 // We receive from procRecv, and we send to procSend
574- MPI_Isend (BufferSendX2[faceLeft].data (), bufferSizeX2[faceLeft], realMPI, procSendX2[faceLeft], 101 ,
575- mygrid->CartComm , &sendRequest[1 ]);
578+ MPI_Isend (BufferSendX2[faceLeft].data (), bufferSizeX2[faceLeft], realMPI, procSendX2[faceLeft],
579+ 101 , mygrid->CartComm , &sendRequest[1 ]);
576580
577- MPI_Irecv (BufferRecvX2[faceRight].data (), bufferSizeX2[faceLeft], realMPI, procRecvX2[faceRight], 101 ,
578- mygrid->CartComm , &recvRequest[1 ]);
581+ MPI_Irecv (BufferRecvX2[faceRight].data (), bufferSizeX2[faceLeft], realMPI, procRecvX2[faceRight],
582+ 101 , mygrid->CartComm , &recvRequest[1 ]);
579583
580584 // Wait for recv to complete (we don't care about the sends)
581585 MPI_Waitall (2 , recvRequest, recvStatus);
582586
583587 #else
584588 MPI_Status status;
585589 // We receive from procRecv, and we send to procSend
586- MPI_Sendrecv (BufferSendX2[faceRight].data (), bufferSizeX2[faceRight], realMPI, procSendX2[faceRight], 200 ,
587- BufferRecvX2[faceLeft].data (), bufferSizeX2[faceRight], realMPI, procRecvX2[faceLeft], 200 ,
588- mygrid->CartComm , &status);
590+ MPI_Sendrecv (BufferSendX2[faceRight].data (), bufferSizeX2[faceRight], realMPI,
591+ procSendX2[faceRight], 200 ,
592+ BufferRecvX2[faceLeft].data (), bufferSizeX2[faceRight], realMPI,
593+ procRecvX2[faceLeft], 200 ,
594+ mygrid->CartComm , &status);
589595
590596
591597 // Send to the left
592598 // We receive from procRecv, and we send to procSend
593- MPI_Sendrecv (BufferSendX2[faceLeft].data (), bufferSizeX2[faceLeft], realMPI, procSendX2[faceLeft], 201 ,
594- BufferRecvX2[faceRight].data (), bufferSizeX2[faceLeft], realMPI, procRecvX2[faceRight], 201 ,
599+ MPI_Sendrecv (BufferSendX2[faceLeft].data (), bufferSizeX2[faceLeft], realMPI,
600+ procSendX2[faceLeft], 201 ,
601+ BufferRecvX2[faceRight].data (), bufferSizeX2[faceLeft], realMPI,
602+ procRecvX2[faceRight], 201 ,
595603 mygrid->CartComm , &status);
596604 #endif
597605#endif
@@ -765,34 +773,38 @@ void Mpi::ExchangeX3(IdefixArray4D<real> Vc, IdefixArray4D<real> Vs) {
765773 MPI_Request recvRequest[2 ];
766774
767775 // We receive from procRecv, and we send to procSend
768- MPI_Isend (BufferSendX3[faceRight].data (), bufferSizeX3[faceRight], realMPI, procSendX3[faceRight], 100 ,
769- mygrid->CartComm , &sendRequest[0 ]);
776+ MPI_Isend (BufferSendX3[faceRight].data (), bufferSizeX3[faceRight], realMPI, procSendX3[faceRight],
777+ 100 , mygrid->CartComm , &sendRequest[0 ]);
770778
771- MPI_Irecv (BufferRecvX3[faceLeft].data (), bufferSizeX3[faceRight], realMPI, procRecvX3[faceLeft], 100 ,
772- mygrid->CartComm , &recvRequest[0 ]);
779+ MPI_Irecv (BufferRecvX3[faceLeft].data (), bufferSizeX3[faceRight], realMPI, procRecvX3[faceLeft],
780+ 100 , mygrid->CartComm , &recvRequest[0 ]);
773781
774782 // Send to the left
775783 // We receive from procRecv, and we send to procSend
776- MPI_Isend (BufferSendX3[faceLeft].data (), bufferSizeX3[faceLeft], realMPI, procSendX3[faceLeft], 101 ,
777- mygrid->CartComm , &sendRequest[1 ]);
784+ MPI_Isend (BufferSendX3[faceLeft].data (), bufferSizeX3[faceLeft], realMPI, procSendX3[faceLeft],
785+ 101 , mygrid->CartComm , &sendRequest[1 ]);
778786
779- MPI_Irecv (BufferRecvX3[faceRight].data (), bufferSizeX3[faceLeft], realMPI, procRecvX3[faceRight], 101 ,
780- mygrid->CartComm , &recvRequest[1 ]);
787+ MPI_Irecv (BufferRecvX3[faceRight].data (), bufferSizeX3[faceLeft], realMPI, procRecvX3[faceRight],
788+ 101 , mygrid->CartComm , &recvRequest[1 ]);
781789
782790 // Wait for recv to complete (we don't care about the sends)
783791 MPI_Waitall (2 , recvRequest, recvStatus);
784792
785793 #else
786794 MPI_Status status;
787795 // We receive from procRecv, and we send to procSend
788- MPI_Sendrecv (BufferSendX3[faceRight].data (), bufferSizeX3[faceRight], realMPI, procSendX3[faceRight], 300 ,
789- BufferRecvX3[faceLeft].data (), bufferSizeX3[faceRight], realMPI, procRecvX3[faceLeft], 300 ,
796+ MPI_Sendrecv (BufferSendX3[faceRight].data (), bufferSizeX3[faceRight], realMPI,
797+ procSendX3[faceRight], 300 ,
798+ BufferRecvX3[faceLeft].data (), bufferSizeX3[faceRight], realMPI,
799+ procRecvX3[faceLeft], 300 ,
790800 mygrid->CartComm , &status);
791801
792802 // Send to the left
793803 // We receive from procRecv, and we send to procSend
794- MPI_Sendrecv (BufferSendX3[faceLeft].data (), bufferSizeX3[faceLeft], realMPI, procSendX3[faceLeft], 301 ,
795- BufferRecvX3[faceRight].data (), bufferSizeX3[faceLeft], realMPI, procRecvX3[faceRight], 301 ,
804+ MPI_Sendrecv (BufferSendX3[faceLeft].data (), bufferSizeX3[faceLeft], realMPI,
805+ procSendX3[faceLeft], 301 ,
806+ BufferRecvX3[faceRight].data (), bufferSizeX3[faceLeft], realMPI,
807+ procRecvX3[faceRight], 301 ,
796808 mygrid->CartComm , &status);
797809 #endif
798810#endif
0 commit comments