diff --git a/exchange-test.c b/exchange-test.c index 9ddb7391cf6ba8da8a6931be120fa046eac45307..4e52e226e0d950b8e73edbc5bce5b7c328e9999b 100644 --- a/exchange-test.c +++ b/exchange-test.c @@ -16,6 +16,39 @@ * ******************************************************************************/ +/** + * This program simulates the exchange of packed cells between the proxies in + * SWIFT. + * + * What we do here is exchange a fixed sized data area between all MPI ranks, + * so each rank sends that data to all the other ranks, and so in return also + * receives data from all the other ranks. The sends and receives are done + * asynchronously. + * + * So we start isends to send the size of the data to all the other ranks + * (note the size not fixed in SWIFT and all ranks are not in general + * contacted), and also irecvs to receive the sizes from the other ranks. At + * this stage we also start the isends of the data. The data areas are filled + * with the number of the rank that they originate in, so we can check that + * the local data areas have been updated. + * + * We then waitany for the sizes to be received, on receipt we can then setup + * the irecvs for the data (since we now know the size). + * + * We then wait for the local isends to complete (must happen), and proceed to + * waitany for the data to arrive locally. Finally we wait for all the + * irecvs to complete. + * + * This works with Intel MPI 2018 and 2019, but not 2020 when using 16 ranks, + * a data size greater than 32MB and using transport over Mellanox + * (verbs;ofi_rxm). So the simple test: + * + * mpirun -np 16 ./exchange-test + * + * Fails reporting that the local data is not updated, but the irecv for the + * data receipt is supposed to be completed. + */ + #include <limits.h> #include <mpi.h> #include <pthread.h>