From 8360c935f70c9d4230258247b126210ac2001b7d Mon Sep 17 00:00:00 2001
From: "Peter W. Draper" <p.w.draper@durham.ac.uk>
Date: Fri, 13 Mar 2020 13:03:10 +0000
Subject: [PATCH] Add an overview of what the code does and how to get the
 error

---
 exchange-test.c | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/exchange-test.c b/exchange-test.c
index 9ddb739..4e52e22 100644
--- a/exchange-test.c
+++ b/exchange-test.c
@@ -16,6 +16,39 @@
  *
  ******************************************************************************/
 
+/**
+ * This program simulates the exchange of packed cells between the proxies in
+ * SWIFT.
+ *
+ * What we do here is exchange a fixed sized data area between all MPI ranks,
+ * so each rank sends that data to all the other ranks, and so in return also
+ * receives data from all the other ranks. The sends and receives are done
+ * asynchronously.
+ *
+ * So we start isends to send the size of the data to all the other ranks
+ * (note the size not fixed in SWIFT and all ranks are not in general
+ * contacted), and also irecvs to receive the sizes from the other ranks. At
+ * this stage we also start the isends of the data. The data areas are filled
+ * with the number of the rank that they originate in, so we can check that
+ * the local data areas have been updated.
+ *
+ * We then waitany for the sizes to be received, on receipt we can then setup
+ * the irecvs for the data (since we now know the size).
+ *
+ * We then wait for the local isends to complete (must happen), and proceed to
+ * waitany for the data to arrive locally. Finally we wait for all the
+ * irecvs to complete.
+ *
+ * This works with Intel MPI 2018 and 2019, but not 2020 when using 16 ranks,
+ * a data size greater than 32MB and using transport over Mellanox
+ * (verbs;ofi_rxm). So the simple test:
+ *
+ *    mpirun -np 16 ./exchange-test
+ *
+ * Fails reporting that the local data is not updated, but the irecv for the
+ * data receipt is supposed to be completed.
+ */
+
 #include <limits.h>
 #include <mpi.h>
 #include <pthread.h>
-- 
GitLab