Use MPI_Waitall to see if that helps or not. It does neither

README.md

 SWIFTmpistepsim
 ===============
 
+Variation: this branch uses an `MPI_Waitall` call to synchronously wait
+for all the available requests, rather than polling using MPI_Test.
+The result runs slightly slower, but takes roughly similar times as
+the MPI_Test version.
+
 This project is a standalone part of [SWIFT](http://www.swiftsim.com) that
 aims to roughly simulate the MPI interactions that taking a single step of a
 SWIFT simulation makes. Making it possible to more easily see the performance

swiftmpistepsim.c

@@ -175,7 +175,7 @@ static void *inject_thread(void *arg) {
 
 /**
  * @brief main loop to run over a queue of MPI requests and test for when they
- * complete. Returns the total amount of time spent in calls to MPI_Test and
+ * complete. Returns the total amount of time spent in calls to MPI_Waitall and
  * the number of times it was called.
  *
  * @param logs the list of logs pointing to requests.
@@ -183,16 +183,16 @@ static void *inject_thread(void *arg) {
  *                logs.
  * @param todos pointer to the variable containing the number of requests that
  *              are still active.
- * @param sum the total number of ticks spent in calls to MPI_Test.
- * @param ncalls the total number of calls to MPI_Test.
- * @param mint the minimum ticks an MPI_Test call took.
- * @param maxt the maximum ticks an MPI_Test call took.
+ * @param sum the total number of ticks spent in calls to MPI_Waitall.
+ * @param ncalls the total number of calls to MPI_Waitall.
+ * @param mint the minimum ticks an MPI_Waitall call took.
+ * @param maxt the maximum ticks an MPI_Waitall call took.
  */
 static void queue_runner(struct mpiuse_log_entry **logs, int volatile *nr_logs,
                          int volatile *todos, double *sum, int *ncalls,
                          ticks *mint, ticks *maxt) {
 
-  /* Global MPI_Test statistics. */
+  /* Global MPI_Waitall statistics. */
   int lncalls = 0;
   double lsum = 0.0;
   ticks lmint = INT_MAX;
@@ -201,40 +201,42 @@ static void queue_runner(struct mpiuse_log_entry **logs, int volatile *nr_logs,
   /* We loop while new requests are being injected and we still have requests
    * to complete. */
   while (injecting || (!injecting && *todos > 0)) {
+
+    /* Make a list of active requests. */
     int nlogs = *nr_logs;
+    MPI_Request *reqs = (MPI_Request *)calloc(nlogs, sizeof(MPI_Request));
+    int *inds = (int *)calloc(nlogs, sizeof(int));
+    int nactlogs = 0;
     for (int k = 0; k < nlogs; k++) {
       struct mpiuse_log_entry *log = logs[k];
       if (log != NULL && !log->done) {
-        ticks tics = getticks();
-        int res;
-        MPI_Status stat;
-        int err = MPI_Test(&log->req, &res, &stat);
-        if (err != MPI_SUCCESS) {
-          error("MPI_Test call failed");
-        }
-
-        /* Increment etc. of statistics about time in MPI_Test. */
-        ticks dt = getticks() - tics;
-        log->tsum += (double)dt;
-        lsum += (double)dt;
-
-        log->nr_tests++;
-        lncalls++;
-
-        if (dt < log->tmin) log->tmin = dt;
-        if (dt > log->tmax) log->tmax = dt;
-        if (dt < lmint) lmint = dt;
-        if (dt > lmaxt) lmaxt = dt;
-
-        if (res) {
-          /* Done, clean up. */
-          log->done = 1;
-          log->endtic = getticks();
-          free(log->data);
-          atomic_dec(todos);
-        }
+          reqs[nactlogs] = logs[k]->req;
+          inds[nactlogs] = k;
+          nactlogs++;
       }
     }
+    if (nactlogs > 0) {
+      ticks tics = getticks();
+      int err = MPI_Waitall(nactlogs, reqs, MPI_STATUSES_IGNORE);
+      if (err != MPI_SUCCESS) error("MPI_Waitall call failed");
+
+      /* Done, clean up. */
+      for (int k = 0; k < nactlogs; k++) {
+        struct mpiuse_log_entry *log = logs[inds[k]];
+        log->done = 1;
+        log->endtic = getticks();
+        free(log->data);
+      }
+
+      atomic_sub(todos, nactlogs);
+      lncalls++;
+      ticks dt = getticks() - tics;
+      if (dt > lmaxt) lmaxt = dt;
+      if (dt < lmint) lmint = dt;
+      lsum += dt;
+    }
+    free(reqs);
+    free(inds);
   }
 
   /* All done. */
@@ -260,7 +262,7 @@ static void *send_thread(void *arg) {
   queue_runner(sends_queue, &nr_sends, &todo_send, &sum, &ncalls, &mint, &maxt);
 
   message(
-      "%d MPI_Test calls took: %.3f, mean time %.3f, min time %.3f, max time "
+      "%d MPI_Waitall calls took: %.3f, mean time %.3f, min time %.3f, max time "
       "%.3f (%s)",
       ncalls, clocks_from_ticks(sum), clocks_from_ticks(sum / ncalls),
       clocks_from_ticks(mint), clocks_from_ticks(maxt), clocks_getunit());
@@ -287,7 +289,7 @@ static void *recv_thread(void *arg) {
   queue_runner(recvs_queue, &nr_recvs, &todo_recv, &sum, &ncalls, &mint, &maxt);
 
   message(
-      "%d MPI_Test calls took: %.3f, mean time %.3f, min time %.3f, max time "
+      "%d MPI_Waitall calls took: %.3f, mean time %.3f, min time %.3f, max time "
       "%.3f (%s)",
       ncalls, clocks_from_ticks(sum), clocks_from_ticks(sum / ncalls),
       clocks_from_ticks(mint), clocks_from_ticks(maxt), clocks_getunit());