Use a single pinned memory buffer

Adding more can make the latency better (since we seem to use ibv_post_srq_recv), but get some odd effects at higher scalings, so not doing that

swiftmpirdmastepsim.c

@@ -198,6 +198,11 @@ static void *send_thread(void *arg) {
     message("%d connected to remote server %s %d on %d", myrank, server_ip,
             rank, BASE_PORT + myrank);
 
+  /* Need to assign to a buffer to register memory. Make large enough for any
+     message. */
+  auto *sendBuffer =
+    new infinity::memory::Buffer(context, tobytes(MESSAGE_SIZE));
+
   for (int k = 0; k < nr_send; k++) {
     struct mpiuse_log_entry *log = send_queue[k];
 
@@ -221,22 +226,21 @@ static void *send_thread(void *arg) {
     dataptr[2] = log->size;
     dataptr[3] = log->tag;
 
-    /* Need to assign to a buffer to register memory. */
-    auto *sendBuffer =
-        new infinity::memory::Buffer(context, dataptr, tobytes(datasize));
+    /* Copy data into buffer. */
+    memcpy(sendBuffer->getData(), dataptr, tobytes(datasize));
 
     // And send
     infinity::requests::RequestToken requestToken(context);
-    qp->send(sendBuffer, &requestToken);
+    qp->send(sendBuffer, tobytes(datasize), &requestToken);
     requestToken.waitUntilCompleted();
     log->endtic = getticks();
 
-    delete sendBuffer;  // XXX Can we reuse ?
   }
 
   message("took %.3f %s.", clocks_from_ticks(getticks() - starttics),
           clocks_getunit());
 
+  delete sendBuffer;
   delete qp;
   delete qpFactory;
   delete context;