Revert "Use a single pinned memory buffer", the behaviour after that

change is very non-linear. Fast for small messages, but larger ones
are discontinously longer running...

This reverts commit e8fe313d.

swiftmpirdmastepsim.c

@@ -198,11 +198,6 @@ static void *send_thread(void *arg) {
     message("%d connected to remote server %s %d on %d", myrank, server_ip,
             rank, BASE_PORT + myrank);
 
-  /* Need to assign to a buffer to register memory. Make large enough for any
-     message. */
-  auto *sendBuffer =
-    new infinity::memory::Buffer(context, tobytes(MESSAGE_SIZE));
-
   for (int k = 0; k < nr_send; k++) {
     struct mpiuse_log_entry *log = send_queue[k];
 
@@ -226,21 +221,22 @@ static void *send_thread(void *arg) {
     dataptr[2] = log->size;
     dataptr[3] = log->tag;
 
-    /* Copy data into buffer. */
-    memcpy(sendBuffer->getData(), dataptr, tobytes(datasize));
+    /* Need to assign to a buffer to register memory. */
+    auto *sendBuffer =
+        new infinity::memory::Buffer(context, dataptr, tobytes(datasize));
 
     // And send
     infinity::requests::RequestToken requestToken(context);
-    qp->send(sendBuffer, tobytes(datasize), &requestToken);
+    qp->send(sendBuffer, &requestToken);
     requestToken.waitUntilCompleted();
     log->endtic = getticks();
 
+    delete sendBuffer;  // XXX Can we reuse ?
   }
 
   message("took %.3f %s.", clocks_from_ticks(getticks() - starttics),
           clocks_getunit());
 
-  delete sendBuffer;
   delete qp;
   delete qpFactory;
   delete context;
@@ -263,7 +259,7 @@ static void *recv_thread(void *arg) {
   // Create buffer to receive messages. This is big enough for the largest
   // message.
   auto *receiveBuffer =
-    new infinity::memory::Buffer(context, tobytes(MESSAGE_SIZE));
+      new infinity::memory::Buffer(context, tobytes(MESSAGE_SIZE));
   context->postReceiveBuffer(receiveBuffer);
 
   // Port binding.