Skip to content
Snippets Groups Projects
Commit e8fe313d authored by Peter W. Draper's avatar Peter W. Draper
Browse files

Use a single pinned memory buffer

Adding more can make the latency better (since we seem to use ibv_post_srq_recv), but get some odd effects at higher scalings, so not doing that
parent dfbd2cb1
No related branches found
No related tags found
1 merge request!8Draft: RDMA version with wrapped infinity calls
......@@ -198,6 +198,11 @@ static void *send_thread(void *arg) {
message("%d connected to remote server %s %d on %d", myrank, server_ip,
rank, BASE_PORT + myrank);
/* Need to assign to a buffer to register memory. Make large enough for any
message. */
auto *sendBuffer =
new infinity::memory::Buffer(context, tobytes(MESSAGE_SIZE));
for (int k = 0; k < nr_send; k++) {
struct mpiuse_log_entry *log = send_queue[k];
......@@ -221,22 +226,21 @@ static void *send_thread(void *arg) {
dataptr[2] = log->size;
dataptr[3] = log->tag;
/* Need to assign to a buffer to register memory. */
auto *sendBuffer =
new infinity::memory::Buffer(context, dataptr, tobytes(datasize));
/* Copy data into buffer. */
memcpy(sendBuffer->getData(), dataptr, tobytes(datasize));
// And send
infinity::requests::RequestToken requestToken(context);
qp->send(sendBuffer, &requestToken);
qp->send(sendBuffer, tobytes(datasize), &requestToken);
requestToken.waitUntilCompleted();
log->endtic = getticks();
delete sendBuffer; // XXX Can we reuse ?
}
message("took %.3f %s.", clocks_from_ticks(getticks() - starttics),
clocks_getunit());
delete sendBuffer;
delete qp;
delete qpFactory;
delete context;
......@@ -259,7 +263,7 @@ static void *recv_thread(void *arg) {
// Create buffer to receive messages. This is big enough for the largest
// message.
auto *receiveBuffer =
new infinity::memory::Buffer(context, tobytes(MESSAGE_SIZE));
new infinity::memory::Buffer(context, tobytes(MESSAGE_SIZE));
context->postReceiveBuffer(receiveBuffer);
// Port binding.
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment