Skip to content
Snippets Groups Projects
Commit ace196f8 authored by Peter W. Draper's avatar Peter W. Draper
Browse files

Formatting

parent 1c2f92dd
Branches
No related tags found
1 merge request!8Draft: RDMA version with wrapped infinity calls
Show whitespace changes
Inline Side-by-side
swiftmpirdmaonestepsim.c
+ 73
− 79
View file @ ace196f8
......@@ -167,7 +167,9 @@ static BLOCKTYPE toblocks(BLOCKTYPE nr_bytes) {
*
* @result the number of bytes.
*/
static BLOCKTYPE tobytes(BLOCKTYPE nr_blocks) { return (nr_blocks * BYTESINBLOCK); }
static BLOCKTYPE tobytes(BLOCKTYPE nr_blocks) {
return (nr_blocks * BYTESINBLOCK);
}
/**
* @brief fill a data area with given value.
......@@ -223,8 +225,7 @@ static void *send_thread(void *arg) {
/* Short circuit. */
if (nr_sends[rank] == 0) {
if (verbose)
message("took 0.0 %s.", clocks_getunit());
if (verbose) message("took 0.0 %s.", clocks_getunit());
return NULL;
}
......@@ -240,14 +241,13 @@ static void *send_thread(void *arg) {
server_ip, rank, BASE_PORT + myrank);
auto *qp = qpFactory->connectToRemoteHost(server_ip, BASE_PORT + myrank);
if (verbose)
message("%d connected to remote server %s %d on %d", myrank, server_ip, rank,
BASE_PORT + myrank);
message("%d connected to remote server %s %d on %d", myrank, server_ip,
rank, BASE_PORT + myrank);
auto *remoteBufferToken = (infinity::memory::RegionToken *)qp->getUserData();
/* Register some memory for use by RDMA, make it large enough for our
* biggest message. */
auto *sendBuffer = new infinity::memory::Buffer(context,
tobytes(max_size));
auto *sendBuffer = new infinity::memory::Buffer(context, tobytes(max_size));
/* Queue of our sends. */
struct mpiuse_log_entry **send_queue = sends_queue[rank];
......@@ -287,8 +287,8 @@ static void *send_thread(void *arg) {
for (size_t j = 0; j < counts; j++) {
if (ranktag_lists[INDEX3(MAX_NR_RANKS, nr_ranks, log->rank,
log->otherrank, j)] == ranktag) {
log->offset = ranktag_offsets[INDEX3(MAX_NR_RANKS, nr_ranks,
log->rank, log->otherrank, j)];
log->offset = ranktag_offsets[INDEX3(MAX_NR_RANKS, nr_ranks, log->rank,
log->otherrank, j)];
found = 1;
break;
}
......@@ -297,12 +297,13 @@ static void *send_thread(void *arg) {
error(
"Failed sending a message of size %zd to %d/%d "
"@ %zd\n, no offset found for ranktag %zd, counts = %zd",
datasize, log->otherrank, log->subtype, log->offset, ranktag,
counts);
datasize, log->otherrank, log->subtype, log->offset, ranktag, counts);
}
if (verbose)
message("sending message subtype %d from %d to %d todo: %d/%d, offset %ld size %ld",
message(
"sending message subtype %d from %d to %d todo: %d/%d, offset %ld "
"size %ld",
log->subtype, myrank, log->otherrank, nr_sends[rank] - k,
nr_sends[rank], log->offset, datasize);
......@@ -313,8 +314,7 @@ static void *send_thread(void *arg) {
remoteBufferToken, // destination
tobytes(log->offset), // remoteOffset
tobytes(datasize), // sizeInBytes
infinity::queues::OperationFlags(),
&requestToken);
infinity::queues::OperationFlags(), &requestToken);
requestToken.waitUntilCompleted();
requestToken.reset();
......@@ -325,20 +325,18 @@ static void *send_thread(void *arg) {
remoteBufferToken, // destination
tobytes(log->offset), // remoteOffset
BYTESINBLOCK, // sizeInBytes
infinity::queues::OperationFlags(),
&requestToken);
infinity::queues::OperationFlags(), &requestToken);
requestToken.waitUntilCompleted(); // Since we reuse the sendBuffer.
// requestToken.reset();
log->endtic = getticks();
}
delete sendBuffer;
if (verbose)
message("sent %d to %d", nr_sends[rank], rank);
message("took %.3f %s sending to rank %d", clocks_from_ticks(getticks() - starttics),
clocks_getunit(), rank);
if (verbose) message("sent %d to %d", nr_sends[rank], rank);
message("took %.3f %s sending to rank %d",
clocks_from_ticks(getticks() - starttics), clocks_getunit(), rank);
delete sendBuffer;
delete qp;
delete qpFactory;
delete context;
......@@ -357,8 +355,7 @@ static void *recv_thread(void *arg) {
int todo_recv = nr_recvs[rank];
int nr_recv = nr_recvs[rank];
if (todo_recv == 0) {
if (verbose)
message("took 0.0 %s.", clocks_getunit());
if (verbose) message("took 0.0 %s.", clocks_getunit());
starting[rank] = 0; // Also fudge.
return NULL;
}
......@@ -380,11 +377,10 @@ static void *recv_thread(void *arg) {
}
qpFactory->bindToPort(BASE_PORT + rank);
if (verbose)
message("Blocking for first message on %d", BASE_PORT + rank);
if (verbose) message("Blocking for first message on %d", BASE_PORT + rank);
starting[rank] = 0; // really need to do this in acceptIncomingConnection().
auto qp = qpFactory->acceptIncomingConnection
(bufferToken, sizeof(infinity::memory::RegionToken));
auto qp = qpFactory->acceptIncomingConnection(
bufferToken, sizeof(infinity::memory::RegionToken));
if (verbose)
message("Accepting incoming connections on %d", BASE_PORT + rank);
......@@ -421,7 +417,8 @@ static void *recv_thread(void *arg) {
// int tag = dataptr[3];
if (verbose)
message("receive message subtype %d from %d to %d todo: %d/%d,"
message(
"receive message subtype %d from %d to %d todo: %d/%d,"
" offset %ld size %ld",
log->subtype, myrank, log->otherrank, todo_recv, nr_recv,
log->offset, toblocks(log->size) + HEADER_SIZE);
......@@ -442,13 +439,14 @@ static void *recv_thread(void *arg) {
}
}
message("took %.3f %s listening for rank %d", clocks_from_ticks(getticks() - starttics),
clocks_getunit(), rank);
message("took %.3f %s listening for rank %d",
clocks_from_ticks(getticks() - starttics), clocks_getunit(), rank);
delete qp;
// XXX Let this leak, tearing it down can be premature for the remote.
// delete qp;
qp->getLocalDeviceId(); // Stop compiler complaining.
delete qpFactory;
/* Thread exits. */
return NULL;
}
......@@ -474,11 +472,11 @@ static size_t pick_logs() {
size_t maxsize = 0;
/* Queues of send and receive logs. */
struct mpiuse_log_entry **send_queue = (struct mpiuse_log_entry **)
calloc(nlogs, sizeof(struct mpiuse_log_entry *));
struct mpiuse_log_entry **send_queue = (struct mpiuse_log_entry **)calloc(
nlogs, sizeof(struct mpiuse_log_entry *));
int nr_send = 0;
struct mpiuse_log_entry **recv_queue = (struct mpiuse_log_entry **)
calloc(nlogs, sizeof(struct mpiuse_log_entry *));
struct mpiuse_log_entry **recv_queue = (struct mpiuse_log_entry **)calloc(
nlogs, sizeof(struct mpiuse_log_entry *));
int nr_recv = 0;
for (size_t k = 0; k < nlogs; k++) {
......@@ -533,8 +531,8 @@ static size_t pick_logs() {
/* Recvs */
if (nr_recvs[k] > 0) {
recvs_queue[k] = (struct mpiuse_log_entry **)
calloc(nr_recvs[k], sizeof(struct mpiuse_log_entry *));
recvs_queue[k] = (struct mpiuse_log_entry **)calloc(
nr_recvs[k], sizeof(struct mpiuse_log_entry *));
int i = 0;
for (int j = 0; j < nr_recv; j++) {
struct mpiuse_log_entry *log = recv_queue[j];
......@@ -543,15 +541,14 @@ static size_t pick_logs() {
i++;
}
}
}
else {
} else {
recvs_queue[k] = NULL;
}
/* Sends. */
if (nr_sends[k] > 0) {
sends_queue[k] = (struct mpiuse_log_entry **)
calloc(nr_sends[k], sizeof(struct mpiuse_log_entry *));
sends_queue[k] = (struct mpiuse_log_entry **)calloc(
nr_sends[k], sizeof(struct mpiuse_log_entry *));
int i = 0;
for (int j = 0; j < nr_send; j++) {
struct mpiuse_log_entry *log = send_queue[j];
......@@ -560,17 +557,16 @@ static size_t pick_logs() {
i++;
}
}
}
else {
} else {
sends_queue[k] = NULL;
}
}
/* Offsets and ranktags. */
ranktag_offsets = (size_t *)
calloc(MAX_NR_RANKS * nr_ranks * max_logs, sizeof(size_t));
ranktag_lists = (size_t *)
calloc(MAX_NR_RANKS * nr_ranks * max_logs, sizeof(size_t));
ranktag_offsets =
(size_t *)calloc(MAX_NR_RANKS * nr_ranks * max_logs, sizeof(size_t));
ranktag_lists =
(size_t *)calloc(MAX_NR_RANKS * nr_ranks * max_logs, sizeof(size_t));
// ranktag_counts = (size_t *)calloc(MAX_NR_RANKS * nr_ranks, sizeof(size_t));
/* Setup the ranktag offsets for our receive windows. Also define the sizes
......@@ -692,9 +688,8 @@ int main(int argc, char *argv[]) {
MPI_COMM_WORLD);
// MPI_Allreduce(MPI_IN_PLACE, ranktag_counts, MAX_NR_RANKS * nr_ranks,
// MPI_AINT, MPI_SUM, MPI_COMM_WORLD);
MPI_Allreduce(MPI_IN_PLACE, ranktag_lists,
MAX_NR_RANKS * nr_ranks * max_logs, MPI_AINT, MPI_SUM,
MPI_COMM_WORLD);
MPI_Allreduce(MPI_IN_PLACE, ranktag_lists, MAX_NR_RANKS * nr_ranks * max_logs,
MPI_AINT, MPI_SUM, MPI_COMM_WORLD);
/* Now for the RDMA setup. We need the IP addresses of all the ranks. */
......@@ -762,8 +757,7 @@ int main(int argc, char *argv[]) {
/* Reset time as previous can be thought of as setup costs? */
MPI_Barrier(MPI_COMM_WORLD); // Vital...
if (myrank == 0)
message("All synchronized, restarting time.");
if (myrank == 0) message("All synchronized, restarting time.");
clocks_set_cpufreq(0);
/* Now we have a thread per rank to send the messages. */
......
......
swiftmpirdmastepsim.c
+ 20
− 20
View file @ ace196f8
......@@ -127,7 +127,9 @@ static int toblocks(BLOCKTYPE nr_bytes) {
*
* @result the number of bytes.
*/
static BLOCKTYPE tobytes(BLOCKTYPE nr_blocks) { return (nr_blocks * BYTESINBLOCK); }
static BLOCKTYPE tobytes(BLOCKTYPE nr_blocks) {
return (nr_blocks * BYTESINBLOCK);
}
/**
* @brief fill a data area with given value.
......@@ -191,7 +193,8 @@ static void *send_thread(void *arg) {
// message("%d waiting for connection to remote server %s %d on %d", myrank,
// server_ip, rank, BASE_PORT + myrank);
auto *qp = qpFactory->connectToRemoteHost(server_ip, BASE_PORT + myrank);
//message("%d connected to remote server %s %d on %d", myrank, server_ip, rank,
// message("%d connected to remote server %s %d on %d", myrank, server_ip,
// rank,
// BASE_PORT + myrank);
for (int k = 0; k < nr_send; k++) {
......@@ -218,8 +221,8 @@ static void *send_thread(void *arg) {
dataptr[3] = log->tag;
/* Need to assign to a buffer to register memory. */
auto *sendBuffer = new infinity::memory::Buffer(context, dataptr,
tobytes(datasize));
auto *sendBuffer =
new infinity::memory::Buffer(context, dataptr, tobytes(datasize));
// And send
infinity::requests::RequestToken requestToken(context);
......@@ -263,8 +266,7 @@ static void *recv_thread(void *arg) {
message("%d binding to %d on port %d", myrank, rank, BASE_PORT + rank);
qpFactory->bindToPort(BASE_PORT + rank);
if (verbose)
message("Blocking for first message on %d", BASE_PORT + rank);
if (verbose) message("Blocking for first message on %d", BASE_PORT + rank);
starting[rank] = 0; // really need to do this in acceptIncomingConnection().
auto qp = qpFactory->acceptIncomingConnection();
if (verbose)
......@@ -365,11 +367,11 @@ static size_t pick_logs() {
size_t maxsize = 0;
/* Queues of send and receive logs. */
send_queue = (struct mpiuse_log_entry **)
calloc(nlogs, sizeof(struct mpiuse_log_entry *));
send_queue = (struct mpiuse_log_entry **)calloc(
nlogs, sizeof(struct mpiuse_log_entry *));
nr_send = 0;
struct mpiuse_log_entry **recv_queue = (struct mpiuse_log_entry **)
calloc(nlogs, sizeof(struct mpiuse_log_entry *));
struct mpiuse_log_entry **recv_queue = (struct mpiuse_log_entry **)calloc(
nlogs, sizeof(struct mpiuse_log_entry *));
int nr_recv = 0;
for (size_t k = 0; k < nlogs; k++) {
......@@ -413,8 +415,8 @@ static size_t pick_logs() {
}
for (int k = 0; k < nr_ranks; k++) {
if (nr_recvs[k] > 0) {
recvs_queue[k] = (struct mpiuse_log_entry **)
calloc(nr_recvs[k], sizeof(struct mpiuse_log_entry *));
recvs_queue[k] = (struct mpiuse_log_entry **)calloc(
nr_recvs[k], sizeof(struct mpiuse_log_entry *));
int i = 0;
for (int j = 0; j < nr_recv; j++) {
struct mpiuse_log_entry *log = recv_queue[j];
......@@ -423,14 +425,12 @@ static size_t pick_logs() {
i++;
}
}
}
else {
} else {
recvs_queue[k] = NULL;
}
}
free(recv_queue);
if (verbose) {
message("maxsize = %zd, nr_send = %d, nr_recv = %d", maxsize, nr_send,
nr_recv);
......
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment