From 605773bb2d7397de050fd78fe6deca1fd932e78e Mon Sep 17 00:00:00 2001
From: "Peter W. Draper" <p.w.draper@durham.ac.uk>
Date: Fri, 18 Sep 2020 18:25:00 +0100
Subject: [PATCH] Extra ranktags that are used to speed up lookup, very
marginal gains
---
swiftmpirdmaonestepsim2.c | 39 ++++++++++++++++++++++++++++++++-------
1 file changed, 32 insertions(+), 7 deletions(-)
diff --git a/swiftmpirdmaonestepsim2.c b/swiftmpirdmaonestepsim2.c
index 1f1ef4b..cb590ea 100644
--- a/swiftmpirdmaonestepsim2.c
+++ b/swiftmpirdmaonestepsim2.c
@@ -275,6 +275,32 @@ static void *send_thread(void *arg) {
* biggest message. */
auto *sendBuffer = new infinity::memory::Buffer(context, tobytes(max_size));
+ /* Extract the offset lists that we use. */
+ int nr = 0;
+ int size = (max_logs / 16 + 1);
+ size_t *ranktags = (size_t *)malloc(size * sizeof(size_t));
+ size_t *offsets = (size_t *)malloc(size * sizeof(size_t));
+
+ /* A tag that will match any subtype or tag with our rank for all the otherranks. */
+ for (int k = 0; k < nr_ranks; k++) {
+ size_t matchranktag = toranktag(0, myrank, k, 0);
+ for (size_t j = 0; j < max_logs; j++) {
+ size_t ranktag = ranktag_lists[INDEX3(nr_ranks, nr_ranks, myrank, k, j)];
+ if ((ranktag & matchranktag) == matchranktag) {
+
+ /* Keep this one. */
+ ranktags[nr] = ranktag;
+ offsets[nr] = ranktag_offsets[INDEX3(nr_ranks, nr_ranks, myrank, k, j)];
+ nr++;
+ if (nr >= size) {
+ size += (max_logs / 16 + 1);
+ ranktags = (size_t *)realloc(ranktags, size * sizeof(size_t));
+ offsets = (size_t *)realloc(offsets, size * sizeof(size_t));
+ }
+ }
+ }
+ }
+
// Startup complete, so start timing and release the receive thread.
MPI_Barrier(MPI_COMM_WORLD); // Vital for synchronization.
clocks_set_cpufreq(cpufreq);
@@ -310,19 +336,18 @@ static void *send_thread(void *arg) {
log->offset = 0;
int found = 0;
- // XXX must be faster way of doing this... Go back to per rank lists.
- for (size_t j = 0; j < max_logs; j++) {
- if (ranktag_lists[INDEX3(nr_ranks, nr_ranks, myrank, log->otherrank, j)] == ranktag) {
- log->offset = ranktag_offsets[INDEX3(nr_ranks, nr_ranks, myrank, log->otherrank, j)];
+ for (int j = 0; j < nr; j++) {
+ if (ranktags[j] == ranktag) {
+ log->offset = offsets[j];
found = 1;
break;
}
}
if (!found) {
error(
- "Failed sending a message of size %zd to %d/%d "
- "@ %zd\n, no offset found for ranktag %zd",
- datasize, log->otherrank, log->subtype, log->offset, ranktag);
+ "Failed sending a message of size %zd to %d/%d "
+ "@ %zd\n, no offset found for ranktag %zd",
+ datasize, log->otherrank, log->subtype, log->offset, ranktag);
}
if (verbose)
--
GitLab