Extra ranktags that are used to speed up lookup, very marginal gains

swiftmpirdmaonestepsim2.c

@@ -275,6 +275,32 @@ static void *send_thread(void *arg) {
    * biggest message. */
   auto *sendBuffer = new infinity::memory::Buffer(context, tobytes(max_size));
 
+  /* Extract the offset lists that we use. */
+  int nr = 0;
+  int size = (max_logs / 16 + 1);
+  size_t *ranktags = (size_t *)malloc(size  * sizeof(size_t));
+  size_t *offsets = (size_t *)malloc(size  * sizeof(size_t));
+
+  /* A tag that will match any subtype or tag with our rank for all the otherranks. */
+  for (int k = 0; k < nr_ranks; k++) {
+    size_t matchranktag = toranktag(0, myrank, k, 0);
+    for (size_t j = 0; j < max_logs; j++) {
+      size_t ranktag = ranktag_lists[INDEX3(nr_ranks, nr_ranks, myrank, k, j)];
+      if ((ranktag & matchranktag) == matchranktag) {
+
+        /* Keep this one. */
+        ranktags[nr] = ranktag;
+        offsets[nr] = ranktag_offsets[INDEX3(nr_ranks, nr_ranks, myrank, k, j)];
+        nr++;
+        if (nr >= size) {
+          size += (max_logs / 16 + 1);
+          ranktags = (size_t *)realloc(ranktags, size * sizeof(size_t));
+          offsets = (size_t *)realloc(offsets, size * sizeof(size_t));
+        }
+      }
+    }
+  }
+
   // Startup complete, so start timing and release the receive thread.
   MPI_Barrier(MPI_COMM_WORLD);  // Vital for synchronization.
   clocks_set_cpufreq(cpufreq);
@@ -310,19 +336,18 @@ static void *send_thread(void *arg) {
     log->offset = 0;
     int found = 0;
 
-    // XXX must be faster way of doing this... Go back to per rank lists.
-    for (size_t j = 0; j < max_logs; j++) {
-      if (ranktag_lists[INDEX3(nr_ranks, nr_ranks, myrank, log->otherrank, j)] == ranktag) {
-        log->offset = ranktag_offsets[INDEX3(nr_ranks, nr_ranks, myrank, log->otherrank, j)];
+    for (int j = 0; j < nr; j++) {
+      if (ranktags[j] == ranktag) {
+        log->offset = offsets[j];
         found = 1;
         break;
       }
     }
     if (!found) {
       error(
-            "Failed sending a message of size %zd to %d/%d "
-            "@ %zd\n, no offset found for ranktag %zd",
-            datasize, log->otherrank, log->subtype, log->offset, ranktag);
+          "Failed sending a message of size %zd to %d/%d "
+          "@ %zd\n, no offset found for ranktag %zd",
+          datasize, log->otherrank, log->subtype, log->offset, ranktag);
     }
 
     if (verbose)