Ran the EAGLE_25 step (shared memory)

swiftmpirdmastepsim.c

@@ -70,9 +70,6 @@ static int nr_ranks;
 /* Maximum no. of messages (logs). */
 static size_t max_logs = 0;
 
-/* A power of 2 higher than nr_ranks. */
-static size_t beyond_ranks = 2;
-
 /* Flags for controlling access. */
 static size_t UNLOCKED = (((size_t)2 << 63) - 1);
 
@@ -123,18 +120,30 @@ static int volatile nr_recv = 0;
 static int volatile todo_recv = 0;
 
 /**
- * @brief Convert a rank and tag into a single unique value.
+ * @brief Convert ranks and tag into a single unique value.
  *
- * Assumes beyond_ranks is set to a power of 2 larger than the highest rank
- * and that leaves enough bits for the tag in a size_t.
+ * Assumes there is enough space in a size_t for these values.
  *
- * @param rank the rank
+ * @param sendrank the send rank
+ * @param recvrank the receive rank
  * @param tag the tag
  *
  * @result a unique value based on both values
  */
-static size_t toranktag(int rank, int tag, int size) {
-  return (size_t)rank | (beyond_ranks * (size_t)tag | size);
+static size_t toranktag(int sendrank, int recvrank, int tag) {
+  
+  int shift = (sizeof(int) * 8) - __builtin_clz(nr_ranks); /* XXX could precalc. */
+  //message("nr_ranks = %d, shift = %d", nr_ranks, shift);
+  size_t result = sendrank | recvrank << shift | tag << (shift * 2);
+  return result;
+}
+
+static char *tokey(struct mpiuse_log_entry *log) {
+  static char buf[256];
+  sprintf(buf, "%d/%d/%d/%zd on %d ranktags %zd/%zd", 
+          log->otherrank, log->tag, log->subtype, log->size, log->rank,
+          toranktag(log->rank, log->otherrank, log->tag), toranktag(log->rank, log->otherrank, log->tag));
+  return buf;
 }
 
 /**
@@ -219,21 +228,23 @@ static void *send_thread(void *arg) {
     /* Need to find the offset for this data in the remotes window. We match
      * subtype, tag and rank. Need to search the ranktag_lists for our ranktag
      * value. XXX bisection search XXX */
-    size_t ranktag = toranktag(log->otherrank, log->tag, log->size);
+    size_t ranktag = toranktag(log->rank, log->otherrank, log->tag);
+    //message("looking for %s", tokey(log));
     size_t counts = ranktag_counts[INDEX2(task_subtype_count, log->subtype,
                                           log->otherrank)];
     size_t offset = 0;
     int found = 0;
+    struct mpiuse_log_entry *keeplog = NULL;
 
     counts = max_logs;
 
     for (size_t j = 0; j < counts; j++) {
       if (ranktag_lists[INDEX3(task_subtype_count, nr_ranks, log->subtype,
                                log->otherrank, j)] == ranktag) {
-        if (found) message("duplicate ranktag for subtype %d/%d ranktag: %zd",
-                           log->otherrank, log->subtype, ranktag);
+        if (found) message("duplicate %zd: %s of %s", ranktag, tokey(log), tokey(keeplog));
         offset = ranktag_offsets[INDEX3(task_subtype_count, nr_ranks,
                                         log->subtype, log->otherrank, j)];
+        keeplog = log;
         //message("%d/%d located offset %zd one of %zd ranktag: %zd", log->otherrank,
         //        log->subtype, offset, counts, ranktag);
         found = 1;
@@ -246,8 +257,8 @@ static void *send_thread(void *arg) {
             " found for ranktag %zd, counts = %zd", datasize, log->otherrank, log->subtype,
             offset, ranktag, counts);
     }
-    message("Sending a message of size %zd to %d/%d @ %zd", datasize,
-            log->otherrank, log->subtype, offset);
+    //message("Sending a message of size %zd to %d/%d @ %zd", datasize,
+    //        log->otherrank, log->subtype, offset);
 
 
     /* And send data to other rank. */
@@ -276,17 +287,17 @@ static void *send_thread(void *arg) {
     ret = MPI_Win_flush(log->otherrank, mpi_window[log->subtype]);
     if (ret != MPI_SUCCESS) mpi_error_message(ret, "MPI_Win_flush failed");
 
-    if (verbose) {
-      if (oldval[0] == dataptr[0]) {
-        message("sent a message to %d/%d (%zd:%zd:%zd @ %zd %d/%d)",
-                log->otherrank, log->subtype, dataptr[0], oldval[0], newval[0],
-                offset, k, nr_send);
-      } else {
-        message("failed to send a message to %d/%d (%zd:%zd:%zd) @ %zd %d/%d",
-                log->otherrank, log->subtype, dataptr[0], oldval[0], newval[0],
-                offset, k, nr_send);
-      }
-    }
+    //if (verbose) {
+    //  if (oldval[0] == dataptr[0]) {
+    //    message("sent a message to %d/%d (%zd:%zd:%zd @ %zd %d/%d)",
+    //            log->otherrank, log->subtype, dataptr[0], oldval[0], newval[0],
+    //            offset, k, nr_send);
+    //  } else {
+    //    message("failed to send a message to %d/%d (%zd:%zd:%zd) @ %zd %d/%d",
+    //            log->otherrank, log->subtype, dataptr[0], oldval[0], newval[0],
+    //            offset, k, nr_send);
+    //  }
+    //}
     //sleep(1);
   }
 
@@ -436,7 +447,8 @@ static void pick_logs() {
       if (log->rank == myrank) {
         log->done = 0;
         log->data = NULL;
-        log->ranktag = toranktag(log->rank, log->tag, log->size);
+        log->ranktag = toranktag(log->otherrank, log->rank, log->tag);
+        //message("add %s", tokey(log));
         if (log->type == task_type_send) {
           send_queue[nr_send] = log;
           nr_send++;
@@ -510,9 +522,6 @@ int main(int argc, char *argv[]) {
   if (res != MPI_SUCCESS)
     error("Call to MPI_Comm_rank failed with error %i.", res);
 
-  /* Need a power of 2 higher than nr_ranks. */
-  while (beyond_ranks * 2 < (size_t)nr_ranks) beyond_ranks *= 2;
-
   /* Handle the command-line, we expect a mpiuse data file to read and various
    * options. */
   int opt;