diff --git a/swiftmpirdmastepsim.c b/swiftmpirdmastepsim.c
index c29b85d10f3ec002924141a94c49f8bbd80ee74b..f72ddbd7b4f7b3e00c8bedecd6e003c5036ca790 100644
--- a/swiftmpirdmastepsim.c
+++ b/swiftmpirdmastepsim.c
@@ -70,9 +70,6 @@ static int nr_ranks;
/* Maximum no. of messages (logs). */
static size_t max_logs = 0;
-/* A power of 2 higher than nr_ranks. */
-static size_t beyond_ranks = 2;
-
/* Flags for controlling access. */
static size_t UNLOCKED = (((size_t)2 << 63) - 1);
@@ -123,18 +120,30 @@ static int volatile nr_recv = 0;
static int volatile todo_recv = 0;
/**
- * @brief Convert a rank and tag into a single unique value.
+ * @brief Convert ranks and tag into a single unique value.
*
- * Assumes beyond_ranks is set to a power of 2 larger than the highest rank
- * and that leaves enough bits for the tag in a size_t.
+ * Assumes there is enough space in a size_t for these values.
*
- * @param rank the rank
+ * @param sendrank the send rank
+ * @param recvrank the receive rank
* @param tag the tag
*
* @result a unique value based on both values
*/
-static size_t toranktag(int rank, int tag, int size) {
- return (size_t)rank | (beyond_ranks * (size_t)tag | size);
+static size_t toranktag(int sendrank, int recvrank, int tag) {
+
+ int shift = (sizeof(int) * 8) - __builtin_clz(nr_ranks); /* XXX could precalc. */
+ //message("nr_ranks = %d, shift = %d", nr_ranks, shift);
+ size_t result = sendrank | recvrank << shift | tag << (shift * 2);
+ return result;
+}
+
+static char *tokey(struct mpiuse_log_entry *log) {
+ static char buf[256];
+ sprintf(buf, "%d/%d/%d/%zd on %d ranktags %zd/%zd",
+ log->otherrank, log->tag, log->subtype, log->size, log->rank,
+ toranktag(log->rank, log->otherrank, log->tag), toranktag(log->rank, log->otherrank, log->tag));
+ return buf;
}
/**
@@ -219,21 +228,23 @@ static void *send_thread(void *arg) {
/* Need to find the offset for this data in the remotes window. We match
* subtype, tag and rank. Need to search the ranktag_lists for our ranktag
* value. XXX bisection search XXX */
- size_t ranktag = toranktag(log->otherrank, log->tag, log->size);
+ size_t ranktag = toranktag(log->rank, log->otherrank, log->tag);
+ //message("looking for %s", tokey(log));
size_t counts = ranktag_counts[INDEX2(task_subtype_count, log->subtype,
log->otherrank)];
size_t offset = 0;
int found = 0;
+ struct mpiuse_log_entry *keeplog = NULL;
counts = max_logs;
for (size_t j = 0; j < counts; j++) {
if (ranktag_lists[INDEX3(task_subtype_count, nr_ranks, log->subtype,
log->otherrank, j)] == ranktag) {
- if (found) message("duplicate ranktag for subtype %d/%d ranktag: %zd",
- log->otherrank, log->subtype, ranktag);
+ if (found) message("duplicate %zd: %s of %s", ranktag, tokey(log), tokey(keeplog));
offset = ranktag_offsets[INDEX3(task_subtype_count, nr_ranks,
log->subtype, log->otherrank, j)];
+ keeplog = log;
//message("%d/%d located offset %zd one of %zd ranktag: %zd", log->otherrank,
// log->subtype, offset, counts, ranktag);
found = 1;
@@ -246,8 +257,8 @@ static void *send_thread(void *arg) {
" found for ranktag %zd, counts = %zd", datasize, log->otherrank, log->subtype,
offset, ranktag, counts);
}
- message("Sending a message of size %zd to %d/%d @ %zd", datasize,
- log->otherrank, log->subtype, offset);
+ //message("Sending a message of size %zd to %d/%d @ %zd", datasize,
+ // log->otherrank, log->subtype, offset);
/* And send data to other rank. */
@@ -276,17 +287,17 @@ static void *send_thread(void *arg) {
ret = MPI_Win_flush(log->otherrank, mpi_window[log->subtype]);
if (ret != MPI_SUCCESS) mpi_error_message(ret, "MPI_Win_flush failed");
- if (verbose) {
- if (oldval[0] == dataptr[0]) {
- message("sent a message to %d/%d (%zd:%zd:%zd @ %zd %d/%d)",
- log->otherrank, log->subtype, dataptr[0], oldval[0], newval[0],
- offset, k, nr_send);
- } else {
- message("failed to send a message to %d/%d (%zd:%zd:%zd) @ %zd %d/%d",
- log->otherrank, log->subtype, dataptr[0], oldval[0], newval[0],
- offset, k, nr_send);
- }
- }
+ //if (verbose) {
+ // if (oldval[0] == dataptr[0]) {
+ // message("sent a message to %d/%d (%zd:%zd:%zd @ %zd %d/%d)",
+ // log->otherrank, log->subtype, dataptr[0], oldval[0], newval[0],
+ // offset, k, nr_send);
+ // } else {
+ // message("failed to send a message to %d/%d (%zd:%zd:%zd) @ %zd %d/%d",
+ // log->otherrank, log->subtype, dataptr[0], oldval[0], newval[0],
+ // offset, k, nr_send);
+ // }
+ //}
//sleep(1);
}
@@ -436,7 +447,8 @@ static void pick_logs() {
if (log->rank == myrank) {
log->done = 0;
log->data = NULL;
- log->ranktag = toranktag(log->rank, log->tag, log->size);
+ log->ranktag = toranktag(log->otherrank, log->rank, log->tag);
+ //message("add %s", tokey(log));
if (log->type == task_type_send) {
send_queue[nr_send] = log;
nr_send++;
@@ -510,9 +522,6 @@ int main(int argc, char *argv[]) {
if (res != MPI_SUCCESS)
error("Call to MPI_Comm_rank failed with error %i.", res);
- /* Need a power of 2 higher than nr_ranks. */
- while (beyond_ranks * 2 < (size_t)nr_ranks) beyond_ranks *= 2;
-
/* Handle the command-line, we expect a mpiuse data file to read and various
* options. */
int opt;