diff --git a/swiftmpirdmastepsim.c b/swiftmpirdmastepsim.c index c29b85d10f3ec002924141a94c49f8bbd80ee74b..f72ddbd7b4f7b3e00c8bedecd6e003c5036ca790 100644 --- a/swiftmpirdmastepsim.c +++ b/swiftmpirdmastepsim.c @@ -70,9 +70,6 @@ static int nr_ranks; /* Maximum no. of messages (logs). */ static size_t max_logs = 0; -/* A power of 2 higher than nr_ranks. */ -static size_t beyond_ranks = 2; - /* Flags for controlling access. */ static size_t UNLOCKED = (((size_t)2 << 63) - 1); @@ -123,18 +120,30 @@ static int volatile nr_recv = 0; static int volatile todo_recv = 0; /** - * @brief Convert a rank and tag into a single unique value. + * @brief Convert ranks and tag into a single unique value. * - * Assumes beyond_ranks is set to a power of 2 larger than the highest rank - * and that leaves enough bits for the tag in a size_t. + * Assumes there is enough space in a size_t for these values. * - * @param rank the rank + * @param sendrank the send rank + * @param recvrank the receive rank * @param tag the tag * * @result a unique value based on both values */ -static size_t toranktag(int rank, int tag, int size) { - return (size_t)rank | (beyond_ranks * (size_t)tag | size); +static size_t toranktag(int sendrank, int recvrank, int tag) { + + int shift = (sizeof(int) * 8) - __builtin_clz(nr_ranks); /* XXX could precalc. */ + //message("nr_ranks = %d, shift = %d", nr_ranks, shift); + size_t result = sendrank | recvrank << shift | tag << (shift * 2); + return result; +} + +static char *tokey(struct mpiuse_log_entry *log) { + static char buf[256]; + sprintf(buf, "%d/%d/%d/%zd on %d ranktags %zd/%zd", + log->otherrank, log->tag, log->subtype, log->size, log->rank, + toranktag(log->rank, log->otherrank, log->tag), toranktag(log->rank, log->otherrank, log->tag)); + return buf; } /** @@ -219,21 +228,23 @@ static void *send_thread(void *arg) { /* Need to find the offset for this data in the remotes window. We match * subtype, tag and rank. Need to search the ranktag_lists for our ranktag * value. XXX bisection search XXX */ - size_t ranktag = toranktag(log->otherrank, log->tag, log->size); + size_t ranktag = toranktag(log->rank, log->otherrank, log->tag); + //message("looking for %s", tokey(log)); size_t counts = ranktag_counts[INDEX2(task_subtype_count, log->subtype, log->otherrank)]; size_t offset = 0; int found = 0; + struct mpiuse_log_entry *keeplog = NULL; counts = max_logs; for (size_t j = 0; j < counts; j++) { if (ranktag_lists[INDEX3(task_subtype_count, nr_ranks, log->subtype, log->otherrank, j)] == ranktag) { - if (found) message("duplicate ranktag for subtype %d/%d ranktag: %zd", - log->otherrank, log->subtype, ranktag); + if (found) message("duplicate %zd: %s of %s", ranktag, tokey(log), tokey(keeplog)); offset = ranktag_offsets[INDEX3(task_subtype_count, nr_ranks, log->subtype, log->otherrank, j)]; + keeplog = log; //message("%d/%d located offset %zd one of %zd ranktag: %zd", log->otherrank, // log->subtype, offset, counts, ranktag); found = 1; @@ -246,8 +257,8 @@ static void *send_thread(void *arg) { " found for ranktag %zd, counts = %zd", datasize, log->otherrank, log->subtype, offset, ranktag, counts); } - message("Sending a message of size %zd to %d/%d @ %zd", datasize, - log->otherrank, log->subtype, offset); + //message("Sending a message of size %zd to %d/%d @ %zd", datasize, + // log->otherrank, log->subtype, offset); /* And send data to other rank. */ @@ -276,17 +287,17 @@ static void *send_thread(void *arg) { ret = MPI_Win_flush(log->otherrank, mpi_window[log->subtype]); if (ret != MPI_SUCCESS) mpi_error_message(ret, "MPI_Win_flush failed"); - if (verbose) { - if (oldval[0] == dataptr[0]) { - message("sent a message to %d/%d (%zd:%zd:%zd @ %zd %d/%d)", - log->otherrank, log->subtype, dataptr[0], oldval[0], newval[0], - offset, k, nr_send); - } else { - message("failed to send a message to %d/%d (%zd:%zd:%zd) @ %zd %d/%d", - log->otherrank, log->subtype, dataptr[0], oldval[0], newval[0], - offset, k, nr_send); - } - } + //if (verbose) { + // if (oldval[0] == dataptr[0]) { + // message("sent a message to %d/%d (%zd:%zd:%zd @ %zd %d/%d)", + // log->otherrank, log->subtype, dataptr[0], oldval[0], newval[0], + // offset, k, nr_send); + // } else { + // message("failed to send a message to %d/%d (%zd:%zd:%zd) @ %zd %d/%d", + // log->otherrank, log->subtype, dataptr[0], oldval[0], newval[0], + // offset, k, nr_send); + // } + //} //sleep(1); } @@ -436,7 +447,8 @@ static void pick_logs() { if (log->rank == myrank) { log->done = 0; log->data = NULL; - log->ranktag = toranktag(log->rank, log->tag, log->size); + log->ranktag = toranktag(log->otherrank, log->rank, log->tag); + //message("add %s", tokey(log)); if (log->type == task_type_send) { send_queue[nr_send] = log; nr_send++; @@ -510,9 +522,6 @@ int main(int argc, char *argv[]) { if (res != MPI_SUCCESS) error("Call to MPI_Comm_rank failed with error %i.", res); - /* Need a power of 2 higher than nr_ranks. */ - while (beyond_ranks * 2 < (size_t)nr_ranks) beyond_ranks *= 2; - /* Handle the command-line, we expect a mpiuse data file to read and various * options. */ int opt;