diff --git a/mpiuse.c b/mpiuse.c
index 77bd4a1009172ede87c6f6623ba60934d1c39359..9d8d037dcce15d064cdccaf224c0b35161941900 100644
--- a/mpiuse.c
+++ b/mpiuse.c
@@ -211,7 +211,7 @@ void mpiuse_dump_logs(int nranks, const char *dumpfile) {
ticks basetics = 0;
for (size_t k = 0; k < nlogs; k++) {
struct mpiuse_log_entry *log = &mpiuse_log[k];
- if (log->rank == myrank) {
+ if (log->rank == myrank && log->endtic > 0) {
if (basetics == 0) basetics = log->tic;
fprintf(fd,
"%lld %.4f %.4f %.4f %.6f %d %d %d %d %d %d %zd %d %.4f %.6f "
@@ -274,7 +274,7 @@ static double gauss_rand_upper(void) {
V1 = U1 - 1.0;
V2 = U2 - 1.0;
S = V1 * V1 + V2 * V2;
- } while(S >= 1.0 || S == 0.0);
+ } while (S >= 1.0 || S == 0.0);
return fabs(V1 * sqrt(-2.0 * log(S) / S));
}
@@ -292,15 +292,15 @@ static double gauss_rand_upper(void) {
* @param seed the random seed, use same for fixed sequences.
* @param uniform whether to use a uniform distribution other gaussian, unless
* cdf is defined, in which case this parameter is ignored.
- * @param cdf text file containing a normalized CDF to use as a basis for inverse
- * transform sampling of the randoms. NULL for no file.
+ * @param cdf text file containing a normalized CDF to use as a basis for
+ * inverse transform sampling of the randoms. NULL for no file.
*/
void mpiuse_log_generate(int nr_nodes, int nr_logs, int size, int random,
long int seed, int uniform, const char *cdf) {
/* Only used for CDF, may need to increase these. */
int nvals = 0;
- double imin[256], imax[256], value[256];
+ double imin[1024], imax[1024], value[1024];
/* Note that each rank exchanges messages with all the others and each "log"
* has the same size. */
@@ -337,7 +337,7 @@ void mpiuse_log_generate(int nr_nodes, int nr_logs, int size, int random,
if (cdf) {
/* CDF randoms. */
double rand = drand48();
-
+
/* Binary search for containing bin for this rand. */
unsigned int lower = 0;
unsigned int upper = nvals;
@@ -363,8 +363,10 @@ void mpiuse_log_generate(int nr_nodes, int nr_logs, int size, int random,
for (int i = 0; i < nr_nodes; i++) {
for (int j = 0; j < nr_nodes; j++) {
if (i != j) {
- mpiuse_log_allocation(i, 1, k, SEND_TYPE, NO_SUBTYPE, 1, logsize, j, tag);
- mpiuse_log_allocation(j, 1, k, RECV_TYPE, NO_SUBTYPE, 1, logsize, i, tag);
+ mpiuse_log_allocation(i, 1, k, SEND_TYPE, NO_SUBTYPE, 1, logsize, j,
+ tag);
+ mpiuse_log_allocation(j, 1, k, RECV_TYPE, NO_SUBTYPE, 1, logsize, i,
+ tag);
}
}
}
diff --git a/swiftmpifakestepsim.c b/swiftmpifakestepsim.c
index 80b9e5e3ef046e67e41fcdf11cc52b3ed2c74d5c..9e80be89ff7b6d5e860ccb20658a27eb1b3262ea 100644
--- a/swiftmpifakestepsim.c
+++ b/swiftmpifakestepsim.c
@@ -41,6 +41,9 @@ static int datacheck = 0;
/* Default seed for pseudorandoms. */
static long int default_seed = 1987654321;
+/* MPI communicator for each rank. XXX static XXX. */
+static MPI_Comm node_comms[512];
+
/* The local queues. */
static struct mpiuse_log_entry **volatile reqs_queue;
static int volatile ind_req = 0;
@@ -64,7 +67,7 @@ static int volatile todo_send = 0;
static void datacheck_fill(size_t size, void *data) {
unsigned char *p = (unsigned char *)data;
for (size_t i = 0; i < size; i++) {
- p[i] = 170; /* 10101010 in bits. */
+ p[i] = 170; /* 10101010 in bits. */
}
}
@@ -114,7 +117,7 @@ static void *inject_thread(void *arg) {
/* And send. */
err = MPI_Isend(log->data, log->size, MPI_BYTE, log->otherrank, log->tag,
- MPI_COMM_WORLD, &log->req);
+ node_comms[log->rank], &log->req);
/* Add a new send request. */
int ind = atomic_inc(&nr_sends);
@@ -126,7 +129,7 @@ static void *inject_thread(void *arg) {
/* Ready to receive. */
log->data = calloc(log->size, 1);
err = MPI_Irecv(log->data, log->size, MPI_BYTE, log->otherrank, log->tag,
- MPI_COMM_WORLD, &log->req);
+ node_comms[log->otherrank], &log->req);
/* Add a new recv request. */
int ind = atomic_inc(&nr_recvs);
@@ -134,7 +137,7 @@ static void *inject_thread(void *arg) {
atomic_inc(&todo_recv);
}
if (err != MPI_SUCCESS) error("Failed to activate send or recv");
-
+
ind_req++;
}
@@ -233,8 +236,7 @@ static void queue_runner(struct mpiuse_log_entry **logs, int volatile *nr_logs,
*/
static void *send_thread(void *arg) {
- if (verbose) message("%d: send thread starts (%d)", *((int *)arg),
- injecting);
+ if (verbose) message("%d: send thread starts (%d)", *((int *)arg), injecting);
ticks starttics = getticks();
int ncalls;
@@ -339,9 +341,9 @@ static void pick_logs(void) {
* @brief usage help.
*/
static void usage(char *argv[]) {
- fprintf(stderr, "Usage: %s [-vfgcx] nr_messages logfile.dat\n",
- argv[0]);
- fprintf(stderr, " options: -v verbose, -d data check, -s size (bytes/scale), \n"
+ fprintf(stderr, "Usage: %s [-vfgcx] nr_messages logfile.dat\n", argv[0]);
+ fprintf(stderr,
+ " options: -v verbose, -d data check, -s size (bytes/scale), \n"
"\t[-r uniform random from 1 to size, | \n"
"\t-r -g half gaussian random from 1 with 2.5 sigma size., | \n"
"\t-r -c <file> use cdf from file, size is a scale factor.,] \n"
@@ -405,7 +407,7 @@ int main(int argc, char *argv[]) {
return 1;
}
}
- if (optind >= argc -1) {
+ if (optind >= argc - 1) {
if (myrank == 0) usage(argv);
return 1;
}
@@ -419,25 +421,35 @@ int main(int argc, char *argv[]) {
if (myrank == 0) {
if (random) {
if (cdf != NULL) {
- message("Generating %d fake logs for %d ranks with randoms"
- " based on cdf %s scaled by factor %d", nr_logs, nr_nodes,
- cdf,size);
+ message(
+ "Generating %d fake logs for %d ranks with randoms"
+ " based on cdf %s scaled by factor %d",
+ nr_logs, nr_nodes, cdf, size);
} else if (uniform) {
- message("Generating %d fake logs for %d ranks with random distribution"
- " using size %d", nr_logs, nr_nodes, size);
+ message(
+ "Generating %d fake logs for %d ranks with random distribution"
+ " using size %d",
+ nr_logs, nr_nodes, size);
} else {
- message("Generating %d fake logs for %d ranks with gaussian random "
- "distribution using size %d as 2.5 sigma", nr_logs, nr_nodes, size);
+ message(
+ "Generating %d fake logs for %d ranks with gaussian random "
+ "distribution using size %d as 2.5 sigma",
+ nr_logs, nr_nodes, size);
}
} else {
- message("Generating %d fake logs for %d ranks of size %d",
- nr_logs, nr_nodes, size);
+ message("Generating %d fake logs for %d ranks of size %d", nr_logs,
+ nr_nodes, size);
}
}
mpiuse_log_generate(nr_nodes, nr_logs, size, random, seed, uniform, cdf);
int nranks = mpiuse_nr_ranks();
+ /* Create communicators for each MPI rank. */
+ for (int i = 0; i < nr_nodes; i++) {
+ MPI_Comm_dup(MPI_COMM_WORLD, &node_comms[i]);
+ }
+
/* Each rank requires its own queue, so extract them. */
pick_logs();
@@ -448,7 +460,8 @@ int main(int argc, char *argv[]) {
message("Start of MPI tests");
message("==================");
if (verbose) {
- if (datacheck) message("checking data pattern on send and recv completion");
+ if (datacheck)
+ message("checking data pattern on send and recv completion");
}
}
diff --git a/swiftmpistepsim.c b/swiftmpistepsim.c
index f6aaadc396c731fd99130ba42b7b9e380eaca39f..5722398fcdf05de61d866b09f48a8928ffaa1e41 100644
--- a/swiftmpistepsim.c
+++ b/swiftmpistepsim.c
@@ -76,7 +76,7 @@ static double log_clocks_cpufreq = 2194844448.0;
static void datacheck_fill(size_t size, void *data) {
unsigned char *p = (unsigned char *)data;
for (size_t i = 0; i < size; i++) {
- p[i] = 170; /* 10101010 in bits. */
+ p[i] = 170; /* 10101010 in bits. */
}
}
@@ -488,7 +488,8 @@ int main(int argc, char *argv[]) {
message("==================");
if (verbose) {
if (!usetics) message("using fast untimed injections");
- if (datacheck) message("checking data pattern on send and recv completion");
+ if (datacheck)
+ message("checking data pattern on send and recv completion");
}
}