diff --git a/Makefile b/Makefile
index eef066dd1b00a624194537ff2a2d7384d2aa383f..500e52e7c323cb9a54c31c5d13cfd43a224f1840 100644
--- a/Makefile
+++ b/Makefile
@@ -1,8 +1,10 @@
+CFLAGS = -g -O0 -Wall
+
+
all: swiftmpistepsim
swiftmpistepsim: swiftmpistepsim.c mpiuse.c mpiuse.h atomic.h cycle.h clocks.h clocks.c
- $(CC) -g -O0 -Wall -o swiftmpistepsim swiftmpistepsim.c mpiuse.c clocks.c -I/usr/include/mpi -lmpi -lpthread
+ $(CC) $(CFLAGS) -o swiftmpistepsim swiftmpistepsim.c mpiuse.c clocks.c -I/usr/include/mpi -lmpi -lpthread
clean:
rm swiftmpistepsim
-
diff --git a/mpiuse.c b/mpiuse.c
index fad598e3ad95b27809a983ba641ff430719c3fda..bae6a88e3b5b4336d7081eb06bd16a4b332078f1 100644
--- a/mpiuse.c
+++ b/mpiuse.c
@@ -31,10 +31,12 @@
/* Local includes. */
#include "atomic.h"
+#include "clocks.h"
#include "cycle.h"
#include "error.h"
-/* XXX threading support not needed, should remove. */
+/* Our rank. */
+extern int myrank;
/* The initial size and increment of the log entries buffer. */
#define MPIUSE_INITLOG 1000000
@@ -134,7 +136,7 @@ void mpiuse_log_allocation(int rank, int step, size_t tic, int type,
/**
* @brief restore the log from a dump.
*
- * @param filename name of file with the previus dump in.
+ * @param filename name of file with the previous dump in.
*/
void mpiuse_log_restore(const char *filename) {
@@ -166,6 +168,64 @@ void mpiuse_log_restore(const char *filename) {
fclose(fd);
}
+/**
+ * @brief dump the logs for all ranks to a file.
+ *
+ * @param nranks the number of ranks.
+ * @param dumpfile the file to write
+ */
+void mpiuse_dump_logs(int nranks, const char *dumpfile) {
+
+ /* Make sure output file is empty, only on one rank. */
+ FILE *fd;
+ if (myrank == 0) {
+ fd = fopen(dumpfile, "w");
+
+ /* Header. */
+ fprintf(fd, "# logticin logtic injtic endtic dtic step rank otherrank itype "
+ " isubtype tag size nr_tests tsum tmin tmax\n");
+ fclose(fd);
+ }
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ /* Loop over all ranks, one by one, getting each rank to append their
+ * logs. */
+ for (int k = 0; k < nranks; k++) {
+
+ /* Rank 0 decides the index of the writing node, this happens
+ * one-by-one. */
+ int kk = k;
+ MPI_Bcast(&kk, 1, MPI_INT, 0, MPI_COMM_WORLD);
+
+ if (k == myrank) {
+
+ /* Open file and position at end. */
+ fd = fopen(dumpfile, "a");
+
+ /* And append our logs. Note log->tic is not necessarily from this
+ machine, so the conversion to ms may be suspect. */
+ size_t nlogs = mpiuse_log_count;
+ for (size_t k = 0; k < nlogs; k++) {
+ struct mpiuse_log_entry *log = &mpiuse_log[k];
+ if (log->rank == myrank && log->endtic > 0) {
+ fprintf(fd, "%lld %.4f %.4f %.4f %.6f %d %d %d %d %d %d %zd %d %.4f %.6f %.6f\n",
+ log->tic,
+ clocks_from_ticks(log->tic),
+ clocks_from_ticks(log->injtic - clocks_start_ticks),
+ clocks_from_ticks(log->endtic - clocks_start_ticks),
+ clocks_from_ticks(log->endtic - log->injtic),
+ log->step, log->rank,
+ log->otherrank, log->type, log->subtype, log->tag, log->size,
+ log->nr_tests, clocks_from_ticks(log->tsum),
+ clocks_from_ticks(log->tmin), clocks_from_ticks(log->tmax));
+ }
+ }
+ fclose(fd);
+ }
+ }
+
+}
+
/**
* @brief return the number of log entries.
*
diff --git a/mpiuse.h b/mpiuse.h
index 0725bc9e096b0f5a73260f40e7110fdcea9407f3..707ab27547b14fc701fcfd53254d29d603007891 100644
--- a/mpiuse.h
+++ b/mpiuse.h
@@ -41,7 +41,7 @@ struct mpiuse_log_entry {
* same as delivered, need to match across ranks to see that. */
int activation;
- /* Ticks at time of this action. */
+ /* Ticks at time of this action (original log). */
ticks tic;
/* Rank of otherside of communication. */
@@ -53,11 +53,35 @@ struct mpiuse_log_entry {
/* Memory of the request. */
size_t size;
+ /* Extra information generated by the simulator. */
+ /* --------------------------------------------- */
+
+ /* Interaction is complete. */
+ int done;
+
/* Pointer to the data. */
void *data;
/* The request associated with the call. */
MPI_Request req;
+
+ /* The time that the injection started. */
+ ticks injtic;
+
+ /* The time that the exchange complete. */
+ ticks endtic;
+
+ /* The number of times that the exchange was tested. */
+ int nr_tests;
+
+ /* Sum of ticks that we tested. */
+ double tsum;
+
+ /* Maximum time in a test. */
+ ticks tmax;
+
+ /* Minimum time in a test. */
+ ticks tmin;
};
/* API. */
@@ -68,5 +92,6 @@ struct mpiuse_log_entry *mpiuse_get_log(int ind);
void mpiuse_log_restore(const char *filename);
int mpiuse_nr_logs(void);
int mpiuse_nr_ranks(void);
+void mpiuse_dump_logs(int nranks, const char *logfile);
#endif /* SWIFT_MPIUSE_H */
diff --git a/swiftmpistepsim.c b/swiftmpistepsim.c
index 18320c58fe3ad4a91501d1fe65a091050e149b93..c1f27739340ef91a05fcf0fa6cb64e152643eda2 100644
--- a/swiftmpistepsim.c
+++ b/swiftmpistepsim.c
@@ -20,6 +20,7 @@
#include <mpi.h>
#include <pthread.h>
#include <unistd.h>
+#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
@@ -114,7 +115,16 @@ static void *inject_thread(void *arg) {
}
}
- // Differences to SWIFT: MPI_BYTE might overflow, should use MPI_Type(?).
+ /* Initialise new log elements. */
+ log->done = 0;
+ log->nr_tests = 0;
+ log->tsum = 0.0;
+ log->tmax = 0;
+ log->tmin = INT_MAX;
+ log->endtic = 0;
+ log->injtic = getticks();
+
+ /* Differences to SWIFT: MPI_BYTE not the MPI_Type. */
int err = 0;
if (log->type == task_type_send) {
err = MPI_Isend(log->data, log->size, MPI_BYTE, log->otherrank, log->tag,
@@ -180,10 +190,10 @@ static void queue_runner(struct mpiuse_log_entry **logs, int volatile *nr_logs,
int volatile *todos, double *sum, int *ncalls,
ticks *mint, ticks *maxt) {
- /* MPI_Test statistics. */
+ /* Global MPI_Test statistics. */
int lncalls = 0;
double lsum = 0.0;
- ticks lmint = log_clocks_cpufreq;
+ ticks lmint = INT_MAX;
ticks lmaxt = 0;
/* We loop while new requests are being injected and we still have requests
@@ -192,7 +202,7 @@ static void queue_runner(struct mpiuse_log_entry **logs, int volatile *nr_logs,
int nlogs = *nr_logs;
for (int k = 0; k < nlogs; k++) {
struct mpiuse_log_entry *log = logs[k];
- if (log != NULL) {
+ if (log != NULL && !log->done) {
ticks tics = getticks();
int res;
MPI_Status stat;
@@ -200,16 +210,25 @@ static void queue_runner(struct mpiuse_log_entry **logs, int volatile *nr_logs,
if (err != MPI_SUCCESS) {
error("MPI_Test call failed");
}
+
+ /* Increment etc. of statistics about time in MPI_Test. */
ticks dt = getticks() - tics;
+ log->tsum += (double)dt;
lsum += (double)dt;
+
+ log->nr_tests++;
lncalls++;
+
+ if (dt < log->tmin) log->tmin = dt;
+ if (dt > log->tmax) log->tmax = dt;
if (dt < lmint) lmint = dt;
if (dt > lmaxt) lmaxt = dt;
- if (res) {
+ if (res) {
/* Done, clean up. */
+ log->done = 1;
+ log->endtic = getticks();
free(log->data);
- logs[k] = NULL;
atomic_dec(todos);
}
}
@@ -334,7 +353,7 @@ static void pick_logs(void) {
* @brief usage help.
*/
static void usage(char *argv[]) {
- fprintf(stderr, "Usage: %s [-vf] SWIFT_mpiuse-log-file.dat\n", argv[0]);
+ fprintf(stderr, "Usage: %s [-vf] SWIFT_mpiuse-log-file.dat logfile.dat\n", argv[0]);
fprintf(stderr, " options: -v verbose, -f fast injections\n");
fflush(stderr);
}
@@ -370,11 +389,12 @@ int main(int argc, char *argv[]) {
return 1;
}
}
- if (optind >= argc) {
+ if (optind >= argc - 1) {
if (myrank == 0) usage(argv);
return 1;
}
char *infile = argv[optind];
+ char *logfile = argv[optind+1];
/* Now we read the SWIFT MPI logger output that defines the communcations
* we will undertake and the time differences between injections into the
@@ -420,10 +440,17 @@ int main(int argc, char *argv[]) {
pthread_join(sendthread, NULL);
pthread_join(recvthread, NULL);
+ /* Dump the updated MPI logs. */
+ MPI_Barrier(MPI_COMM_WORLD);
+ fflush(stdout);
+ if (myrank == 0) message("Dumping updated log");
+ mpiuse_dump_logs(nranks, logfile);
+
/* Shutdown MPI. */
res = MPI_Finalize();
if (res != MPI_SUCCESS)
error("call to MPI_Finalize failed with error %i.", res);
+
if (myrank == 0) message("Bye");
return 0;