diff --git a/.clang-format b/.clang-format
new file mode 100644
index 0000000000000000000000000000000000000000..49524f944860b51ba9aa461b3299b70a68df46ae
--- /dev/null
+++ b/.clang-format
@@ -0,0 +1,6 @@
+---
+Language: Cpp
+BasedOnStyle: Google
+KeepEmptyLinesAtTheStartOfBlocks: true
+PenaltyBreakAssignment: 2
+...
diff --git a/Makefile b/Makefile
index 500e52e7c323cb9a54c31c5d13cfd43a224f1840..75a2e49df9978b6383cf26aa2ad88be12d501164 100644
--- a/Makefile
+++ b/Makefile
@@ -1,10 +1,9 @@
CFLAGS = -g -O0 -Wall
-
all: swiftmpistepsim
swiftmpistepsim: swiftmpistepsim.c mpiuse.c mpiuse.h atomic.h cycle.h clocks.h clocks.c
- $(CC) $(CFLAGS) -o swiftmpistepsim swiftmpistepsim.c mpiuse.c clocks.c -I/usr/include/mpi -lmpi -lpthread
+ mpicc $(CFLAGS) -o swiftmpistepsim swiftmpistepsim.c mpiuse.c clocks.c
clean:
- rm swiftmpistepsim
+ rm -f swiftmpistepsim
diff --git a/format.sh b/format.sh
new file mode 100755
index 0000000000000000000000000000000000000000..91346334c9b2eaf9fbb343aba44f8a02d866d1ef
--- /dev/null
+++ b/format.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# Clang format command, can be overridden using CLANG_FORMAT_CMD.
+# We currrently use version 5.0 so any overrides should provide that.
+clang=${CLANG_FORMAT_CMD:="clang-format-5.0"}
+
+# Formatting command
+cmd="$clang -style=file $(git ls-files | grep '\.[ch]$')"
+
+# Test if `clang-format-5.0` works
+command -v $clang > /dev/null
+if [[ $? -ne 0 ]]
+then
+ echo "ERROR: cannot find $clang"
+ exit 1
+fi
+
+# Print the help
+function show_help {
+ echo -e "This script formats SWIFT according to Google style"
+ echo -e " -h, --help \t Show this help"
+ echo -e " -t, --test \t Test if SWIFT is well formatted"
+}
+
+# Parse arguments (based on https://stackoverflow.com/questions/192249)
+TEST=0
+while [[ $# -gt 0 ]]
+do
+ key="$1"
+
+ case $key in
+ # print the help and exit
+ -h|--help)
+ show_help
+ exit
+ ;;
+ # check if the code is well formatted
+ -t|--test)
+ TEST=1
+ shift
+ ;;
+ # unknown option
+ *)
+ echo "Argument '$1' not implemented"
+ show_help
+ exit
+ ;;
+ esac
+done
+
+# Run the required commands
+if [[ $TEST -eq 1 ]]
+then
+ # Note trapping the exit status from both commands in the pipe. Also note
+ # do not use -q in grep as that closes the pipe on first match and we get
+ # a SIGPIPE error.
+ echo "Testing if SWIFT is correctly formatted"
+ $cmd -output-replacements-xml | grep "<replacement " > /dev/null
+ status=("${PIPESTATUS[@]}")
+
+ # Trap if first command failed. Note 141 is SIGPIPE, that happens when no
+ # output
+ if [[ ${status[0]} -ne 0 ]]
+ then
+ echo "ERROR: $clang command failed"
+ exit 1
+ fi
+
+ # Check formatting
+ if [[ ${status[1]} -eq 0 ]]
+ then
+ echo "ERROR: needs formatting"
+ exit 1
+ else
+ echo "...is correctly formatted"
+ fi
+else
+ echo "Formatting SWIFT"
+ $cmd -i
+fi
diff --git a/post-process.py b/post-process.py
index 746db3503a9c1f6db7196595c7b4796a99100379..458fae25a26a1a2a6be033449dc3c7ef94e294fd 100755
--- a/post-process.py
+++ b/post-process.py
@@ -1,9 +1,12 @@
#!/usr/bin/env python
"""
Usage:
- post-process.py [options] output-log
+ post-process.py [options] output-log matched-log
-Match the sends and recvs across the ranks and output a unified log.
+Match the sends and recvs across the ranks so that the timings for message
+completion can be checked. This is written to the file matched_log. Also
+produces simple report of some interesting values and produces plots comparing
+various timings.
This file is part of SWIFT.
@@ -24,6 +27,11 @@ You should have received a copy of the GNU Lesser General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
+import matplotlib
+#matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy
+import pylab as pl
import sys
import argparse
@@ -33,6 +41,9 @@ parser = argparse.ArgumentParser(description="Match MPI reports")
parser.add_argument(
"input",
help="Output log from simulator")
+parser.add_argument(
+ "output",
+ help="Matched entries from the input data")
parser.add_argument(
"-v",
"--verbose",
@@ -43,6 +54,7 @@ parser.add_argument(
)
args = parser.parse_args()
infile = args.input
+outfile = args.output
# Indices for words in a line.
logticcol=0
@@ -63,8 +75,14 @@ mincol=14
maxcol=15
# Keyed lines.
-sends = {}
-recvs = {}
+keysends = {}
+keyrecvs = {}
+
+# Indexed lines.
+sends = []
+recvs = []
+nsends = 0
+nrecvs = 0
# Gather keys from the input log. We created dicts with matchable keys
# for when sends start and recvs end. Other pairings are possible...
@@ -80,10 +98,12 @@ with open(infile, "r") as fp:
words[isubtypecol] + "/" + \
words[tagcol] + "/" + \
words[sizecol]
- if not key in sends:
- sends[key] = [line[:-1]]
+ if not key in keysends:
+ keysends[key] = [nsends]
else:
- sends[key].append(line[:-1])
+ keysends[key].append(nsends)
+ sends.append(words)
+ nsends = nsends + 1
elif words[itypecol] == "23":
key = words[rankcol] + "/" + \
@@ -91,21 +111,68 @@ with open(infile, "r") as fp:
words[isubtypecol] + "/" + \
words[tagcol] + "/" + \
words[sizecol]
- if not key in recvs:
- recvs[key] = [line[:-1]]
+ if not key in keyrecvs:
+ keyrecvs[key] = [nrecvs]
else:
- recvs[key].append(line[:-1])
+ keyrecvs[key].append(nrecvs)
+ recvs.append(words)
+ nrecvs = nrecvs + 1
# Now output. Note we could have unmatched recv keys, we don't check for that.
-print "# send_logticin send_logtic send_injtic send_endtic send_dtic send_step send_rank send_otherrank send_itype send_isubtype send_tag send_size send_nr_tests send_tsum send_tmin send_tmax recv_logticin recv_logtic recv_injtic recv_endtic recv_dtic recv_step recv_rank recv_otherrank recv_itype recv_isubtype recv_tag recv_size recv_nr_tests recv_tsum recv_tmin recv_tmax"
-for key in sends:
- if key in recvs:
- if len(sends[key]) == 1 and len(recvs[key]) == 1:
- print sends[key][0], recvs[key][0]
+msends = [None] * nsends
+with open(outfile, "w") as fp:
+ fp.write("# send_logticin send_logtic send_injtic send_endtic send_dtic send_step send_rank send_otherrank send_itype send_isubtype send_tag send_size send_nr_tests send_tsum send_tmin send_tmax recv_logticin recv_logtic recv_injtic recv_endtic recv_dtic recv_step recv_rank recv_otherrank recv_itype recv_isubtype recv_tag recv_size recv_nr_tests recv_tsum recv_tmin recv_tmax\n")
+ for key in keysends:
+ if key in keyrecvs:
+ if len(keysends[key]) == 1 and len(keyrecvs[key]) == 1:
+ isend = keysends[key][0]
+ irecv = keyrecvs[key][0]
+ msends[isend] = irecv
+ fp.write(" ".join(sends[isend]) + " " + " ".join(recvs[irecv]) + "\n")
+ else:
+ print "# ERROR: found ", len(keysends[key]), "/", len(keyrecvs[key]), " matches for key: ", key, " should be 1/1"
else:
- print "# ERROR: found ", len(sends[key]), "/", len(recvs[key]), " matches for key: ", key, " should be 1/1"
- else:
- print "# ERROR: missing recv key: ", key
-
+ print "# ERROR: missing recv key: ", key
+
+print "# Matched sends and recvs written to file: ", outfile
+
+# Reorder recvs to same order as sends.
+recvs = [recvs[i] for i in msends]
+
+# Do a plot. Display and saves the graphic, uncomment the "Agg" line above to just save.
+def doplot(x, y, xlabel, ylabel, title, outpng):
+ axmax1 = max(x)
+ axmax2 = max(y)
+ axmax = max([axmax1, axmax2])
+ fig,ax = plt.subplots()
+ ax.set_xlim(0,axmax)
+ ax.set_ylim(0,axmax)
+ ax.plot(x, y, ',')
+ ax.set_xlabel(xlabel)
+ ax.set_ylabel(ylabel)
+ ax.set_title("SWIFTmpistepsim plot: " + title)
+ fig.tight_layout()
+ plt.savefig(outpng, bbox_inches="tight")
+ print "# Saved plot to: ", outpng
+ plt.show()
+
+# Plot no. 1: sends injection time against time of local handoff.
+send_injects = [float(send[injcol]) for send in [line for line in sends]]
+send_ends = [float(send[endcol]) for send in [line for line in sends]]
+doplot(send_injects, send_ends, "Message start time",
+ "Message local completion time", "local send completions",
+ "local_sends.png")
+
+# Plot no. 2: recv injection time against time of local handoff.
+recv_injects = [float(recv[injcol]) for recv in [line for line in recvs]]
+recv_ends = [float(recv[endcol]) for recv in [line for line in recvs]]
+doplot(recv_injects, recv_ends, "Message start time",
+ "Message local completion time", "local recv completions",
+ "local_recvs.png")
+
+# Plot no. 3: send injection time against time of remote completion.
+doplot(send_injects, recv_ends, "Message start time",
+ "Global message completion time", "message completion times",
+ "completions.png")
sys.exit(0)
diff --git a/swiftmpistepsim.c b/swiftmpistepsim.c
index 6181b28f6101adfe1c73c05d68ad9ad7c8e866db..b6acea367938b98f42c8b2a83a9d0aeff345f67c 100644
--- a/swiftmpistepsim.c
+++ b/swiftmpistepsim.c
@@ -41,6 +41,9 @@ static int usetics = 1;
/* The wait between injections, nanosecs. */
static long long waitns = 0;
+/* Set a data pattern and check we get this back, slow... */
+static int datacheck = 0;
+
/* Integer types of send and recv tasks, must match log. */
static const int task_type_send = 22;
static const int task_type_recv = 23;
@@ -69,7 +72,37 @@ static int volatile todo_send = 0;
// XXX need to store this in the data file.
static double log_clocks_cpufreq = 2194844448.0;
+/**
+ * @brief fill a data area with a pattern that can be checked for changes.
+ *
+ * @param size size of data in bytes.
+ * @param data the data to fill.
+ */
+static void datacheck_fill(size_t size, void *data) {
+ unsigned char *p = (unsigned char *)data;
+ for (size_t i = 0; i < size; i++) {
+ p[i] = 170; /* 10101010 in bits. */
+ }
+}
+
+/**
+ * @brief test a filled data area for our pattern.
+ *
+ * @param size size of data in bytes.
+ * @param data the data to fill.
+ *
+ * @result 1 on success, 0 otherwise.
+ */
+static int datacheck_test(size_t size, void *data) {
+ unsigned char *p = (unsigned char *)data;
+ for (size_t i = 0; i < size; i++) {
+ if (p[i] != 170) return 0;
+ }
+ return 1;
+}
+
static void injection_runner(int qid) {
+
if (verbose) message("%d: injection thread starts", qid);
ticks starttics = getticks();
struct mpiuse_log_entry **reqs = reqs_queue[qid];
@@ -130,6 +163,12 @@ static void injection_runner(int qid) {
/* Differences to SWIFT: MPI_BYTE not the MPI_Type. */
int err = 0;
if (log->type == task_type_send) {
+ log->data = calloc(log->size, 1);
+
+ /* Fill data with pattern. */
+ if (datacheck) datacheck_fill(log->size, log->data);
+
+ /* And send. */
err = MPI_Isend(log->data, log->size, MPI_BYTE, log->otherrank, log->tag,
subtypeMPI_comms[log->subtype], &log->req);
@@ -139,6 +178,9 @@ static void injection_runner(int qid) {
atomic_inc(&todo_send);
} else {
+
+ /* Ready to receive. */
+ log->data = calloc(log->size, 1);
err = MPI_Irecv(log->data, log->size, MPI_BYTE, log->otherrank, log->tag,
subtypeMPI_comms[log->subtype], &log->req);
@@ -253,6 +295,12 @@ static void queue_runner(struct mpiuse_log_entry **logs, int volatile *nr_logs,
if (dt > lmaxt) lmaxt = dt;
if (res) {
+ /* Check data sent data is unchanged and received data is as
+ * expected. */
+ if (datacheck && !datacheck_test(log->size, log->data)) {
+ error("Data mismatch on completion");
+ }
+
/* Done, clean up. */
log->done = 1;
log->endtic = getticks();
@@ -349,11 +397,11 @@ static void pick_logs(void) {
/* Duplicate of logs. */
struct mpiuse_log_entry **reqs = (struct mpiuse_log_entry **)malloc(sizeof(struct mpiuse_log_entry *) * nlogs);
int nreqs = 0;
- sends_queue = (struct mpiuse_log_entry **)malloc(
- sizeof(struct mpiuse_log_entry *) * nlogs);
+ sends_queue = (struct mpiuse_log_entry **)calloc(
+ nlogs, sizeof(struct mpiuse_log_entry *));
nr_sends = 0;
- recvs_queue = (struct mpiuse_log_entry **)malloc(
- sizeof(struct mpiuse_log_entry *) * nlogs);
+ recvs_queue = (struct mpiuse_log_entry **)calloc(
+ nlogs, sizeof(struct mpiuse_log_entry *));
nr_recvs = 0;
for (int k = 0; k < nlogs; k++) {
@@ -361,13 +409,10 @@ static void pick_logs(void) {
if (log->rank == myrank && log->activation) {
if (log->type == task_type_send || log->type == task_type_recv) {
- /* Allocate space for data. */
- log->data = calloc(log->size, 1);
-
/* And keep this log. */
+ log->data = NULL;
reqs[nreqs] = log;
nreqs++;
-
} else {
error("task type '%d' is not a known send or recv task", log->type);
}
@@ -436,8 +481,11 @@ int main(int argc, char *argv[]) {
/* Handle the command-line, we expect a mpiuse data file to read and various
* options. */
int opt;
- while ((opt = getopt(argc, argv, "vfn:")) != -1) {
+ while ((opt = getopt(argc, argv, "vfdn:")) != -1) {
switch (opt) {
+ case 'd':
+ datacheck = 1;
+ break;
case 'f':
usetics = 0;
break;
@@ -486,6 +534,10 @@ int main(int argc, char *argv[]) {
message("Start of MPI tests");
message("==================");
if (waitns > 0) message("adding fixed waits of %lld ns", waitns);
+ if (verbose) {
+ if (!usetics) message("using fast untimed injections");
+ if (datacheck) message("checking data pattern on send and recv completion");
+ }
}
/* Make three threads, one for injecting tasks and two to check for