From b856248218c486db79a46dd01a5c38742a4208f4 Mon Sep 17 00:00:00 2001
From: "Peter W. Draper" <p.w.draper@durham.ac.uk>
Date: Thu, 26 Sep 2019 15:55:14 +0100
Subject: [PATCH] Add output file rather than output to stdout and use indices
for matching in prep for using these later
---
post-process.py | 60 ++++++++++++++++++++++++++++++++++---------------
1 file changed, 42 insertions(+), 18 deletions(-)
diff --git a/post-process.py b/post-process.py
index 746db35..ce70ff1 100755
--- a/post-process.py
+++ b/post-process.py
@@ -1,9 +1,12 @@
#!/usr/bin/env python
"""
Usage:
- post-process.py [options] output-log
+ post-process.py [options] output-log matched-log
-Match the sends and recvs across the ranks and output a unified log.
+Match the sends and recvs across the ranks so that the timings for message
+completion can be checked. This is written to the file matched_log. Also
+produces simple report of some interesting values and produces plots comparing
+various timings.
This file is part of SWIFT.
@@ -24,6 +27,10 @@ You should have received a copy of the GNU Lesser General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
+import matplotlib
+matplotlib.use("Agg")
+import numpy
+import pylab as pl
import sys
import argparse
@@ -33,6 +40,9 @@ parser = argparse.ArgumentParser(description="Match MPI reports")
parser.add_argument(
"input",
help="Output log from simulator")
+parser.add_argument(
+ "output",
+ help="Matched entries from the input data")
parser.add_argument(
"-v",
"--verbose",
@@ -43,6 +53,7 @@ parser.add_argument(
)
args = parser.parse_args()
infile = args.input
+outfile = args.output
# Indices for words in a line.
logticcol=0
@@ -63,8 +74,14 @@ mincol=14
maxcol=15
# Keyed lines.
-sends = {}
-recvs = {}
+keysends = {}
+keyrecvs = {}
+
+# Indexed lines.
+sends = []
+recvs = []
+nsends = 0
+nrecvs = 0
# Gather keys from the input log. We created dicts with matchable keys
# for when sends start and recvs end. Other pairings are possible...
@@ -80,10 +97,12 @@ with open(infile, "r") as fp:
words[isubtypecol] + "/" + \
words[tagcol] + "/" + \
words[sizecol]
- if not key in sends:
- sends[key] = [line[:-1]]
+ if not key in keysends:
+ keysends[key] = [nsends]
else:
- sends[key].append(line[:-1])
+ keysends[key].append(nsends)
+ sends.append(words)
+ nsends = nsends + 1
elif words[itypecol] == "23":
key = words[rankcol] + "/" + \
@@ -91,21 +110,26 @@ with open(infile, "r") as fp:
words[isubtypecol] + "/" + \
words[tagcol] + "/" + \
words[sizecol]
- if not key in recvs:
- recvs[key] = [line[:-1]]
+ if not key in keyrecvs:
+ keyrecvs[key] = [nrecvs]
else:
- recvs[key].append(line[:-1])
+ keyrecvs[key].append(nrecvs)
+ recvs.append(words)
+ nrecvs = nrecvs + 1
# Now output. Note we could have unmatched recv keys, we don't check for that.
-print "# send_logticin send_logtic send_injtic send_endtic send_dtic send_step send_rank send_otherrank send_itype send_isubtype send_tag send_size send_nr_tests send_tsum send_tmin send_tmax recv_logticin recv_logtic recv_injtic recv_endtic recv_dtic recv_step recv_rank recv_otherrank recv_itype recv_isubtype recv_tag recv_size recv_nr_tests recv_tsum recv_tmin recv_tmax"
-for key in sends:
- if key in recvs:
- if len(sends[key]) == 1 and len(recvs[key]) == 1:
- print sends[key][0], recvs[key][0]
+with open(outfile, "w") as fp:
+ fp.write("# send_logticin send_logtic send_injtic send_endtic send_dtic send_step send_rank send_otherrank send_itype send_isubtype send_tag send_size send_nr_tests send_tsum send_tmin send_tmax recv_logticin recv_logtic recv_injtic recv_endtic recv_dtic recv_step recv_rank recv_otherrank recv_itype recv_isubtype recv_tag recv_size recv_nr_tests recv_tsum recv_tmin recv_tmax\n")
+ for key in keysends:
+ if key in keyrecvs:
+ if len(keysends[key]) == 1 and len(keyrecvs[key]) == 1:
+ isend = keysends[key][0]
+ irecv = keyrecvs[key][0]
+ fp.write(" ".join(sends[isend]) + " " + " ".join(recvs[irecv]) + "\n")
+ else:
+ print "# ERROR: found ", len(keysends[key]), "/", len(keyrecvs[key]), " matches for key: ", key, " should be 1/1"
else:
- print "# ERROR: found ", len(sends[key]), "/", len(recvs[key]), " matches for key: ", key, " should be 1/1"
- else:
- print "# ERROR: missing recv key: ", key
+ print "# ERROR: missing recv key: ", key
sys.exit(0)
--
GitLab