-
Peter W. Draper authored
# Conflicts: # swiftmpistepsim.c
Peter W. Draper authored# Conflicts: # swiftmpistepsim.c
post-process.py 5.76 KiB
#!/usr/bin/env python
"""
Usage:
post-process.py [options] output-log matched-log
Match the sends and recvs across the ranks so that the timings for message
completion can be checked. This is written to the file matched_log. Also
produces simple report of some interesting values and produces plots comparing
various timings.
This file is part of SWIFT.
Copyright (C) 2019 Peter W. Draper (p.w.draper@durham.ac.uk)
All Rights Reserved.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy
import pylab as pl
import sys
import argparse
# Handle the command line.
parser = argparse.ArgumentParser(description="Match MPI reports")
parser.add_argument(
"input",
help="Output log from simulator")
parser.add_argument(
"output",
help="Matched entries from the input data")
parser.add_argument(
"-v",
"--verbose",
dest="verbose",
help="Verbose output",
default=False,
action="store_true",
)
args = parser.parse_args()
infile = args.input
outfile = args.output
# Indices for words in a line.
logticcol=0
logcol=1
injcol=2
endcol=3
dticcol=4
stepcol=5
rankcol=6
otherrankcol=7
itypecol=8
isubtypecol=9
tagcol=10
sizecol=11
ntestcol=12
sumcol=13
mincol=14
maxcol=15
# Keyed lines.
keysends = {}
keyrecvs = {}
# Indexed lines.
sends = []
recvs = []
nsends = 0
nrecvs = 0
# Gather keys from the input log. We created dicts with matchable keys
# for when sends start and recvs end. Other pairings are possible...
# Note size of completion recv is negative.
with open(infile, "r") as fp:
for line in fp:
if line[0] == '#':
continue
words = line.split()
if words[itypecol] == "25":
key = words[otherrankcol] + "/" + \
words[rankcol] + "/" + \
words[isubtypecol] + "/" + \
words[tagcol] + "/" + \
words[sizecol]
if not key in keysends:
keysends[key] = [nsends]
else:
keysends[key].append(nsends)
sends.append(words)
nsends = nsends + 1
elif words[itypecol] == "26":
key = words[rankcol] + "/" + \
words[otherrankcol] + "/" + \
words[isubtypecol] + "/" + \
words[tagcol] + "/" + \
words[sizecol]
if not key in keyrecvs:
keyrecvs[key] = [nrecvs]
else:
keyrecvs[key].append(nrecvs)
recvs.append(words)
nrecvs = nrecvs + 1
# Now output. Note we could have unmatched recv keys, we don't check for that.
msends = [None] * nsends
with open(outfile, "w") as fp:
fp.write("# send_logticin send_logtic send_injtic send_endtic send_dtic send_step send_rank send_otherrank send_itype send_isubtype send_tag send_size send_nr_tests send_tsum send_tmin send_tmax recv_logticin recv_logtic recv_injtic recv_endtic recv_dtic recv_step recv_rank recv_otherrank recv_itype recv_isubtype recv_tag recv_size recv_nr_tests recv_tsum recv_tmin recv_tmax\n")
for key in keysends:
if key in keyrecvs:
if len(keysends[key]) == 1 and len(keyrecvs[key]) == 1:
isend = keysends[key][0]
irecv = keyrecvs[key][0]
msends[isend] = irecv
fp.write(" ".join(sends[isend]) + " " + " ".join(recvs[irecv]) + "\n")
else:
print "# ERROR: found ", len(keysends[key]), "/", len(keyrecvs[key]), " matches for key: ", key, " should be 1/1"
else:
print "# ERROR: missing recv key: ", key
print "# Matched sends and recvs written to file: ", outfile
# Reorder recvs to same order as sends.
recvs = [recvs[i] for i in msends]
# Do a plot. Display and saves the graphic, uncomment the "Agg" line above to just save.
def doplot(x, y, xlabel, ylabel, title, outpng):
axmax1 = max(x)
axmax2 = max(y)
axmax = max([axmax1, axmax2])
fig,ax = plt.subplots()
ax.set_xlim(0,axmax)
ax.set_ylim(0,axmax)
ax.plot(x, y, ',')
ax.set_xlabel(xlabel)
ax.set_ylabel(ylabel)
ax.set_title("SWIFTmpistepsim plot: " + title)
fig.tight_layout()
plt.savefig(outpng, bbox_inches="tight")
print "# Saved plot to: ", outpng
plt.show()
# Plot no. 1: sends injection time against time of local handoff.
send_injects = [float(send[injcol]) for send in [line for line in sends]]
send_ends = [float(send[endcol]) for send in [line for line in sends]]
doplot(send_injects, send_ends, "Message start time",
"Message local completion time", "local send completions",
"local_sends.png")
# Plot no. 2: recv injection time against time of local handoff.
recv_injects = [float(recv[injcol]) for recv in [line for line in recvs]]
recv_ends = [float(recv[endcol]) for recv in [line for line in recvs]]
doplot(recv_injects, recv_ends, "Message start time",
"Message local completion time", "local recv completions",
"local_recvs.png")
# Plot no. 3: send injection time against time of remote completion.
doplot(send_injects, recv_ends, "Message start time",
"Global message completion time", "message completion times",
"completions.png")
sys.exit(0)