diff --git a/post-process.py b/post-process.py index 746db3503a9c1f6db7196595c7b4796a99100379..ce70ff1b7ad9ab8f1c4c6e21c15a6e55f655b13c 100755 --- a/post-process.py +++ b/post-process.py @@ -1,9 +1,12 @@ #!/usr/bin/env python """ Usage: - post-process.py [options] output-log + post-process.py [options] output-log matched-log -Match the sends and recvs across the ranks and output a unified log. +Match the sends and recvs across the ranks so that the timings for message +completion can be checked. This is written to the file matched_log. Also +produces simple report of some interesting values and produces plots comparing +various timings. This file is part of SWIFT. @@ -24,6 +27,10 @@ You should have received a copy of the GNU Lesser General Public License along with this program. If not, see <http://www.gnu.org/licenses/>. """ +import matplotlib +matplotlib.use("Agg") +import numpy +import pylab as pl import sys import argparse @@ -33,6 +40,9 @@ parser = argparse.ArgumentParser(description="Match MPI reports") parser.add_argument( "input", help="Output log from simulator") +parser.add_argument( + "output", + help="Matched entries from the input data") parser.add_argument( "-v", "--verbose", @@ -43,6 +53,7 @@ parser.add_argument( ) args = parser.parse_args() infile = args.input +outfile = args.output # Indices for words in a line. logticcol=0 @@ -63,8 +74,14 @@ mincol=14 maxcol=15 # Keyed lines. -sends = {} -recvs = {} +keysends = {} +keyrecvs = {} + +# Indexed lines. +sends = [] +recvs = [] +nsends = 0 +nrecvs = 0 # Gather keys from the input log. We created dicts with matchable keys # for when sends start and recvs end. Other pairings are possible... @@ -80,10 +97,12 @@ with open(infile, "r") as fp: words[isubtypecol] + "/" + \ words[tagcol] + "/" + \ words[sizecol] - if not key in sends: - sends[key] = [line[:-1]] + if not key in keysends: + keysends[key] = [nsends] else: - sends[key].append(line[:-1]) + keysends[key].append(nsends) + sends.append(words) + nsends = nsends + 1 elif words[itypecol] == "23": key = words[rankcol] + "/" + \ @@ -91,21 +110,26 @@ with open(infile, "r") as fp: words[isubtypecol] + "/" + \ words[tagcol] + "/" + \ words[sizecol] - if not key in recvs: - recvs[key] = [line[:-1]] + if not key in keyrecvs: + keyrecvs[key] = [nrecvs] else: - recvs[key].append(line[:-1]) + keyrecvs[key].append(nrecvs) + recvs.append(words) + nrecvs = nrecvs + 1 # Now output. Note we could have unmatched recv keys, we don't check for that. -print "# send_logticin send_logtic send_injtic send_endtic send_dtic send_step send_rank send_otherrank send_itype send_isubtype send_tag send_size send_nr_tests send_tsum send_tmin send_tmax recv_logticin recv_logtic recv_injtic recv_endtic recv_dtic recv_step recv_rank recv_otherrank recv_itype recv_isubtype recv_tag recv_size recv_nr_tests recv_tsum recv_tmin recv_tmax" -for key in sends: - if key in recvs: - if len(sends[key]) == 1 and len(recvs[key]) == 1: - print sends[key][0], recvs[key][0] +with open(outfile, "w") as fp: + fp.write("# send_logticin send_logtic send_injtic send_endtic send_dtic send_step send_rank send_otherrank send_itype send_isubtype send_tag send_size send_nr_tests send_tsum send_tmin send_tmax recv_logticin recv_logtic recv_injtic recv_endtic recv_dtic recv_step recv_rank recv_otherrank recv_itype recv_isubtype recv_tag recv_size recv_nr_tests recv_tsum recv_tmin recv_tmax\n") + for key in keysends: + if key in keyrecvs: + if len(keysends[key]) == 1 and len(keyrecvs[key]) == 1: + isend = keysends[key][0] + irecv = keyrecvs[key][0] + fp.write(" ".join(sends[isend]) + " " + " ".join(recvs[irecv]) + "\n") + else: + print "# ERROR: found ", len(keysends[key]), "/", len(keyrecvs[key]), " matches for key: ", key, " should be 1/1" else: - print "# ERROR: found ", len(sends[key]), "/", len(recvs[key]), " matches for key: ", key, " should be 1/1" - else: - print "# ERROR: missing recv key: ", key + print "# ERROR: missing recv key: ", key sys.exit(0)