Add scripts that change the number of messages, but preserve the data volume

testdata/doubledata.py

+#!/usr/bin/env python
+"""
+Usage:
+    doubledata.py [options] input-log doubled-log
+
+Split pairs of send and recv log data entries to increase the number of
+messages that we send to represent the data of a step. Preserves the data
+volume, but not the tagging.
+
+This file is part of SWIFT.
+
+Copyright (C) 2019 Peter W. Draper (p.w.draper@durham.ac.uk)
+All Rights Reserved.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published
+by the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+import argparse
+import sys
+from operator import itemgetter
+
+#  Handle the command line.
+parser = argparse.ArgumentParser(description="Pair MPI logs")
+
+parser.add_argument(
+    "input",
+    help="Output log from simulator")
+parser.add_argument(
+    "output",
+    help="Doubled log")
+parser.add_argument(
+    "-v",
+    "--verbose",
+    dest="verbose",
+    help="Verbose output",
+    default=False,
+    action="store_true",
+)
+args = parser.parse_args()
+infile = args.input
+outfile = args.output
+
+#  Indices for words in a line.
+sticcol = 0
+eticcol = 1
+dticcol = 2
+stepcol = 3
+rankcol = 4
+otherrankcol = 5
+typecol = 6
+itypecol = 7
+subtypecol = 8
+isubtypecol = 9
+activationcol = 10
+tagcol = 11
+sizecol = 12
+sumcol = 13
+
+# The plan is to pair the sends and receives across all ranks so that
+# we can split each pair into two and give everything new tags that also
+# match.
+
+#  Keyed lines.
+keysends = {}
+keyrecvs = {}
+
+#  Indexed lines.
+sends = []
+recvs = []
+nsends = 0
+nrecvs = 0
+
+#  Generate keys that are unique between send/recv pairs and gather the
+#  associated lines initiation lines.
+with open(infile, "r") as fp:
+    for line in fp:
+        if line[0] == '#':
+            continue
+        words = line.split()
+        if words[activationcol] == 0:
+            continue
+
+        if words[itypecol] == "22":
+            key = words[otherrankcol] + "/" + \
+                  words[rankcol] + "/" + \
+                  words[isubtypecol] + "/" + \
+                  words[tagcol] + "/" + \
+                  words[sizecol]
+            if not key in keysends:
+                keysends[key] = [nsends]
+            else:
+                keysends[key].append(nsends)
+            sends.append(words)
+            nsends = nsends + 1
+
+        elif words[itypecol] == "23":
+            key = words[rankcol] + "/" + \
+                  words[otherrankcol] + "/" + \
+                  words[isubtypecol] + "/" + \
+                  words[tagcol] + "/" + \
+                  words[sizecol]
+            if not key in keyrecvs:
+                keyrecvs[key] = [nrecvs]
+            else:
+                keyrecvs[key].append(nrecvs)
+            recvs.append(words)
+            nrecvs = nrecvs + 1
+
+print "# Read " + str(nsends) + " sends and " + str(nrecvs) + " recvs"
+
+# Now get the indices of the matches.
+msends = [None] * nsends
+for key in keysends:
+    if key in keyrecvs:
+        isend = keysends[key][0]
+        irecv = keyrecvs[key][0]
+        msends[isend] = irecv
+
+# Reorder recvs to same order as sends.
+recvs = [recvs[i] for i in msends]
+
+# Now we can split and retag.
+splits = []
+ctag = 0
+for i in range(nsends):
+    swords = sends[i]
+    rwords = recvs[i]
+    size = str(float(swords[sizecol]) / 2.0)
+    swords[sizecol] = size
+    rwords[sizecol] = size
+
+    ctag = ctag + 1
+    stag = str(ctag)
+    swords[tagcol] = stag
+    rwords[tagcol] = stag
+    splits.append(swords)
+    splits.append(rwords)
+
+    ctag = ctag + 1
+    stag = str(ctag)
+    swords[tagcol] = stag
+    rwords[tagcol] = stag
+    splits.append(swords)
+    splits.append(rwords)
+
+# Sort by tic.
+splits = sorted(splits, key=lambda x: int(x[sticcol]))
+
+#  And output.
+with open(outfile, "w") as fp:
+    fp.write("# stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum\n")
+    for line in splits:
+        fp.write(" ".join(line) + "\n")
+
+sys.exit(0)

testdata/pairdata.py

+#!/usr/bin/env python
+"""
+Usage:
+    pairdata.py [options] output-log paired-log
+
+Combine pairs of send and recv log data entries to reduce the number of
+messages that we send to represent the data of a step. Uses a simple strategy
+identifying sends and recvs of the same ranks and type/subtypes that are close
+in time and combining them. We need to keep the tags and sizes matched across
+the ranks so we pick the tag of the first in time and sum the sizes.
+
+This file is part of SWIFT.
+
+Copyright (C) 2019 Peter W. Draper (p.w.draper@durham.ac.uk)
+All Rights Reserved.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published
+by the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+import argparse
+import sys
+from operator import itemgetter
+
+#  Handle the command line.
+parser = argparse.ArgumentParser(description="Pair MPI logs")
+
+parser.add_argument(
+    "input",
+    help="Output log from simulator")
+parser.add_argument(
+    "output",
+    help="Paired log")
+parser.add_argument(
+    "-v",
+    "--verbose",
+    dest="verbose",
+    help="Verbose output",
+    default=False,
+    action="store_true",
+)
+args = parser.parse_args()
+infile = args.input
+outfile = args.output
+
+#  Indices for words in a line.
+sticcol = 0
+eticcol = 1
+dticcol = 2
+stepcol = 3
+rankcol = 4
+otherrankcol = 5
+typecol = 6
+itypecol = 7
+subtypecol = 8
+isubtypecol = 9
+activationcol = 10
+tagcol = 11
+sizecol = 12
+sumcol = 13
+
+#  Indexed lines.
+lines = []
+nlines = 0
+
+#  Gather lines from the input log. We only want activation lines.
+with open(infile, "r") as fp:
+    for line in fp:
+        if line[0] == '#':
+            continue
+        words = line.split()
+        if words[activationcol] == "1":
+            lines.append(words)
+            nlines = nlines + 1
+
+#  Sort by tag as we need a stable join (sends and recvs are not ordered).
+lines = sorted(lines, key=lambda x: float(x[tagcol]))
+if args.verbose:
+    print "# Read ", nlines, " activation logs from ", infile
+
+# Now locate pairs.
+nmatches = 0
+nnlines = 0
+with open(outfile, "w") as fp:
+    for n in range(0, nlines):
+        nwords = lines[n]
+
+        # If not already used.
+        if nwords[rankcol] == "-1":
+            continue
+
+        # Check remaining lines for a match.
+        for m in range(n + 1, nlines):
+            mwords = lines[m]
+
+            if (nwords[rankcol] == mwords[rankcol] and
+                nwords[otherrankcol] == mwords[otherrankcol] and
+                nwords[itypecol] == mwords[itypecol] and
+                nwords[isubtypecol] == mwords[isubtypecol]):
+
+                #  Matching send or recv of same type to and from same
+                #  ranks. Use sum of sizes and min of tags and tics.
+                nwords[sizecol] = str(int(nwords[sizecol]) + int(mwords[sizecol]))
+                nwords[tagcol] = str(min(int(nwords[tagcol]), int(mwords[tagcol])))
+                nwords[sticcol] = str(min(int(nwords[sticcol]), int(mwords[sticcol])))
+
+                #  Don't use this other entry again. Will be skipped or
+                #  fail to match.
+                mwords[rankcol] = "-1"
+                nmatches = nmatches + 1
+                break
+
+        # And output.
+        fp.write(" ".join(nwords) + "\n")
+        nnlines = nnlines + 1
+
+if args.verbose:
+    print "# Matched ", nmatches, " wrote ", nnlines, " lines to ", outfile
+
+sys.exit(0)