Merge branch 'task-plots' into 'master'

Update single node task plots Update the non-MPI task plot scripts to offer same speed ups as the MPI versions. See merge request !83

Merge branch 'task-plots' into 'master'
472bab56 · Matthieu Schaller · a77c1d1b · 50c3c158 · 472bab56 · 472bab56
Commit 472bab56 authored 9 years ago by Matthieu Schaller
--- a/examples/plot_tasks.py
+++ b/examples/plot_tasks.py
-###############################################################################
+#!/usr/bin/env python
- # This file is part of SWIFT.
+"""
- # Copyright (c) 2015 Pedro Gonnet (pedro.gonnet@durham.ac.uk),
+Usage:
- #                    Bert Vandenbroucke (bert.vandenbroucke@ugent.be)
+    plot_tasks.py input.dat output.png [time-range-ms]
- #                    Matthieu Schaller (matthieu.schaller@durham.ac.uk)
- #
+where input.dat is a thread info file for a step.  Use the '-y interval'
- # This program is free software: you can redistribute it and/or modify
+flag of the swift MPI commands to create these. The output plot will be
- # it under the terms of the GNU Lesser General Public License as published
+called 'output.png'. Use the time-range-ms in millisecs to produce
- # by the Free Software Foundation, either version 3 of the License, or
+plots with the same time span.
- # (at your option) any later version.
- #
+This file is part of SWIFT.
- # This program is distributed in the hope that it will be useful,
+Copyright (c) 2015 Pedro Gonnet (pedro.gonnet@durham.ac.uk),
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
+                   Bert Vandenbroucke (bert.vandenbroucke@ugent.be)
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+                   Matthieu Schaller (matthieu.schaller@durham.ac.uk)
- # GNU General Public License for more details.
+          (c) 2016 Peter W. Draper (p.w.draper@durham.ac.uk)
- #
- # You should have received a copy of the GNU Lesser General Public License
+This program is free software: you can redistribute it and/or modify
- # along with this program.  If not, see <http://www.gnu.org/licenses/>.
+it under the terms of the GNU Lesser General Public License as published
- #
+by the Free Software Foundation, either version 3 of the License, or
- ##############################################################################
+(at your option) any later version.
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+You should have received a copy of the GNU Lesser General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
 import matplotlib
+import matplotlib.collections as collections
 matplotlib.use('Agg')
 import pylab as pl
 import numpy as np
 import sys
+#  CPU ticks per second.
 CPU_CLOCK = 2.7e9
-params = {'axes.labelsize': 10,
+#  Basic plot configuration.
-'axes.titlesize': 10,
+PLOT_PARAMS = {"axes.labelsize": 10,
-'text.fontsize': 12,
+               "axes.titlesize": 10,
-'legend.fontsize': 12,
+               "font.size": 12,
-'xtick.labelsize': 10,
+               "legend.fontsize": 12,
-'ytick.labelsize': 10,
+               "xtick.labelsize": 10,
-'text.usetex': True,
+               "ytick.labelsize": 10,
-'figure.figsize' : (12., 4.),
+               "figure.figsize" : (16., 4.),
-'figure.subplot.left'    : 0.03,
+               "figure.subplot.left" : 0.03,
-'figure.subplot.right'   : 0.995  ,
+               "figure.subplot.right" : 0.995,
-'figure.subplot.bottom'  : 0.1  ,
+               "figure.subplot.bottom" : 0.1,
-'figure.subplot.top'     : 0.99  ,
+               "figure.subplot.top" : 0.99,
-'figure.subplot.wspace'  : 0.  ,
+               "figure.subplot.wspace" : 0.,
-'figure.subplot.hspace'  : 0.  ,
+               "figure.subplot.hspace" : 0.,
-'lines.markersize' : 6,
+               "lines.markersize" : 6,
-'lines.linewidth' : 3.,
+               "lines.linewidth" : 3.
-'text.latex.unicode': True
+               }
-}
+pl.rcParams.update(PLOT_PARAMS)
-pl.rcParams.update(params)
-pl.rc('font',**{'family':'sans-serif','sans-serif':['Times']})
+#  Tasks and subtypes. Indexed as in tasks.h.
+TASKTYPES = ["none", "sort", "self", "pair", "sub", "ghost", "kick1", "kick2",
+             "send", "recv", "grav_pp", "grav_mm", "grav_up", "grav_down",
+             "psort", "split_cell", "rewait", "count"]
-types = {"0": "task_none",
-         "1": "task_sort",
+TASKCOLOURS = {"none": "black",
-         "2": "task_self",
+               "sort": "lightblue",
-         "3": "task_pair",
+               "self": "greenyellow",
-         "4": "task_sub",
+               "pair": "navy",
-         "5": "task_ghost",
+               "sub": "hotpink",
-         "6": "task_kick1",
+               "ghost": "cyan",
-         "7": "task_kick2",
+               "kick1": "maroon",
-         "8": "task_send",
+               "kick2": "green",
-         "9": "task_recv",
+               "send": "yellow",
-         "10": "task_grav_pp",
+               "recv": "magenta",
-         "11": "task_grav_mm",
+               "grav_pp": "mediumorchid",
-         "12": "task_grav_up",
+               "grav_mm": "mediumturquoise",
-         "13": "task_grav_down",
+               "grav_up": "mediumvioletred",
-         "14": "task_psort",
+               "grav_down": "mediumnightblue",
-         "15": "task_split_cell",
+               "psort": "steelblue",
-         "16": "task_rewait",
+               "split_cell": "seagreen",
-         "17": "task_count"}
+               "rewait": "olive",
+               "count": "powerblue"}
-subtypes = {"0": "subtask_none",
-            "1": "subtask_density",
+SUBTYPES = ["none", "density", "force", "grav", "count"]
-            "2": "subtask_force",
-            "3": "subtask_grav",
+SUBCOLOURS = {"none": "black",
-            "4": "subtask_count"}
+              "density": "red",
+              "force": "blue",
-# Assign colours for all types.
+              "grav": "indigo",
-colors = ["red","blue","green","yellow","cyan","magenta","black"]
+              "count": "purple"}
-colors = colors + list(matplotlib.colors.cnames)
-index = 0
+#  Show docs if help is requested.
+if len( sys.argv ) == 2 and ( sys.argv[1][0:2] == "-h" or sys.argv[1][0:3] == "--h" ):
-subtypecolors = {}
+    from pydoc import help
-for key in subtypes:
+    help( "__main__" )
-    subtypecolors[subtypes[key]] = colors[index]
+    sys.exit( 0 )
-    print subtypes[key], " = ", colors[index]
-    index = index + 1
+#  Handle command-line.
+if len( sys.argv ) != 3 and len( sys.argv ) != 4:
-taskcolors = {}
+    print "Usage: ", sys.argv[0], "input.dat output.png [time-range-ms]"
-for key in types:
+    sys.exit(1)
-    taskcolors[types[key]] = colors[index]
-    print types[key], " = ", colors[index]
+infile = sys.argv[1]
-    index = index + 1
+outpng = sys.argv[2]
+delta_t = 0
-data = pl.loadtxt("thread_info.dat")
+if len( sys.argv ) == 4:
+    delta_t = int(sys.argv[3]) * CPU_CLOCK / 1000
+#  Read input.
+data = pl.loadtxt( infile )
 nthread = int(max(data[:,0])) + 1
 print "Number of threads:", nthread
-tasks = {}
+# Avoid start and end times of zero.
-tasks[-1] = []
+data = data[data[:,4] != 0]
-for i in range(nthread):
+data = data[data[:,5] != 0]
-    tasks[i] = []
+# Calculate the time range, it not given.
+if delta_t == 0:
+    dt = max(data[:,5]) - min(data[:,4])
+    if dt > delta_t:
+        delta_t = dt
+# Once more doing the real gather and plots this time.
 start_t = min(data[:,4])
-end_t = max(data[:,5])
 data[:,4] -= start_t
 data[:,5] -= start_t
-num_lines = pl.size(data) / 8
+tasks = {}
+tasks[-1] = []
+for i in range(nthread):
+    tasks[i] = []
+num_lines = pl.size(data) / 10
 for line in range(num_lines):
    thread = int(data[line,0])
    tasks[thread].append({})
-    tasks[thread][-1]["type"] = types[ str(int(data[line,1])) ]
+    tasks[thread][-1]["type"] = TASKTYPES[int(data[line,1])]
-    tasks[thread][-1]["subtype"] = subtypes[str(int(data[line,2]))]
+    tasks[thread][-1]["subtype"] = SUBTYPES[int(data[line,2])]
-    tasks[thread][-1]["tic"] = int(data[line,4]) / CPU_CLOCK * 1000
+    tic = int(data[line,4]) / CPU_CLOCK * 1000
-    tasks[thread][-1]["toc"] = int(data[line,5]) / CPU_CLOCK * 1000
+    toc = int(data[line,5]) / CPU_CLOCK * 1000
-    tasks[thread][-1]["t"] = (tasks[thread][-1]["toc"]+tasks[thread][-1]["tic"]) / 2
+    tasks[thread][-1]["tic"] = tic
+    tasks[thread][-1]["toc"] = toc
+    tasks[thread][-1]["t"] = (toc + tic)/ 2
 combtasks = {}
 combtasks[-1] = []
@@ -136,22 +162,68 @@ for thread in range(nthread):
            combtasks[thread][-1]["subtype"] = task["subtype"]
            combtasks[thread][-1]["tic"] = task["tic"]
            combtasks[thread][-1]["toc"] = task["toc"]
-            if task["type"] == 'task_self' or task["type"] == 'task_pair' or task["type"] == 'task_sub':
+            if task["type"] == "self" or task["type"] == "pair" or task["type"] == "sub":
-                combtasks[thread][-1]["color"] = subtypecolors[task["subtype"]]
+                combtasks[thread][-1]["colour"] = SUBCOLOURS[task["subtype"]]
            else:
-                combtasks[thread][-1]["color"] = taskcolors[task["type"]]
+                combtasks[thread][-1]["colour"] = TASKCOLOURS[task["type"]]
            lasttype = task["type"]
        else:
            combtasks[thread][-1]["toc"] = task["toc"]
+typesseen = []
+fig = pl.figure()
+ax = fig.add_subplot(1,1,1)
+ax.set_xlim(0, delta_t * 1.03 * 1000 / CPU_CLOCK)
+ax.set_ylim(0, nthread)
+tictoc = np.zeros(2)
 for i in range(nthread):
-    for task in combtasks[i]:
-        pl.fill_between([task["tic"], task["toc"]], i+0.05, i+0.95, facecolor=task["color"])
-pl.xlabel("Wall clock time [ms]")
-pl.xlim(0, (end_t - start_t)*1.03 * 1000 / CPU_CLOCK)
-pl.ylabel("Thread ID")
-pl.yticks(pl.array(range(nthread)) + 0.5, pl.array(range(nthread)))
-pl.savefig("task_graph.png")
+    #  Collect ranges and colours into arrays.
+    tictocs = np.zeros(len(combtasks[i])*2)
+    colours = np.empty(len(combtasks[i])*2, dtype='object')
+    coloursseen = []
+    j = 0
+    for task in combtasks[i]:
+        tictocs[j] = task["tic"]
+        tictocs[j+1] = task["toc"]
+        colours[j] = task["colour"]
+        colours[j+1] = task["colour"]
+        j = j + 2
+        if task["colour"] not in coloursseen:
+            coloursseen.append(task["colour"])
+        #  Legend support, collections don't add to this.
+        if task["subtype"] != "none":
+            qtask = task["type"] + "/" + task["subtype"]
+        else:
+            qtask = task["type"]
+        if qtask not in typesseen:
+            pl.plot([], [], color=task["colour"], label=qtask)
+            typesseen.append(qtask)
+    #  Now plot each colour, faster to use a mask to select colour ranges.
+    for colour in coloursseen:
+        collection = collections.BrokenBarHCollection.span_where(tictocs, ymin=i+0.05, ymax=i+0.95,
+                                                                 where=colours == colour,
+                                                                 facecolor=colour,
+                                                                 linewidths=0)
+        ax.add_collection(collection)
+#  Legend and room for it.
+nrow = len(typesseen) / 5
+if len(typesseen) * 5 < nrow:
+    nrow = nrow + 1
+ax.fill_between([0, 0], nthread+0.5, nthread + nrow + 0.5, facecolor="white")
+ax.set_ylim(0, nthread + nrow + 1)
+ax.legend(loc=1, shadow=True, mode="expand", ncol=5)
+ax.set_xlabel("Wall clock time [ms]")
+ax.set_ylabel("Thread ID" )
+ax.set_yticks(pl.array(range(nthread)), True)
+pl.show()
+pl.savefig(outpng)
+print "Graphics done, output written to", outpng
+sys.exit(0)
--- a/examples/process_plot_tasks
+++ b/examples/process_plot_tasks
+#!/bin/bash
+#
+# Usage:
+#  process_plot_tasks nprocess time-range-ms
+#
+# Description:
+#  Process all the thread info files in the current directory
+#  creating task graphs for steps and threads.
+#
+#  The input files are created by a run using the "-y interval" flag and
+#  should be named "thread_info-step<n>.dat" in the current directory.
+#  All located files will be processed using "nprocess" concurrent
+#  processes and all plots will have the given time range. An output
+#  HTML file "index.html" will be created to view all the plots.
+#
+#
+# This file is part of SWIFT:
+#
+#  Copyright (C) 2016 Peter W. Draper (p.w.draper@durham.ac.uk)
+#  All Rights Reserved.
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Lesser General Public License as published
+#  by the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#  Handle command-line
+if test "$2" == ""; then
+    echo "Usage: $0 nprocess time-range-ms"
+    exit 1
+fi
+NPROCS=$1
+TIMERANGE=$2
+#  Find all thread info files. Use version sort to get into correct order.
+files=$(ls -v thread_info-step*.dat)
+if test $? != 0; then
+    echo "Failed to find any thread info files"
+    exit 1
+fi
+#  Construct list of names, the step no and names for the graphics.
+list=""
+for f in $files; do
+    s=$(echo $f| sed 's,thread_info-step\(.*\).dat,\1,')
+    list="$list $f $s step${s}r"
+done
+#  And process them,
+echo "Processing thread info files..."
+echo $list | xargs -P $NPROCS -n 3 /bin/bash -c "./plot_tasks.py \$0 \$2 $TIMERANGE"
+echo "Writing output index.html file"
+#  Construct document - serial.
+cat <<EOF > index.html
+ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+  <head>
+    <title>SWIFT task graphs</title>
+  </head>
+  <body>
+  <h1>SWIFT task graphs</h1>
+EOF
+echo $list | xargs -n 3 | while read f s g; do
+    cat <<EOF >> index.html
+<h2>Step $s</h2>
+EOF
+    cat <<EOF >> index.html
+<a href="step${s}r${i}.png"><img src="step${s}r${i}.png" width=400px/></a>
+EOF
+done
+cat <<EOF >> index.html
+  </body>
+</html>
+EOF
+echo "Finished"
+exit