Commit 472bab56 authored by Matthieu Schaller's avatar Matthieu Schaller
Browse files

Merge branch 'task-plots' into 'master'

Update single node task plots

Update the non-MPI task plot scripts to offer same speed ups as the MPI versions.

See merge request !83
parents a77c1d1b 50c3c158
############################################################################### #!/usr/bin/env python
# This file is part of SWIFT. """
# Copyright (c) 2015 Pedro Gonnet (pedro.gonnet@durham.ac.uk), Usage:
# Bert Vandenbroucke (bert.vandenbroucke@ugent.be) plot_tasks.py input.dat output.png [time-range-ms]
# Matthieu Schaller (matthieu.schaller@durham.ac.uk)
# where input.dat is a thread info file for a step. Use the '-y interval'
# This program is free software: you can redistribute it and/or modify flag of the swift MPI commands to create these. The output plot will be
# it under the terms of the GNU Lesser General Public License as published called 'output.png'. Use the time-range-ms in millisecs to produce
# by the Free Software Foundation, either version 3 of the License, or plots with the same time span.
# (at your option) any later version.
# This file is part of SWIFT.
# This program is distributed in the hope that it will be useful, Copyright (c) 2015 Pedro Gonnet (pedro.gonnet@durham.ac.uk),
# but WITHOUT ANY WARRANTY; without even the implied warranty of Bert Vandenbroucke (bert.vandenbroucke@ugent.be)
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the Matthieu Schaller (matthieu.schaller@durham.ac.uk)
# GNU General Public License for more details. (c) 2016 Peter W. Draper (p.w.draper@durham.ac.uk)
#
# You should have received a copy of the GNU Lesser General Public License This program is free software: you can redistribute it and/or modify
# along with this program. If not, see <http://www.gnu.org/licenses/>. it under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
############################################################################## (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
import matplotlib import matplotlib
import matplotlib.collections as collections
matplotlib.use('Agg') matplotlib.use('Agg')
import pylab as pl import pylab as pl
import numpy as np import numpy as np
import sys import sys
# CPU ticks per second.
CPU_CLOCK = 2.7e9 CPU_CLOCK = 2.7e9
params = {'axes.labelsize': 10, # Basic plot configuration.
'axes.titlesize': 10, PLOT_PARAMS = {"axes.labelsize": 10,
'text.fontsize': 12, "axes.titlesize": 10,
'legend.fontsize': 12, "font.size": 12,
'xtick.labelsize': 10, "legend.fontsize": 12,
'ytick.labelsize': 10, "xtick.labelsize": 10,
'text.usetex': True, "ytick.labelsize": 10,
'figure.figsize' : (12., 4.), "figure.figsize" : (16., 4.),
'figure.subplot.left' : 0.03, "figure.subplot.left" : 0.03,
'figure.subplot.right' : 0.995 , "figure.subplot.right" : 0.995,
'figure.subplot.bottom' : 0.1 , "figure.subplot.bottom" : 0.1,
'figure.subplot.top' : 0.99 , "figure.subplot.top" : 0.99,
'figure.subplot.wspace' : 0. , "figure.subplot.wspace" : 0.,
'figure.subplot.hspace' : 0. , "figure.subplot.hspace" : 0.,
'lines.markersize' : 6, "lines.markersize" : 6,
'lines.linewidth' : 3., "lines.linewidth" : 3.
'text.latex.unicode': True }
} pl.rcParams.update(PLOT_PARAMS)
pl.rcParams.update(params)
pl.rc('font',**{'family':'sans-serif','sans-serif':['Times']}) # Tasks and subtypes. Indexed as in tasks.h.
TASKTYPES = ["none", "sort", "self", "pair", "sub", "ghost", "kick1", "kick2",
"send", "recv", "grav_pp", "grav_mm", "grav_up", "grav_down",
"psort", "split_cell", "rewait", "count"]
types = {"0": "task_none",
"1": "task_sort", TASKCOLOURS = {"none": "black",
"2": "task_self", "sort": "lightblue",
"3": "task_pair", "self": "greenyellow",
"4": "task_sub", "pair": "navy",
"5": "task_ghost", "sub": "hotpink",
"6": "task_kick1", "ghost": "cyan",
"7": "task_kick2", "kick1": "maroon",
"8": "task_send", "kick2": "green",
"9": "task_recv", "send": "yellow",
"10": "task_grav_pp", "recv": "magenta",
"11": "task_grav_mm", "grav_pp": "mediumorchid",
"12": "task_grav_up", "grav_mm": "mediumturquoise",
"13": "task_grav_down", "grav_up": "mediumvioletred",
"14": "task_psort", "grav_down": "mediumnightblue",
"15": "task_split_cell", "psort": "steelblue",
"16": "task_rewait", "split_cell": "seagreen",
"17": "task_count"} "rewait": "olive",
"count": "powerblue"}
subtypes = {"0": "subtask_none",
"1": "subtask_density", SUBTYPES = ["none", "density", "force", "grav", "count"]
"2": "subtask_force",
"3": "subtask_grav", SUBCOLOURS = {"none": "black",
"4": "subtask_count"} "density": "red",
"force": "blue",
# Assign colours for all types. "grav": "indigo",
colors = ["red","blue","green","yellow","cyan","magenta","black"] "count": "purple"}
colors = colors + list(matplotlib.colors.cnames)
index = 0 # Show docs if help is requested.
if len( sys.argv ) == 2 and ( sys.argv[1][0:2] == "-h" or sys.argv[1][0:3] == "--h" ):
subtypecolors = {} from pydoc import help
for key in subtypes: help( "__main__" )
subtypecolors[subtypes[key]] = colors[index] sys.exit( 0 )
print subtypes[key], " = ", colors[index]
index = index + 1 # Handle command-line.
if len( sys.argv ) != 3 and len( sys.argv ) != 4:
taskcolors = {} print "Usage: ", sys.argv[0], "input.dat output.png [time-range-ms]"
for key in types: sys.exit(1)
taskcolors[types[key]] = colors[index]
print types[key], " = ", colors[index] infile = sys.argv[1]
index = index + 1 outpng = sys.argv[2]
delta_t = 0
data = pl.loadtxt("thread_info.dat") if len( sys.argv ) == 4:
delta_t = int(sys.argv[3]) * CPU_CLOCK / 1000
# Read input.
data = pl.loadtxt( infile )
nthread = int(max(data[:,0])) + 1 nthread = int(max(data[:,0])) + 1
print "Number of threads:", nthread print "Number of threads:", nthread
tasks = {} # Avoid start and end times of zero.
tasks[-1] = [] data = data[data[:,4] != 0]
for i in range(nthread): data = data[data[:,5] != 0]
tasks[i] = []
# Calculate the time range, it not given.
if delta_t == 0:
dt = max(data[:,5]) - min(data[:,4])
if dt > delta_t:
delta_t = dt
# Once more doing the real gather and plots this time.
start_t = min(data[:,4]) start_t = min(data[:,4])
end_t = max(data[:,5])
data[:,4] -= start_t data[:,4] -= start_t
data[:,5] -= start_t data[:,5] -= start_t
num_lines = pl.size(data) / 8
tasks = {}
tasks[-1] = []
for i in range(nthread):
tasks[i] = []
num_lines = pl.size(data) / 10
for line in range(num_lines): for line in range(num_lines):
thread = int(data[line,0]) thread = int(data[line,0])
tasks[thread].append({}) tasks[thread].append({})
tasks[thread][-1]["type"] = types[ str(int(data[line,1])) ] tasks[thread][-1]["type"] = TASKTYPES[int(data[line,1])]
tasks[thread][-1]["subtype"] = subtypes[str(int(data[line,2]))] tasks[thread][-1]["subtype"] = SUBTYPES[int(data[line,2])]
tasks[thread][-1]["tic"] = int(data[line,4]) / CPU_CLOCK * 1000 tic = int(data[line,4]) / CPU_CLOCK * 1000
tasks[thread][-1]["toc"] = int(data[line,5]) / CPU_CLOCK * 1000 toc = int(data[line,5]) / CPU_CLOCK * 1000
tasks[thread][-1]["t"] = (tasks[thread][-1]["toc"]+tasks[thread][-1]["tic"]) / 2 tasks[thread][-1]["tic"] = tic
tasks[thread][-1]["toc"] = toc
tasks[thread][-1]["t"] = (toc + tic)/ 2
combtasks = {} combtasks = {}
combtasks[-1] = [] combtasks[-1] = []
...@@ -136,22 +162,68 @@ for thread in range(nthread): ...@@ -136,22 +162,68 @@ for thread in range(nthread):
combtasks[thread][-1]["subtype"] = task["subtype"] combtasks[thread][-1]["subtype"] = task["subtype"]
combtasks[thread][-1]["tic"] = task["tic"] combtasks[thread][-1]["tic"] = task["tic"]
combtasks[thread][-1]["toc"] = task["toc"] combtasks[thread][-1]["toc"] = task["toc"]
if task["type"] == 'task_self' or task["type"] == 'task_pair' or task["type"] == 'task_sub': if task["type"] == "self" or task["type"] == "pair" or task["type"] == "sub":
combtasks[thread][-1]["color"] = subtypecolors[task["subtype"]] combtasks[thread][-1]["colour"] = SUBCOLOURS[task["subtype"]]
else: else:
combtasks[thread][-1]["color"] = taskcolors[task["type"]] combtasks[thread][-1]["colour"] = TASKCOLOURS[task["type"]]
lasttype = task["type"] lasttype = task["type"]
else: else:
combtasks[thread][-1]["toc"] = task["toc"] combtasks[thread][-1]["toc"] = task["toc"]
typesseen = []
fig = pl.figure()
ax = fig.add_subplot(1,1,1)
ax.set_xlim(0, delta_t * 1.03 * 1000 / CPU_CLOCK)
ax.set_ylim(0, nthread)
tictoc = np.zeros(2)
for i in range(nthread): for i in range(nthread):
for task in combtasks[i]:
pl.fill_between([task["tic"], task["toc"]], i+0.05, i+0.95, facecolor=task["color"])
pl.xlabel("Wall clock time [ms]")
pl.xlim(0, (end_t - start_t)*1.03 * 1000 / CPU_CLOCK)
pl.ylabel("Thread ID")
pl.yticks(pl.array(range(nthread)) + 0.5, pl.array(range(nthread)))
pl.savefig("task_graph.png")
# Collect ranges and colours into arrays.
tictocs = np.zeros(len(combtasks[i])*2)
colours = np.empty(len(combtasks[i])*2, dtype='object')
coloursseen = []
j = 0
for task in combtasks[i]:
tictocs[j] = task["tic"]
tictocs[j+1] = task["toc"]
colours[j] = task["colour"]
colours[j+1] = task["colour"]
j = j + 2
if task["colour"] not in coloursseen:
coloursseen.append(task["colour"])
# Legend support, collections don't add to this.
if task["subtype"] != "none":
qtask = task["type"] + "/" + task["subtype"]
else:
qtask = task["type"]
if qtask not in typesseen:
pl.plot([], [], color=task["colour"], label=qtask)
typesseen.append(qtask)
# Now plot each colour, faster to use a mask to select colour ranges.
for colour in coloursseen:
collection = collections.BrokenBarHCollection.span_where(tictocs, ymin=i+0.05, ymax=i+0.95,
where=colours == colour,
facecolor=colour,
linewidths=0)
ax.add_collection(collection)
# Legend and room for it.
nrow = len(typesseen) / 5
if len(typesseen) * 5 < nrow:
nrow = nrow + 1
ax.fill_between([0, 0], nthread+0.5, nthread + nrow + 0.5, facecolor="white")
ax.set_ylim(0, nthread + nrow + 1)
ax.legend(loc=1, shadow=True, mode="expand", ncol=5)
ax.set_xlabel("Wall clock time [ms]")
ax.set_ylabel("Thread ID" )
ax.set_yticks(pl.array(range(nthread)), True)
pl.show()
pl.savefig(outpng)
print "Graphics done, output written to", outpng
sys.exit(0)
#!/bin/bash
#
# Usage:
# process_plot_tasks nprocess time-range-ms
#
# Description:
# Process all the thread info files in the current directory
# creating task graphs for steps and threads.
#
# The input files are created by a run using the "-y interval" flag and
# should be named "thread_info-step<n>.dat" in the current directory.
# All located files will be processed using "nprocess" concurrent
# processes and all plots will have the given time range. An output
# HTML file "index.html" will be created to view all the plots.
#
#
# This file is part of SWIFT:
#
# Copyright (C) 2016 Peter W. Draper (p.w.draper@durham.ac.uk)
# All Rights Reserved.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Handle command-line
if test "$2" == ""; then
echo "Usage: $0 nprocess time-range-ms"
exit 1
fi
NPROCS=$1
TIMERANGE=$2
# Find all thread info files. Use version sort to get into correct order.
files=$(ls -v thread_info-step*.dat)
if test $? != 0; then
echo "Failed to find any thread info files"
exit 1
fi
# Construct list of names, the step no and names for the graphics.
list=""
for f in $files; do
s=$(echo $f| sed 's,thread_info-step\(.*\).dat,\1,')
list="$list $f $s step${s}r"
done
# And process them,
echo "Processing thread info files..."
echo $list | xargs -P $NPROCS -n 3 /bin/bash -c "./plot_tasks.py \$0 \$2 $TIMERANGE"
echo "Writing output index.html file"
# Construct document - serial.
cat <<EOF > index.html
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
<title>SWIFT task graphs</title>
</head>
<body>
<h1>SWIFT task graphs</h1>
EOF
echo $list | xargs -n 3 | while read f s g; do
cat <<EOF >> index.html
<h2>Step $s</h2>
EOF
cat <<EOF >> index.html
<a href="step${s}r${i}.png"><img src="step${s}r${i}.png" width=400px/></a>
EOF
done
cat <<EOF >> index.html
</body>
</html>
EOF
echo "Finished"
exit
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment