Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
SWIFT
SWIFTsim
Commits
c8d31089
Commit
c8d31089
authored
Jul 31, 2017
by
Peter W. Draper
Browse files
Add analysis script for the threadpool and make autoranging of plots the default
parent
23b16d6d
Changes
2
Hide whitespace changes
Inline
Side-by-side
examples/analyse_threadpool_tasks.py
0 → 100755
View file @
c8d31089
#!/usr/bin/env python
"""
Usage:
analsyse_threadpool_tasks.py [options] input.dat
where input.dat is a threadpool dump for a step. Use the '-Y interval' flag
of the swift command to create these.
The output is an analysis of the threadpool task timings, including deadtime
per thread and step, total amount of time spent for each task type, for the
whole step and per thread and the minimum and maximum times spent per task
type.
This file is part of SWIFT.
Copyright (c) 2017 Peter W. Draper (p.w.draper@durham.ac.uk)
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
import
matplotlib
matplotlib
.
use
(
"Agg"
)
import
matplotlib.collections
as
collections
import
matplotlib.ticker
as
plticker
import
pylab
as
pl
import
sys
import
argparse
# Handle the command line.
parser
=
argparse
.
ArgumentParser
(
description
=
"Analyse task dumps"
)
parser
.
add_argument
(
"input"
,
help
=
"Threadpool data file (-y output)"
)
parser
.
add_argument
(
"-v"
,
"--verbose"
,
dest
=
"verbose"
,
help
=
"Verbose output (default: False)"
,
default
=
False
,
action
=
"store_true"
)
args
=
parser
.
parse_args
()
infile
=
args
.
input
# Read header. First two lines.
with
open
(
infile
)
as
infid
:
head
=
[
next
(
infid
)
for
x
in
xrange
(
2
)]
header
=
head
[
1
][
2
:].
strip
()
header
=
eval
(
header
)
nthread
=
int
(
header
[
'num_threads'
])
+
1
CPU_CLOCK
=
float
(
header
[
'cpufreq'
])
/
1000.0
print
"Number of threads: "
,
nthread
-
1
if
args
.
verbose
:
print
"CPU frequency:"
,
CPU_CLOCK
*
1000.0
# Read input.
data
=
pl
.
genfromtxt
(
infile
,
dtype
=
None
,
delimiter
=
" "
)
# Mixed types, so need to separate.
tics
=
[]
tocs
=
[]
funcs
=
[]
threads
=
[]
chunks
=
[]
for
i
in
data
:
if
i
[
0
]
!=
"#"
:
funcs
.
append
(
i
[
0
])
if
i
[
1
]
<
0
:
threads
.
append
(
nthread
-
1
)
else
:
threads
.
append
(
i
[
1
])
chunks
.
append
(
i
[
2
])
tics
.
append
(
i
[
3
])
tocs
.
append
(
i
[
4
])
tics
=
pl
.
array
(
tics
)
tocs
=
pl
.
array
(
tocs
)
funcs
=
pl
.
array
(
funcs
)
threads
=
pl
.
array
(
threads
)
chunks
=
pl
.
array
(
chunks
)
# Recover the start and end time
tic_step
=
min
(
tics
)
toc_step
=
max
(
tocs
)
# Calculate the time range.
total_t
=
(
toc_step
-
tic_step
)
/
CPU_CLOCK
print
"# Data range: "
,
total_t
,
"ms"
print
# Correct times to relative millisecs.
start_t
=
float
(
tic_step
)
tics
=
(
tics
-
start_t
)
/
CPU_CLOCK
tocs
=
(
tocs
-
start_t
)
/
CPU_CLOCK
tasks
=
{}
tasks
[
-
1
]
=
[]
for
i
in
range
(
nthread
):
tasks
[
i
]
=
[]
# Gather into by thread data.
for
i
in
range
(
len
(
tics
)):
tasks
[
threads
[
i
]].
append
([
tics
[
i
],
tocs
[
i
],
funcs
[
i
]])
# Don't actually process the fake thread.
nthread
=
nthread
-
1
# Sort by tic and gather used thread ids.
threadids
=
[]
for
i
in
range
(
nthread
):
if
len
(
tasks
[
i
])
>
0
:
tasks
[
i
]
=
sorted
(
tasks
[
i
],
key
=
lambda
task
:
task
[
0
])
threadids
.
append
(
i
)
# Times per task.
print
"# Task times:"
print
"# -----------"
print
"# {0:<31s}: {1:>7s} {2:>9s} {3:>9s} {4:>9s} {5:>9s} {6:>9s}"
\
.
format
(
"type/subtype"
,
"count"
,
"minimum"
,
"maximum"
,
"sum"
,
"mean"
,
"percent"
)
alltasktimes
=
{}
sidtimes
=
{}
for
i
in
threadids
:
tasktimes
=
{}
for
task
in
tasks
[
i
]:
key
=
task
[
2
]
dt
=
task
[
1
]
-
task
[
0
]
if
not
key
in
tasktimes
:
tasktimes
[
key
]
=
[]
tasktimes
[
key
].
append
(
dt
)
if
not
key
in
alltasktimes
:
alltasktimes
[
key
]
=
[]
alltasktimes
[
key
].
append
(
dt
)
print
"# Thread : "
,
i
for
key
in
sorted
(
tasktimes
.
keys
()):
taskmin
=
min
(
tasktimes
[
key
])
taskmax
=
max
(
tasktimes
[
key
])
tasksum
=
sum
(
tasktimes
[
key
])
print
"{0:33s}: {1:7d} {2:9.4f} {3:9.4f} {4:9.4f} {5:9.4f} {6:9.2f}"
\
.
format
(
key
,
len
(
tasktimes
[
key
]),
taskmin
,
taskmax
,
tasksum
,
tasksum
/
len
(
tasktimes
[
key
]),
tasksum
/
total_t
*
100.0
)
print
print
"# All threads : "
for
key
in
sorted
(
alltasktimes
.
keys
()):
taskmin
=
min
(
alltasktimes
[
key
])
taskmax
=
max
(
alltasktimes
[
key
])
tasksum
=
sum
(
alltasktimes
[
key
])
print
"{0:33s}: {1:7d} {2:9.4f} {3:9.4f} {4:9.4f} {5:9.4f} {6:9.2f}"
\
.
format
(
key
,
len
(
alltasktimes
[
key
]),
taskmin
,
taskmax
,
tasksum
,
tasksum
/
len
(
alltasktimes
[
key
]),
tasksum
/
(
len
(
threadids
)
*
total_t
)
*
100.0
)
print
# Dead times.
print
"# Times not in tasks (deadtimes)"
print
"# ------------------------------"
print
"# Time before first task:"
print
"# no. : {0:>9s} {1:>9s}"
.
format
(
"value"
,
"percent"
)
predeadtimes
=
[]
for
i
in
threadids
:
predeadtime
=
tasks
[
i
][
0
][
0
]
print
"thread {0:2d}: {1:9.4f} {2:9.4f}"
\
.
format
(
i
,
predeadtime
,
predeadtime
/
total_t
*
100.0
)
predeadtimes
.
append
(
predeadtime
)
predeadmin
=
min
(
predeadtimes
)
predeadmax
=
max
(
predeadtimes
)
predeadsum
=
sum
(
predeadtimes
)
print
"# : {0:>9s} {1:>9s} {2:>9s} {3:>9s} {4:>9s} {5:>9s}"
\
.
format
(
"count"
,
"minimum"
,
"maximum"
,
"sum"
,
"mean"
,
"percent"
)
print
"all : {0:9d} {1:9.4f} {2:9.4f} {3:9.4f} {4:9.4f} {5:9.2f}"
\
.
format
(
len
(
predeadtimes
),
predeadmin
,
predeadmax
,
predeadsum
,
predeadsum
/
len
(
predeadtimes
),
predeadsum
/
(
len
(
threadids
)
*
total_t
)
*
100.0
)
print
print
"# Time after last task:"
print
"# no. : {0:>9s} {1:>9s}"
.
format
(
"value"
,
"percent"
)
postdeadtimes
=
[]
for
i
in
threadids
:
postdeadtime
=
total_t
-
tasks
[
i
][
-
1
][
1
]
print
"thread {0:2d}: {1:9.4f} {2:9.4f}"
\
.
format
(
i
,
postdeadtime
,
postdeadtime
/
total_t
*
100.0
)
postdeadtimes
.
append
(
postdeadtime
)
postdeadmin
=
min
(
postdeadtimes
)
postdeadmax
=
max
(
postdeadtimes
)
postdeadsum
=
sum
(
postdeadtimes
)
print
"# : {0:>9s} {1:>9s} {2:>9s} {3:>9s} {4:>9s} {5:>9s}"
\
.
format
(
"count"
,
"minimum"
,
"maximum"
,
"sum"
,
"mean"
,
"percent"
)
print
"all : {0:9d} {1:9.4f} {2:9.4f} {3:9.4f} {4:9.4f} {5:9.2f}"
\
.
format
(
len
(
postdeadtimes
),
postdeadmin
,
postdeadmax
,
postdeadsum
,
postdeadsum
/
len
(
postdeadtimes
),
postdeadsum
/
(
len
(
threadids
)
*
total_t
)
*
100.0
)
print
# Time in threadpool, i.e. from first to last tasks.
print
"# Time between tasks (threadpool deadtime):"
print
"# no. : {0:>9s} {1:>9s} {2:>9s} {3:>9s} {4:>9s} {5:>9s}"
\
.
format
(
"count"
,
"minimum"
,
"maximum"
,
"sum"
,
"mean"
,
"percent"
)
threadpooldeadtimes
=
[]
for
i
in
threadids
:
deadtimes
=
[]
last
=
tasks
[
i
][
0
][
0
]
for
task
in
tasks
[
i
]:
dt
=
task
[
0
]
-
last
deadtimes
.
append
(
dt
)
last
=
task
[
1
]
# Drop first value, last value already gone.
if
len
(
deadtimes
)
>
1
:
deadtimes
=
deadtimes
[
1
:]
else
:
# Only one task, so no deadtime by definition.
deadtimes
=
[
0.0
]
deadmin
=
min
(
deadtimes
)
deadmax
=
max
(
deadtimes
)
deadsum
=
sum
(
deadtimes
)
print
"thread {0:2d}: {1:9d} {2:9.4f} {3:9.4f} {4:9.4f} {5:9.4f} {6:9.2f}"
\
.
format
(
i
,
len
(
deadtimes
),
deadmin
,
deadmax
,
deadsum
,
deadsum
/
len
(
deadtimes
),
deadsum
/
total_t
*
100.0
)
threadpooldeadtimes
.
extend
(
deadtimes
)
deadmin
=
min
(
threadpooldeadtimes
)
deadmax
=
max
(
threadpooldeadtimes
)
deadsum
=
sum
(
threadpooldeadtimes
)
print
"all : {0:9d} {1:9.4f} {2:9.4f} {3:9.4f} {4:9.4f} {5:9.2f}"
\
.
format
(
len
(
threadpooldeadtimes
),
deadmin
,
deadmax
,
deadsum
,
deadsum
/
len
(
threadpooldeadtimes
),
deadsum
/
(
len
(
threadids
)
*
total_t
)
*
100.0
)
print
# All times in step.
print
"# All deadtimes:"
print
"# no. : {0:>9s} {1:>9s} {2:>9s} {3:>9s} {4:>9s} {5:>9s}"
\
.
format
(
"count"
,
"minimum"
,
"maximum"
,
"sum"
,
"mean"
,
"percent"
)
alldeadtimes
=
[]
for
i
in
threadids
:
deadtimes
=
[]
last
=
0
for
task
in
tasks
[
i
]:
dt
=
task
[
0
]
-
last
deadtimes
.
append
(
dt
)
last
=
task
[
1
]
dt
=
total_t
-
last
deadtimes
.
append
(
dt
)
deadmin
=
min
(
deadtimes
)
deadmax
=
max
(
deadtimes
)
deadsum
=
sum
(
deadtimes
)
print
"thread {0:2d}: {1:9d} {2:9.4f} {3:9.4f} {4:9.4f} {5:9.4f} {6:9.2f}"
\
.
format
(
i
,
len
(
deadtimes
),
deadmin
,
deadmax
,
deadsum
,
deadsum
/
len
(
deadtimes
),
deadsum
/
total_t
*
100.0
)
alldeadtimes
.
extend
(
deadtimes
)
deadmin
=
min
(
alldeadtimes
)
deadmax
=
max
(
alldeadtimes
)
deadsum
=
sum
(
alldeadtimes
)
print
"all : {0:9d} {1:9.4f} {2:9.4f} {3:9.4f} {4:9.4f} {5:9.2f}"
\
.
format
(
len
(
alldeadtimes
),
deadmin
,
deadmax
,
deadsum
,
deadsum
/
len
(
alldeadtimes
),
deadsum
/
(
len
(
threadids
)
*
total_t
)
*
100.0
)
print
sys
.
exit
(
0
)
examples/process_plot_threadpool
View file @
c8d31089
#!/bin/bash
#
# Usage:
# process_plot_threadpool nprocess time-range-ms
# process_plot_threadpool nprocess
[
time-range-ms
]
#
# Description:
# Process all the threadpool info files in the current directory
...
...
@@ -10,8 +10,8 @@
# The input files are created by a run using the "-Y interval" flag and
# should be named "threadpool_info-step
<n>
.dat" in the current directory.
# All located files will be processed using "nprocess" concurrent
# processes and all plots will have the
given
time range
. An output
# HTML file "index.html" will be created to view all the plots.
# processes and all plots will have the
same
time range
if one is given.
#
An output
HTML file "index.html" will be created to view all the plots.
#
#
# This file is part of SWIFT:
...
...
@@ -33,12 +33,17 @@
# along with this program. If not, see
<http:
//
www.gnu.org
/
licenses
/>
.
# Handle command-line
if test "$
2
" =
=
""; then
echo "Usage: $0 nprocess time-range-ms"
if test "$
1
" = ""; then
echo "Usage: $0 nprocess
[
time-range-ms
]
"
exit 1
fi
NPROCS=$1
TIMERANGE=$2
TIMERANGE=0
LIMIT="(autoranged)"
if test "$2" != ""; then
TIMERANGE=$2
LIMIT=""
fi
# Find all thread info files. Use version sort to get into correct order.
files=$(ls -v threadpool_info-step*.dat)
...
...
@@ -56,25 +61,26 @@ done
# And process them,
echo "Processing threadpool info files..."
echo $list | xargs -P $NPROCS -n 3 /bin/bash -c "./plot_threadpool.py --expand 3 --limit $TIMERANGE --width 16 --height 4 \$0 \$2 "
echo $list | xargs -P $NPROCS -n 3 /bin/bash -c "./plot_threadpool.py --expand 1 --limit $TIMERANGE --width 16 --height 4 \$0 \$2 "
echo $list | xargs -P $NPROCS -n 3 /bin/bash -c "./analyse_threadpool_tasks.py \$0 > \$2.stats"
echo "Writing output index.html file"
echo "Writing output
threadpool-
index.html file"
# Construct document - serial.
cat
<
<
EOF
>
index.html
cat
<
<
EOF
>
threadpool-
index.html
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
<title>
SWIFT t
ask graphs
</title>
<title>
SWIFT t
hreadpool tasks $LIMIT
</title>
</head>
<body>
<h1>
SWIFT t
ask graphs
</h1>
<h1>
SWIFT t
hreadpool tasks $LIMIT
</h1>
EOF
echo $list | xargs -n 3 | while read f s g; do
cat
<
<
EOF
>
> index.html
cat
<
<
EOF
>
>
threadpool-
index.html
<h2>
Step $s
</h2>
EOF
cat
<
<
EOF
>
> index.html
cat
<
<
EOF
>
>
threadpool-
index.html
<a
href=
"poolstep${s}r${i}.html"
><img
src=
"poolstep${s}r${i}.png"
width=
400px/
></a>
EOF
cat
<
<
EOF
>
poolstep${s}r${i}.html
...
...
@@ -84,9 +90,15 @@ EOF
<img
src=
"poolstep${s}r${i}.png"
>
<pre>
EOF
cat poolstep${s}r${i}.stats >> poolstep${s}r${i}.html
cat
<
<
EOF
>
> poolstep${s}r${i}.html
</body>
</html>
EOF
done
cat
<
<
EOF
>
> index.html
cat
<
<
EOF
>
>
threadpool-
index.html
</body>
</html>
EOF
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment