Merge branch 'task-dumps' into 'master'

Update task dump processing

Fix some issues in autoranging, add new script to output simple stats
for pre-thread, step and rank. Add support for new drift_grav task.
Incorporate stats in higher-level processing scripts so that they
can be seen in the HTML pages.

See merge request !347

examples/analyse_tasks.py

+#!/usr/bin/env python
+"""
+Usage:
+    analsyse_tasks.py [options] input.dat
+
+where input.dat is a thread info file for a step.  Use the '-y interval' flag
+of the swift command to create these.
+
+The output is an analysis of the task timings, including deadtime per thread
+and step, total amount of time spent for each task type, for the whole step
+and per thread and the minimum and maximum times spent per task type.
+
+This file is part of SWIFT.
+Copyright (c) 2017 Peter W. Draper (p.w.draper@durham.ac.uk)
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published
+by the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.collections as collections
+import matplotlib.ticker as plticker
+import pylab as pl
+import sys
+import argparse
+
+#  Handle the command line.
+parser = argparse.ArgumentParser(description="Analyse task dumps")
+
+parser.add_argument("input", help="Thread data file (-y output)")
+parser.add_argument("-v", "--verbose", dest="verbose",
+                    help="Verbose output (default: False)",
+                    default=False, action="store_true")
+
+args = parser.parse_args()
+infile = args.input
+
+#  Tasks and subtypes. Indexed as in tasks.h.
+TASKTYPES = ["none", "sort", "self", "pair", "sub_self", "sub_pair",
+             "init_grav", "ghost", "extra_ghost", "drift_part",
+             "drift_gpart", "kick1", "kick2", "timestep", "send", "recv",
+             "grav_top_level", "grav_long_range", "grav_mm", "grav_down",
+             "cooling", "sourceterms", "count"]
+
+SUBTYPES = ["none", "density", "gradient", "force", "grav", "external_grav",
+            "tend", "xv", "rho", "gpart", "multipole", "spart", "count"]
+
+#  Read input.
+data = pl.loadtxt( infile )
+
+nthread = int(max(data[:,0])) + 1
+print "Number of threads:", nthread
+
+#  Recover the start and end time
+full_step = data[0,:]
+tic_step = int(full_step[4])
+toc_step = int(full_step[5])
+CPU_CLOCK = float(full_step[-1]) / 1000.0
+data = data[1:,:]
+if args.verbose:
+    print "CPU frequency:", CPU_CLOCK * 1000.0
+
+#  Avoid start and end times of zero.
+data = data[data[:,4] != 0]
+data = data[data[:,5] != 0]
+
+#  Calculate the time range.
+total_t = (toc_step - tic_step)/ CPU_CLOCK
+print "Data range: ", total_t, "ms"
+
+#  Correct times to relative values.
+start_t = float(tic_step)
+data[:,4] -= start_t
+data[:,5] -= start_t
+
+tasks = {}
+tasks[-1] = []
+for i in range(nthread):
+    tasks[i] = []
+
+#  Gather into by thread data.
+num_lines = pl.size(data) / 10
+for line in range(num_lines):
+    thread = int(data[line,0])
+    tic = int(data[line,4]) / CPU_CLOCK
+    toc = int(data[line,5]) / CPU_CLOCK
+    tasktype = int(data[line,1])
+    subtype = int(data[line,2])
+
+    tasks[thread].append([tic,toc,tasktype,subtype])
+
+#  Sort by tic.
+for i in range(nthread):
+    tasks[i] = sorted(tasks[i], key=lambda task: task[0])
+
+#  Times per task.
+print "# Task times:"
+print "# {0:<16s}: {1:>7s} {2:>9s} {3:>9s} {4:>9s} {5:>9s} {6:>9s}"\
+      .format("type/subtype", "count","minimum", "maximum",
+              "sum", "mean", "percent")
+alltasktimes = {}
+for i in range(nthread):
+    tasktimes = {}
+    for task in tasks[i]:
+        key = TASKTYPES[task[2]] + "/" + SUBTYPES[task[3]]
+        dt = task[1] - task[0]
+        if not key in tasktimes:
+            tasktimes[key] = []
+        tasktimes[key].append(dt)
+
+        if not key in alltasktimes:
+            alltasktimes[key] = []
+        alltasktimes[key].append(dt)
+
+    print "# Thread : ", i
+    for key in sorted(tasktimes.keys()):
+        taskmin = min(tasktimes[key])
+        taskmax = max(tasktimes[key])
+        tasksum = sum(tasktimes[key])
+        print "{0:18s}: {1:7d} {2:9.4f} {3:9.4f} {4:9.4f} {5:9.4f} {6:9.2f}"\
+              .format(key, len(tasktimes[key]), taskmin, taskmax, tasksum,
+                      tasksum / len(tasktimes[key]), tasksum / total_t * 100.0)
+    print
+
+print "# All threads : "
+for key in sorted(alltasktimes.keys()):
+    taskmin = min(alltasktimes[key])
+    taskmax = max(alltasktimes[key])
+    tasksum = sum(alltasktimes[key])
+    print "{0:18s}: {1:7d} {2:9.4f} {3:9.4f} {4:9.4f} {5:9.4f} {6:9.2f}"\
+          .format(key, len(alltasktimes[key]), taskmin, taskmax, tasksum,
+                  tasksum / len(alltasktimes[key]),
+                  tasksum / (nthread * total_t) * 100.0)
+print
+
+#  Dead times.
+print "# Deadtimes:"
+print "# no.    : {0:>9s} {1:>9s} {2:>9s} {3:>9s} {4:>9s} {5:>9s}"\
+      .format("count", "minimum", "maximum", "sum", "mean", "percent")
+alldeadtimes = []
+for i in range(nthread):
+    deadtimes = []
+    last = 0
+    for task in tasks[i]:
+        dt = task[0] - last
+        deadtimes.append(dt)
+        last = task[1]
+    dt = total_t - last
+    deadtimes.append(dt)
+
+    deadmin = min(deadtimes)
+    deadmax = max(deadtimes)
+    deadsum = sum(deadtimes)
+    print "thread {0:2d}: {1:9d} {2:9.4f} {3:9.4f} {4:9.4f} {5:9.4f} {6:9.2f}"\
+          .format(i, len(deadtimes), deadmin, deadmax, deadsum,
+                  deadsum / len(deadtimes), deadsum / total_t * 100.0)
+    alldeadtimes.extend(deadtimes)
+
+deadmin = min(alldeadtimes)
+deadmax = max(alldeadtimes)
+deadsum = sum(alldeadtimes)
+print "all      : {0:9d} {1:9.4f} {2:9.4f} {3:9.4f} {4:9.4f} {5:9.2f}"\
+      .format(len(alldeadtimes), deadmin, deadmax, deadsum,
+              deadsum / len(alldeadtimes),
+              deadsum / (nthread * total_t ) * 100.0)
+print
+
+
+sys.exit(0)

examples/analyse_tasks_MPI.py

+#!/usr/bin/env python
+"""
+Usage:
+    analsyse_tasks_MPI.py [options] input.dat
+
+where input.dat is a thread info file for an MPI step.  Use the '-y interval'
+flag of the swift command to create these.
+
+The output is an analysis of the task timings, including deadtime per thread
+and step, total amount of time spent for each task type, for the whole step
+and per thread and the minimum and maximum times spent per task type.
+
+This file is part of SWIFT.
+Copyright (c) 2017 Peter W. Draper (p.w.draper@durham.ac.uk)
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published
+by the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.collections as collections
+import matplotlib.ticker as plticker
+import pylab as pl
+import sys
+import argparse
+
+#  Handle the command line.
+parser = argparse.ArgumentParser(description="Analyse task dumps")
+
+parser.add_argument("input", help="Thread data file (-y output)")
+parser.add_argument("-v", "--verbose", dest="verbose",
+                    help="Verbose output (default: False)",
+                    default=False, action="store_true")
+
+args = parser.parse_args()
+infile = args.input
+
+#  Tasks and subtypes. Indexed as in tasks.h.
+TASKTYPES = ["none", "sort", "self", "pair", "sub_self", "sub_pair",
+             "init_grav", "ghost", "extra_ghost", "drift_part",
+             "drift_gpart", "kick1", "kick2", "timestep", "send", "recv",
+             "grav_top_level", "grav_long_range", "grav_mm", "grav_down",
+             "cooling", "sourceterms", "count"]
+
+SUBTYPES = ["none", "density", "gradient", "force", "grav", "external_grav",
+            "tend", "xv", "rho", "gpart", "multipole", "spart", "count"]
+
+#  Read input.
+data = pl.loadtxt( infile )
+
+#  Get the CPU clock to convert ticks into milliseconds.
+full_step = data[0,:]
+CPU_CLOCK = float(full_step[-1]) / 1000.0
+if args.verbose:
+    print "# CPU frequency:", CPU_CLOCK * 1000.0
+
+nranks = int(max(data[:,0])) + 1
+print "# Number of ranks:", nranks
+nthread = int(max(data[:,1])) + 1
+print "# Number of threads:", nthread
+
+#  Avoid start and end times of zero.
+sdata = data[data[:,5] != 0]
+sdata = data[data[:,6] != 0]
+
+#  Now we process all the ranks.
+for rank in range(nranks):
+    print "# Rank", rank
+    data = sdata[sdata[:,0] == rank]
+
+    #  Recover the start and end time
+    full_step = data[0,:]
+    tic_step = int(full_step[5])
+    toc_step = int(full_step[6])
+    data = data[1:,:]
+
+    #  Avoid start and end times of zero.
+    data = data[data[:,5] != 0]
+    data = data[data[:,6] != 0]
+
+    #  Calculate the time range.
+    total_t = (toc_step - tic_step)/ CPU_CLOCK
+    print "# Data range: ", total_t, "ms"
+
+    #  Correct times to relative values.
+    start_t = float(tic_step)
+    data[:,5] -= start_t
+    data[:,6] -= start_t
+    end_t = (toc_step - start_t) / CPU_CLOCK
+
+    tasks = {}
+    tasks[-1] = []
+    for i in range(nthread):
+        tasks[i] = []
+
+    #  Gather into by thread data.
+    num_lines = pl.size(data) / 12
+    for line in range(num_lines):
+        thread = int(data[line,1])
+        tic = int(data[line,5]) / CPU_CLOCK
+        toc = int(data[line,6]) / CPU_CLOCK
+        tasktype = int(data[line,2])
+        subtype = int(data[line,3])
+
+        tasks[thread].append([tic,toc,tasktype,subtype])
+
+    #  Sort by tic.
+    for i in range(nthread):
+        tasks[i] = sorted(tasks[i], key=lambda task: task[0])
+
+    #  Times per task.
+    print "# Task times:"
+    print "# {0:<16s}: {1:>7s} {2:>9s} {3:>9s} {4:>9s} {5:>9s} {6:>9s}"\
+          .format("type/subtype", "count","minimum", "maximum",
+                  "sum", "mean", "percent")
+    alltasktimes = {}
+    for i in range(nthread):
+        tasktimes = {}
+        for task in tasks[i]:
+            key = TASKTYPES[task[2]] + "/" + SUBTYPES[task[3]]
+            dt = task[1] - task[0]
+            if not key in tasktimes:
+                tasktimes[key] = []
+            tasktimes[key].append(dt)
+
+            if not key in alltasktimes:
+                alltasktimes[key] = []
+            alltasktimes[key].append(dt)
+
+        print "# Thread : ", i
+        for key in sorted(tasktimes.keys()):
+            taskmin = min(tasktimes[key])
+            taskmax = max(tasktimes[key])
+            tasksum = sum(tasktimes[key])
+            print "{0:18s}: {1:7d} {2:9.4f} {3:9.4f} {4:9.4f} {5:9.4f} {6:9.2f}"\
+                  .format(key, len(tasktimes[key]), taskmin, taskmax, tasksum,
+                          tasksum / len(tasktimes[key]), tasksum / total_t * 100.0)
+        print
+
+    print "# All threads : "
+    for key in sorted(alltasktimes.keys()):
+        taskmin = min(alltasktimes[key])
+        taskmax = max(alltasktimes[key])
+        tasksum = sum(alltasktimes[key])
+        print "{0:18s}: {1:7d} {2:9.4f} {3:9.4f} {4:9.4f} {5:9.4f} {6:9.2f}"\
+              .format(key, len(alltasktimes[key]), taskmin, taskmax, tasksum,
+                      tasksum / len(alltasktimes[key]),
+                      tasksum / (nthread * total_t) * 100.0)
+    print
+
+    #  Dead times.
+    print "# Deadtimes:"
+    print "# no.    : {0:>9s} {1:>9s} {2:>9s} {3:>9s} {4:>9s} {5:>9s}"\
+          .format("count", "minimum", "maximum", "sum", "mean", "percent")
+    alldeadtimes = []
+    for i in range(nthread):
+        deadtimes = []
+        last = 0
+        for task in tasks[i]:
+            dt = task[0] - last
+            deadtimes.append(dt)
+            last = task[1]
+        dt = total_t - last
+        deadtimes.append(dt)
+
+        deadmin = min(deadtimes)
+        deadmax = max(deadtimes)
+        deadsum = sum(deadtimes)
+        print "thread {0:2d}: {1:9d} {2:9.4f} {3:9.4f} {4:9.4f} {5:9.4f} {6:9.2f}"\
+              .format(i, len(deadtimes), deadmin, deadmax, deadsum,
+                      deadsum / len(deadtimes), deadsum / total_t * 100.0)
+        alldeadtimes.extend(deadtimes)
+
+    deadmin = min(alldeadtimes)
+    deadmax = max(alldeadtimes)
+    deadsum = sum(alldeadtimes)
+    print "all      : {0:9d} {1:9.4f} {2:9.4f} {3:9.4f} {4:9.4f} {5:9.2f}"\
+          .format(len(alldeadtimes), deadmin, deadmax, deadsum,
+                  deadsum / len(alldeadtimes),
+              deadsum / (nthread * total_t ) * 100.0)
+    print
+
+
+sys.exit(0)

examples/plot_tasks.py

@@ -89,9 +89,10 @@ pl.rcParams.update(PLOT_PARAMS)
 
 #  Tasks and subtypes. Indexed as in tasks.h.
 TASKTYPES = ["none", "sort", "self", "pair", "sub_self", "sub_pair",
-             "init_grav", "ghost", "extra_ghost", "drift", "kick1", "kick2",
-             "timestep", "send", "recv", "grav_top_level", "grav_long_range",
-             "grav_mm", "grav_down", "cooling", "sourceterms", "count"]
+             "init_grav", "ghost", "extra_ghost", "drift_part",
+             "drift_gpart", "kick1", "kick2", "timestep", "send", "recv",
+             "grav_top_level", "grav_long_range", "grav_mm", "grav_down",
+             "cooling", "sourceterms", "count"]
 
 SUBTYPES = ["none", "density", "gradient", "force", "grav", "external_grav",
             "tend", "xv", "rho", "gpart", "multipole", "spart", "count"]
@@ -105,14 +106,14 @@ FULLTYPES = ["self/force", "self/density", "self/grav", "sub_self/force",
 
 #  A number of colours for the various types. Recycled when there are
 #  more task types than colours...
-
 colours = ["cyan", "lightgray", "darkblue", "yellow", "tan", "dodgerblue",
-           "sienna", "aquamarine", "bisque", "blue", "green", "brown",
-           "purple", "mocassin", "olivedrab", "chartreuse", "darksage",
-           "darkgreen", "green", "mediumseagreen", "mediumaquamarine",
-           "darkslategrey", "mediumturquoise", "black", "cadetblue", "skyblue",
-           "red", "slategray", "gold", "slateblue", "blueviolet",
-           "mediumorchid", "firebrick", "magenta", "hotpink", "pink"]
+           "sienna", "aquamarine", "bisque", "blue", "green", "lightgreen",
+           "brown", "purple", "moccasin", "olivedrab", "chartreuse",
+           "darksage", "darkgreen", "green", "mediumseagreen",
+           "mediumaquamarine", "darkslategrey", "mediumturquoise",
+           "black", "cadetblue", "skyblue", "red", "slategray", "gold",
+           "slateblue", "blueviolet", "mediumorchid", "firebrick",
+           "magenta", "hotpink", "pink", "orange", "lightgreen"]
 maxcolours = len(colours)
 
 #  Set colours of task/subtype.
@@ -134,9 +135,9 @@ for task in SUBTYPES:
 #  For fiddling with colours...
 if args.verbose:
     print "#Selected colours:"
-    for task in TASKCOLOURS.keys():
+    for task in sorted(TASKCOLOURS.keys()):
         print "# " + task + ": " + TASKCOLOURS[task]
-    for task in SUBCOLOURS.keys():
+    for task in sorted(SUBCOLOURS.keys()):
         print "# " + task + ": " + SUBCOLOURS[task]
 
 #  Read input.
@@ -161,7 +162,7 @@ data = data[data[:,5] != 0]
 #  Calculate the time range, if not given.
 delta_t = delta_t * CPU_CLOCK
 if delta_t == 0:
-    dt = max(data[:,5]) - min(data[:,4])
+    dt = toc_step - tic_step
     if dt > delta_t:
         delta_t = dt
     print "Data range: ", delta_t / CPU_CLOCK, "ms"

examples/plot_tasks_MPI.py

@@ -95,9 +95,10 @@ pl.rcParams.update(PLOT_PARAMS)
 
 #  Tasks and subtypes. Indexed as in tasks.h.
 TASKTYPES = ["none", "sort", "self", "pair", "sub_self", "sub_pair",
-             "init_grav", "ghost", "extra_ghost", "drift", "kick1", "kick2",
-             "timestep", "send", "recv", "grav_top_level", "grav_long_range",
-             "grav_mm", "grav_down", "cooling", "sourceterms", "count"]
+             "init_grav", "ghost", "extra_ghost", "drift_part", "drift_gpart",
+             "kick1", "kick2", "timestep", "send", "recv", "grav_top_level",
+             "grav_long_range", "grav_mm", "grav_down", "cooling",
+             "sourceterms", "count"]
 
 SUBTYPES = ["none", "density", "gradient", "force", "grav", "external_grav",
             "tend", "xv", "rho", "gpart", "multipole", "spart", "count"]
@@ -111,15 +112,14 @@ FULLTYPES = ["self/force", "self/density", "self/grav", "sub_self/force",
 
 #  A number of colours for the various types. Recycled when there are
 #  more task types than colours...
-
 colours = ["cyan", "lightgray", "darkblue", "yellow", "tan", "dodgerblue",
-           "sienna", "aquamarine", "bisque", "blue", "green", "brown",
-           "purple", "mocassin", "olivedrab", "chartreuse", "darksage",
-           "darkgreen", "green", "mediumseagreen", "mediumaquamarine",
-           "darkslategrey", "mediumturquoise", "black", "cadetblue", "skyblue",
-           "red", "slategray", "gold", "slateblue", "blueviolet",
-           "mediumorchid", "firebrick", "magenta", "hotpink", "pink",
-           "orange", "lightgreen"]
+           "sienna", "aquamarine", "bisque", "blue", "green", "lightgreen",
+           "brown", "purple", "moccasin", "olivedrab", "chartreuse",
+           "darksage", "darkgreen", "green", "mediumseagreen",
+           "mediumaquamarine", "darkslategrey", "mediumturquoise",
+           "black", "cadetblue", "skyblue", "red", "slategray", "gold",
+           "slateblue", "blueviolet", "mediumorchid", "firebrick",
+           "magenta", "hotpink", "pink", "orange", "lightgreen"]
 maxcolours = len(colours)
 
 #  Set colours of task/subtype.
@@ -141,9 +141,9 @@ for task in SUBTYPES:
 #  For fiddling with colours...
 if args.verbose:
     print "#Selected colours:"
-    for task in TASKCOLOURS.keys():
+    for task in sorted(TASKCOLOURS.keys()):
         print "# " + task + ": " + TASKCOLOURS[task]
-    for task in SUBCOLOURS.keys():
+    for task in sorted(SUBCOLOURS.keys()):
         print "# " + task + ": " + SUBCOLOURS[task]
 
 #  Read input.
@@ -171,12 +171,14 @@ delta_t = delta_t * CPU_CLOCK
 if delta_t == 0:
     for rank in range(nranks):
         data = sdata[sdata[:,0] == rank]
-        dt = max(data[:,6]) - min(data[:,5])
+        full_step = data[0,:]
+        tic_step = int(full_step[5])
+        toc_step = int(full_step[6])
+        dt = toc_step - tic_step
         if dt > delta_t:
             delta_t = dt
     print "Data range: ", delta_t / CPU_CLOCK, "ms"
 
-
 # Once more doing the real gather and plots this time.
 for rank in range(nranks):
     data = sdata[sdata[:,0] == rank]
@@ -186,6 +188,8 @@ for rank in range(nranks):
     tic_step = int(full_step[5])
     toc_step = int(full_step[6])
     data = data[1:,:]
+    typesseen = []
+    nethread = 0
 
     #  Dummy image for ranks that have no tasks.
     if data.size == 0:

examples/process_plot_tasks

@@ -57,6 +57,7 @@ done
 #  And process them,
 echo "Processing thread info files..."
 echo $list | xargs -P $NPROCS -n 3 /bin/bash -c "./plot_tasks.py --expand 1 --limit $TIMERANGE --width 16 --height 4 \$0 \$2 "
+echo $list | xargs -P $NPROCS -n 3 /bin/bash -c "./analyse_tasks.py \$0 > \$2.stats"
 
 echo "Writing output index.html file"
 #  Construct document - serial.
@@ -75,8 +76,21 @@ echo $list | xargs -n 3 | while read f s g; do
 <h2>Step $s</h2>
 EOF
     cat <<EOF >> index.html
-<a href="step${s}r${i}.png"><img src="step${s}r${i}.png" width=400px/></a>
+<a href="step${s}r${i}.html"><img src="step${s}r${i}.png" width=400px/></a>
 EOF
+    cat <<EOF > step${s}r${i}.html
+ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+<body>
+<img src="step${s}r${i}.png">
+<pre>
+EOF
+cat step${s}r${i}.stats >> step${s}r${i}.html
+cat <<EOF >> step${s}r${i}.html
+</body>
+</html>
+EOF
+
 done
 
 cat <<EOF >> index.html

examples/process_plot_tasks_MPI

@@ -62,6 +62,7 @@ nrank=$(($nrank-1))
 #  And process them,
 echo "Processing thread info files..."
 echo $list | xargs -P $NPROCS -n 3 /bin/bash -c "./plot_tasks_MPI.py --expand 1 --limit $TIMERANGE \$0 \$2 "
+echo $list | xargs -P $NPROCS -n 3 /bin/bash -c "./analyse_tasks_MPI.py \$0 > \$2.stats"
 
 echo "Writing output index.html file"
 #  Construct document - serial.
@@ -78,12 +79,31 @@ EOF
 echo $list | xargs -n 3 | while read f s g; do
     cat <<EOF >> index.html
 <h2>Step $s</h2>
+<ul style="list-style-type:none">
+<li>
 EOF
     for i in $(seq 0 $nrank); do
-        cat <<EOF >> index.html
-<a href="step${s}r${i}.png"><img src="step${s}r${i}.png" width=400px/></a>
-EOF
+        cat <<EOF2 >> index.html
+<a href="step${s}r${i}.html"><img src="step${s}r${i}.png" width=400px/></a>
+EOF2
+    cat <<EOF2 > step${s}r${i}.html
+ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html>
+<body>
+<img src="step${s}r${i}.png">
+<pre>
+EOF2
+cat step${s}r.stats >> step${s}r${i}.html
+cat <<EOF2 >> step${s}r${i}.html
+</pre>
+</body>
+</html>
+EOF2
     done
+cat <<EOF >> index.html
+</li>
+</ul>
+EOF
 done
 
 cat <<EOF >> index.html