Skip to content
GitLab
Commit 2bd23266 authored by Matthieu Schaller's avatar Matthieu Schaller
Browse files

Merged from master

parents 418ef669 b46be395
Hide whitespace changes
Inline Side-by-side
configure.ac
View file @ 2bd23266
......@@ -60,6 +60,7 @@ AC_ARG_ENABLE([mpi],
good_mpi="yes"
if test "$enable_mpi" = "yes"; then
AX_MPI([CC="$MPICC" AC_DEFINE(HAVE_MPI, 1, [Define if you have the MPI library.]) ])
MPI_LIBRARY="Unknown MPI"
# Various MPI implementations require additional libraries when also using
# threads. Use mpirun (on PATH) as that seems to be only command with
......@@ -83,14 +84,17 @@ if test "$enable_mpi" = "yes"; then
case "$version" in
*Intel*MPI*)
MPI_THREAD_LIBS="-mt_mpi"
MPI_LIBRARY="Intel MPI"
AC_MSG_RESULT([Intel MPI])
;;
*Platform*)
MPI_THREAD_LIBS="-lmtmpi"
MPI_LIBRARY="PLATFORM MPI"
AC_MSG_RESULT([PLATFORM MPI])
;;
*"Open MPI"*)
MPI_THREAD_LIBS=""
MPI_LIBRARY="Open MPI"
AC_MSG_RESULT([Open MPI])
# OpenMPI should be 1.8.6 or later, if not complain.
# Version is last word on first line of -version output.
......@@ -109,6 +113,7 @@ if test "$enable_mpi" = "yes"; then
esac
AC_SUBST([MPI_THREAD_LIBS])
fi
AC_DEFINE_UNQUOTED([SWIFT_MPI_LIBRARY], ["$MPI_LIBRARY"], [The MPI library name, if known.])
fi
AM_CONDITIONAL([HAVEMPI],[test $enable_mpi = "yes"])
......@@ -265,6 +270,9 @@ AC_CHECK_FUNC(pthread_setaffinity_np, AC_DEFINE([HAVE_SETAFFINITY],[true],
AM_CONDITIONAL(HAVESETAFFINITY,
[test "$ac_cv_func_pthread_setaffinity_np" = "yes"])
# Check for libnuma.
AC_CHECK_LIB([numa], [numa_available])
# Check for timing functions needed by cycle.h.
AC_HEADER_TIME
AC_CHECK_HEADERS([sys/time.h c_asm.h intrinsics.h mach/mach_time.h])
......
examples/main.c
View file @ 2bd23266
......@@ -79,6 +79,7 @@ int main(int argc, char *argv[]) {
struct engine e;
struct UnitSystem us;
char ICfileName[200] = "";
char dumpfile[30];
float dt_max = 0.0f;
ticks tic;
int nr_nodes = 1, myrank = 0, grid[3] = {1, 1, 1};
......@@ -115,11 +116,26 @@ int main(int argc, char *argv[]) {
/* Greeting message */
if (myrank == 0) greetings();
#if defined(HAVE_SETAFFINITY) && defined(HAVE_LIBNUMA) && defined(_GNU_SOURCE)
/* Ensure the NUMA node on which we initialise (first touch) everything
* doesn't change before engine_init allocates NUMA-local workers. Otherwise,
* we may be scheduled elsewhere between the two times.
*/
cpu_set_t affinity;
CPU_ZERO(&affinity);
CPU_SET(sched_getcpu(), &affinity);
if (sched_setaffinity(0, sizeof(cpu_set_t), &affinity) != 0) {
message("failed to set entry thread's affinity");
} else {
message("set entry thread's affinity");
}
#endif
/* Init the space. */
bzero(&s, sizeof(struct space));
/* Parse the options */
while ((c = getopt(argc, argv, "a:c:d:f:g:m:q:r:s:t:w:yz:")) != -1)
while ((c = getopt(argc, argv, "a:c:d:f:g:m:q:r:s:t:w:y:z:")) != -1)
switch (c) {
case 'a':
if (sscanf(optarg, "%lf", &scaling) != 1)
......@@ -178,7 +194,8 @@ int main(int argc, char *argv[]) {
if (myrank == 0) message("sub size set to %i.", space_subsize);
break;
case 'y':
dump_tasks = 1;
if(sscanf(optarg, "%d", &dump_tasks) != 1)
error("Error parsing dump_tasks (-y)");
break;
case 'z':
if (sscanf(optarg, "%d", &space_splitsize) != 1)
......@@ -414,6 +431,69 @@ int main(int argc, char *argv[]) {
#endif
}
/* Dump the task data using the given frequency. */
if (dump_tasks && (dump_tasks == 1 || j % dump_tasks == 1)) {
#ifdef WITH_MPI
/* Make sure output file is empty, only on one rank. */
sprintf(dumpfile, "thread_info_MPI-step%d.dat", j);
if (myrank == 0) {
file_thread = fopen(dumpfile, "w");
fclose(file_thread);
}
MPI_Barrier(MPI_COMM_WORLD);
for (int i = 0; i < nr_nodes; i++) {
/* Rank 0 decides the index of writing node, this happens one-by-one. */
int kk = i;
MPI_Bcast(&kk, 1, MPI_INT, 0, MPI_COMM_WORLD);
if (i == myrank) {
/* Open file and position at end. */
file_thread = fopen(dumpfile, "a");
fprintf(file_thread, " %03i 0 0 0 0 %lli 0 0 0 0\n", myrank,
e.tic_step);
int count = 0;
for (int l = 0; l < e.sched.nr_tasks; l++)
if (!e.sched.tasks[l].skip && !e.sched.tasks[l].implicit) {
fprintf(
file_thread, " %03i %i %i %i %i %lli %lli %i %i %i\n", myrank,
e.sched.tasks[l].rid, e.sched.tasks[l].type,
e.sched.tasks[l].subtype, (e.sched.tasks[l].cj == NULL),
e.sched.tasks[l].tic, e.sched.tasks[l].toc,
(e.sched.tasks[l].ci != NULL) ? e.sched.tasks[l].ci->count : 0,
(e.sched.tasks[l].cj != NULL) ? e.sched.tasks[l].cj->count : 0,
e.sched.tasks[l].flags);
fflush(stdout);
count++;
}
message("rank %d counted %d tasks", myrank, count);
fclose(file_thread);
}
/* And we wait for all to synchronize. */
MPI_Barrier(MPI_COMM_WORLD);
}
#else
sprintf(dumpfile, "thread_info-step%d.dat", j);
file_thread = fopen(dumpfile, "w");
for (int l = 0; l < e.sched.nr_tasks; l++)
if (!e.sched.tasks[l].skip && !e.sched.tasks[l].implicit)
fprintf(file_thread, " %i %i %i %i %lli %lli %i %i\n",
e.sched.tasks[l].rid, e.sched.tasks[l].type,
e.sched.tasks[l].subtype, (e.sched.tasks[l].cj == NULL),
e.sched.tasks[l].tic, e.sched.tasks[l].toc,
(e.sched.tasks[l].ci == NULL) ? 0 : e.sched.tasks[l].ci->count,
(e.sched.tasks[l].cj == NULL) ? 0 : e.sched.tasks[l].cj->count);
fclose(file_thread);
#endif
}
/* Dump a line of aggregate output. */
/* if (myrank == 0) { */
/* printf("%i %e %.16e %.16e %.16e %.3e %.3e %i %.3e %.3e", j, e.time,
......@@ -447,43 +527,6 @@ int main(int argc, char *argv[]) {
(double)runner_hist_bins[k]);
#endif
/* Dump the task data. */
if (dump_tasks) {
#ifdef WITH_MPI
file_thread = fopen("thread_info_MPI.dat", "w");
for (j = 0; j < nr_nodes; j++) {
MPI_Barrier(MPI_COMM_WORLD);
if (j == myrank) {
fprintf(file_thread, " %03i 0 0 0 0 %lli 0 0 0 0\n", myrank,
e.tic_step);
for (k = 0; k < e.sched.nr_tasks; k++)
if (!e.sched.tasks[k].skip && !e.sched.tasks[k].implicit)
fprintf(
file_thread, " %03i %i %i %i %i %lli %lli %i %i %i\n", myrank,
e.sched.tasks[k].rid, e.sched.tasks[k].type,
e.sched.tasks[k].subtype, (e.sched.tasks[k].cj == NULL),
e.sched.tasks[k].tic, e.sched.tasks[k].toc,
e.sched.tasks[k].ci->count,
(e.sched.tasks[k].cj != NULL) ? e.sched.tasks[k].cj->count : 0,
e.sched.tasks[k].flags);
fflush(stdout);
sleep(1);
}
}
fclose(file_thread);
#else
file_thread = fopen("thread_info.dat", "w");
for (k = 0; k < e.sched.nr_tasks; k++)
if (!e.sched.tasks[k].skip && !e.sched.tasks[k].implicit)
fprintf(file_thread, " %i %i %i %i %lli %lli %i %i\n",
e.sched.tasks[k].rid, e.sched.tasks[k].type,
e.sched.tasks[k].subtype, (e.sched.tasks[k].cj == NULL),
e.sched.tasks[k].tic, e.sched.tasks[k].toc,
e.sched.tasks[k].ci->count,
(e.sched.tasks[k].cj == NULL) ? 0 : e.sched.tasks[k].cj->count);
fclose(file_thread);
#endif
}
if (with_outputs) {
/* Write final output. */
......
examples/plot_tasks.py
View file @ 2bd23266
......@@ -3,27 +3,28 @@
# Copyright (c) 2015 Pedro Gonnet (pedro.gonnet@durham.ac.uk),
# Bert Vandenbroucke (bert.vandenbroucke@ugent.be)
# Matthieu Schaller (matthieu.schaller@durham.ac.uk)
#
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
#
##############################################################################
import matplotlib
matplotlib.use('Agg')
import pylab as pl
#import numpy as np
import numpy as np
import sys
CPU_CLOCK = 2.7e9
......@@ -50,18 +51,47 @@ pl.rc('font',**{'family':'sans-serif','sans-serif':['Times']})
types = {"0": "task_none", "1": "task_sort", "2": "task_self", "3": "task_pair",
"4": "task_pair", "5": "task_sub", "6": "task_ghost",
"7": "task_kick1", "8": "task_kick2", "9": "task_send",
"10": "task_recv", "11": "task_link", "12": "task_grav_pp",
"13": "task_grav_mm", "14": "task_grav_up", "15": "task_grav_down"}
subtypes = {"0": "subtask_none", "1": "subtask_density", "2": "subtask_force",
"3": "subtask_grav"}
colors = {"subtask_density": "g", "subtask_force": "b", "subtask_grav": "y", "subtask_none": "k"}
colors2 = {"task_sort": "r", "task_kick1": "m"}
types = {"0": "task_none",
"1": "task_sort",
"2": "task_self",
"3": "task_pair",
"4": "task_sub",
"5": "task_ghost",
"6": "task_kick1",
"7": "task_kick2",
"8": "task_send",
"9": "task_recv",
"10": "task_grav_pp",
"11": "task_grav_mm",
"12": "task_grav_up",
"13": "task_grav_down",
"14": "task_psort",
"15": "task_split_cell",
"16": "task_rewait",
"17": "task_count"}
subtypes = {"0": "subtask_none",
"1": "subtask_density",
"2": "subtask_force",
"3": "subtask_grav",
"4": "subtask_count"}
# Assign colours for all types.
colors = ["red","blue","green","yellow","cyan","magenta","black"]
colors = colors + list(matplotlib.colors.cnames)
index = 0
subtypecolors = {}
for key in subtypes:
subtypecolors[subtypes[key]] = colors[index]
print subtypes[key], " = ", colors[index]
index = index + 1
taskcolors = {}
for key in types:
taskcolors[types[key]] = colors[index]
print types[key], " = ", colors[index]
index = index + 1
data = pl.loadtxt("thread_info.dat")
......@@ -86,7 +116,7 @@ for line in range(num_lines):
tasks[thread][-1]["subtype"] = subtypes[str(int(data[line,2]))]
tasks[thread][-1]["tic"] = int(data[line,4]) / CPU_CLOCK * 1000
tasks[thread][-1]["toc"] = int(data[line,5]) / CPU_CLOCK * 1000
tasks[thread][-1]["t"] = (tasks[thread][-1]["toc"]+tasks[thread][-1]["tic"]) / 2
tasks[thread][-1]["t"] = (tasks[thread][-1]["toc"]+tasks[thread][-1]["tic"]) / 2
combtasks = {}
combtasks[-1] = []
......@@ -107,9 +137,9 @@ for thread in range(nthread):
combtasks[thread][-1]["tic"] = task["tic"]
combtasks[thread][-1]["toc"] = task["toc"]
if task["type"] == 'task_self' or task["type"] == 'task_pair' or task["type"] == 'task_sub':
combtasks[thread][-1]["color"] = colors[task["subtype"]]
combtasks[thread][-1]["color"] = subtypecolors[task["subtype"]]
else:
combtasks[thread][-1]["color"] = colors2[task["type"]]
combtasks[thread][-1]["color"] = taskcolors[task["type"]]
lasttype = task["type"]
else:
combtasks[thread][-1]["toc"] = task["toc"]
......
examples/plot_tasks_MPI.py 0 → 100755
View file @ 2bd23266
#!/usr/bin/env python
"""
Usage:
plot_tasks_MPI.py input.dat png-output-prefix [time-range-ms]
where input.dat is a thread info file for a step of an MPI run. Use the '-y
interval' flag of the swift MPI commands to create these. The output plots
will be called 'png-output-prefix<mpi-rank>.png', i.e. one each for all the
threads in each MPI rank. Use the time-range-ms in millisecs to produce
plots with the same time span.
See the command 'process_plot_tasks_MPI' to efficiently wrap this command to
process a number of thread info files and create an HTML file to view them.
This file is part of SWIFT.
Copyright (C) 2015 Pedro Gonnet (pedro.gonnet@durham.ac.uk),
Bert Vandenbroucke (bert.vandenbroucke@ugent.be)
Matthieu Schaller (matthieu.schaller@durham.ac.uk)
Peter W. Draper (p.w.draper@durham.ac.uk)
All Rights Reserved.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU Lesser General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
import matplotlib
import matplotlib.collections as collections
matplotlib.use("Agg")
import pylab as pl
import numpy as np
import sys
# CPU ticks per second.
CPU_CLOCK = 2.7e9
# Basic plot configuration.
PLOT_PARAMS = {"axes.labelsize": 10,
"axes.titlesize": 10,
"font.size": 12,
"legend.fontsize": 12,
"xtick.labelsize": 10,
"ytick.labelsize": 10,
"figure.figsize" : (16., 4.),
"figure.subplot.left" : 0.03,
"figure.subplot.right" : 0.995,
"figure.subplot.bottom" : 0.1,
"figure.subplot.top" : 0.99,
"figure.subplot.wspace" : 0.,
"figure.subplot.hspace" : 0.,
"lines.markersize" : 6,
"lines.linewidth" : 3.
}
pl.rcParams.update(PLOT_PARAMS)
# Tasks and subtypes. Indexed as in tasks.h.
TASKTYPES = ["none", "sort", "self", "pair", "sub", "ghost", "kick1", "kick2",
"send", "recv", "grav_pp", "grav_mm", "grav_up", "grav_down",
"psort", "split_cell", "rewait", "count"]
TASKCOLOURS = {"none": "black",
"sort": "lightblue",
"self": "greenyellow",
"pair": "navy",
"sub": "hotpink",
"ghost": "cyan",
"kick1": "maroon",
"kick2": "green",
"send": "yellow",
"recv": "magenta",
"grav_pp": "mediumorchid",
"grav_mm": "mediumturquoise",
"grav_up": "mediumvioletred",
"grav_down": "mediumnightblue",
"psort": "steelblue",
"split_cell": "seagreen",
"rewait": "olive",
"count": "powerblue"}
SUBTYPES = ["none", "density", "force", "grav", "count"]
SUBCOLOURS = {"none": "black",
"density": "red",
"force": "blue",
"grav": "indigo",
"count": "purple"}
# Show docs if help is requested.
if len( sys.argv ) == 2 and ( sys.argv[1][0:2] == "-h" or sys.argv[1][0:3] == "--h" ):
from pydoc import help
help( "__main__" )
sys.exit( 0 )
# Handle command-line.
if len( sys.argv ) != 3 and len( sys.argv ) != 4:
print "Usage: ", sys.argv[0], "input.dat png-output-prefix [time-range-ms]"
sys.exit(1)
infile = sys.argv[1]
outbase = sys.argv[2]
delta_t = 0
if len( sys.argv ) == 4:
delta_t = int(sys.argv[3]) * CPU_CLOCK / 1000
# Read input.
data = pl.loadtxt( infile )
nranks = int(max(data[:,0])) + 1
print "Number of ranks:", nranks
nthread = int(max(data[:,1])) + 1
print "Number of threads:", nthread
# Avoid start and end times of zero.
sdata = data[data[:,5] != 0]
sdata = sdata[sdata[:,6] != 0]
# Each rank can have different clock (compute node), but we want to use the
# same delta times range for comparisons, so we suck it up and take the hit of
# precalculating this, unless the user knows better.
if delta_t == 0:
for rank in range(nranks):
data = sdata[sdata[:,0] == rank]
dt = max(data[:,6]) - min(data[:,5])
if dt > delta_t:
delta_t = dt
# Once more doing the real gather and plots this time.
for rank in range(nranks):
data = sdata[sdata[:,0] == rank]
start_t = min(data[:,5])
data[:,5] -= start_t
data[:,6] -= start_t
tasks = {}
tasks[-1] = []
for i in range(nthread):
tasks[i] = []
num_lines = pl.size(data) / 10
for line in range(num_lines):
thread = int(data[line,1])
tasks[thread].append({})
tasks[thread][-1]["type"] = TASKTYPES[int(data[line,2])]
tasks[thread][-1]["subtype"] = SUBTYPES[int(data[line,3])]
tic = int(data[line,5]) / CPU_CLOCK * 1000
toc = int(data[line,6]) / CPU_CLOCK * 1000
tasks[thread][-1]["tic"] = tic
tasks[thread][-1]["toc"] = toc
tasks[thread][-1]["t"] = (toc + tic)/ 2
combtasks = {}
combtasks[-1] = []
for i in range(nthread):
combtasks[i] = []
for thread in range(nthread):
tasks[thread] = sorted(tasks[thread], key=lambda l: l["t"])
lasttype = ""
types = []
for task in tasks[thread]:
if task["type"] not in types:
types.append(task["type"])
if lasttype == "" or not lasttype == task["type"]:
combtasks[thread].append({})
combtasks[thread][-1]["type"] = task["type"]
combtasks[thread][-1]["subtype"] = task["subtype"]
combtasks[thread][-1]["tic"] = task["tic"]
combtasks[thread][-1]["toc"] = task["toc"]
if task["type"] == "self" or task["type"] == "pair" or task["type"] == "sub":
combtasks[thread][-1]["colour"] = SUBCOLOURS[task["subtype"]]
else:
combtasks[thread][-1]["colour"] = TASKCOLOURS[task["type"]]
lasttype = task["type"]
else:
combtasks[thread][-1]["toc"] = task["toc"]
typesseen = []
fig = pl.figure()
ax = fig.add_subplot(1,1,1)
ax.set_xlim(0, delta_t * 1.03 * 1000 / CPU_CLOCK)
ax.set_ylim(0, nthread)
tictoc = np.zeros(2)
for i in range(nthread):
# Collect ranges and colours into arrays.
tictocs = np.zeros(len(combtasks[i])*2)
colours = np.empty(len(combtasks[i])*2, dtype='object')
coloursseen = []
j = 0
for task in combtasks[i]:
tictocs[j] = task["tic"]
tictocs[j+1] = task["toc"]
colours[j] = task["colour"]
colours[j+1] = task["colour"]
j = j + 2
if task["colour"] not in coloursseen:
coloursseen.append(task["colour"])
# Legend support, collections don't add to this.
if task["subtype"] != "none":
qtask = task["type"] + "/" + task["subtype"]
else:
qtask = task["type"]
if qtask not in typesseen:
pl.plot([], [], color=task["colour"], label=qtask)
typesseen.append(qtask)
# Now plot each colour, faster to use a mask to select colour ranges.
for colour in coloursseen:
collection = collections.BrokenBarHCollection.span_where(tictocs, ymin=i+0.05, ymax=i+0.95,
where=colours == colour,
facecolor=colour,
linewidths=0)
ax.add_collection(collection)
# Legend and room for it.
nrow = len(typesseen) / 5
if len(typesseen) * 5 < nrow:
nrow = nrow + 1
ax.fill_between([0, 0], nthread+0.5, nthread + nrow + 0.5, facecolor="white")
ax.set_ylim(0, nthread + nrow + 1)
ax.legend(loc=1, shadow=True, mode="expand", ncol=5)
ax.set_xlabel("Wall clock time [ms]")
ax.set_ylabel("Thread ID for MPI Rank " + str(rank) )
ax.set_yticks(pl.array(range(nthread)), True)
pl.show()
outpng = outbase + str(rank) + ".png"
pl.savefig(outpng)
print "Graphics done, output written to", outpng
sys.exit(0)
examples/process_plot_tasks_MPI 0 → 100755
View file @ 2bd23266
#!/bin/bash
#
# Usage:
# process_plot_tasks_MPI nprocess time-range-ms
#
# Description:
# Process all the thread info files in the current directory
# creating task graphs for all MPI ranks and all threads.
#
# The input files are created by a run using the "-y interval" flag and
# should be named "thread_info_MPI-step<n>.dat" in the current directory.
# All located files will be processed using "nprocess" concurrent
# processes and all plots will have the given time range. An output
# HTML file "index.html" will be created to view all the plots.
#
#
# This file is part of SWIFT:
#
# Copyright (C) 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
# All Rights Reserved.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Handle command-line
if test "$2" == ""; then
echo "Usage: $0 nprocess time-range-ms"
exit 1
fi
NPROCS=$1
TIMERANGE=$2
# Find all thread info files. Use version sort to get into correct order.
files=$(ls -v thread_info_MPI-step*.dat)
if test $? != 0; then