Commit 9a668d20 authored by Matthieu Schaller's avatar Matthieu Schaller
Browse files

Documentation of the extra functions.

parent b484ebee
......@@ -329,7 +329,7 @@ fi
AM_CONDITIONAL([HAVEPARALLELHDF5],[test "$have_parallel_hdf5" = "yes"])
# Check for setaffinity.
AC_CHECK_FUNC(pthread_setaffinity_np, AC_DEFINE([HAVE_SETAFFINITY],[true],
AC_CHECK_FUNC(pthread_setaffinity_np, AC_DEFINE([HAVE_SETAFFINITY],[1],
[Defined if pthread_setaffinity_np exists.]) )
AM_CONDITIONAL(HAVESETAFFINITY,
[test "$ac_cv_func_pthread_setaffinity_np" = "yes"])
......
......@@ -116,15 +116,11 @@ int main(int argc, char *argv[]) {
fflush(stdout);
#endif
/* Let's pin the main thread */
#if defined(HAVE_SETAFFINITY) && defined(HAVE_LIBNUMA)
if (((ENGINE_POLICY) &
engine_policy_setaffinity) == engine_policy_setaffinity) {
/* Ensure the NUMA node on which we initialise (first touch) everything
* doesn't change before engine_init allocates NUMA-local workers.
* Otherwise, we may be scheduled elsewhere between the two times.
*/
if (((ENGINE_POLICY) & engine_policy_setaffinity) ==
engine_policy_setaffinity)
engine_pin();
}
#endif
/* Welcome to SWIFT, you made the right choice */
......
......@@ -65,6 +65,11 @@ const char *engine_policy_names[13] = {
/** The rank of the engine as a global variable (for messages). */
int engine_rank;
#ifdef HAVE_SETAFFINITY
/** The initial affinity of the main thread (set by engin_pin()) */
static cpu_set_t entry_affinity;
#endif
/**
* @brief Link a density/force task to a cell.
*
......@@ -2298,20 +2303,30 @@ void engine_split(struct engine *e, struct partition *initial_partition) {
}
#ifdef HAVE_SETAFFINITY
static cpu_set_t entry_affinity;
static bool use_entry_affinity = false;
/**
* @brief Returns the initial affinity the main thread is using.
*/
static cpu_set_t *engine_entry_affinity() {
static int use_entry_affinity = 0;
static cpu_set_t *engine_entry_affinity(void) {
if (!use_entry_affinity) {
pthread_t engine = pthread_self();
pthread_getaffinity_np(engine, sizeof entry_affinity, &entry_affinity);
use_entry_affinity = true;
pthread_getaffinity_np(engine, sizeof(entry_affinity), &entry_affinity);
use_entry_affinity = 1;
}
return &entry_affinity;
}
#endif
void engine_pin(void) {
/**
* @brief Ensure the NUMA node on which we initialise (first touch) everything
* doesn't change before engine_init allocates NUMA-local workers.
*/
void engine_pin() {
#ifdef HAVE_SETAFFINITY
cpu_set_t *entry_affinity = engine_entry_affinity();
int pin;
......@@ -2321,16 +2336,25 @@ void engine_pin(void) {
cpu_set_t affinity;
CPU_ZERO(&affinity);
CPU_SET(pin, &affinity);
if (sched_setaffinity(0, sizeof affinity, &affinity) != 0) {
if (sched_setaffinity(0, sizeof(affinity), &affinity) != 0) {
error("failed to set engine's affinity");
}
#else
error("SWIFT was not compiled with support for pinning.");
#endif
}
static void engine_unpin(void) {
/**
* @brief Unpins the main thread.
*/
void engine_unpin() {
#ifdef HAVE_SETAFFINITY
pthread_t main_thread = pthread_self();
pthread_setaffinity_np(main_thread, sizeof entry_affinity, &entry_affinity);
}
pthread_setaffinity_np(main_thread, sizeof(entry_affinity), &entry_affinity);
#else
error("SWIFT was not compiled with support for pinning.");
#endif
}
/**
* @brief init an engine with the given number of threads, queues, and
......@@ -2388,7 +2412,7 @@ void engine_init(struct engine *e, struct space *s,
if (nr_queues <= 0) nr_queues = e->nr_threads;
s->nr_queues = nr_queues;
/* Deal with affinity. */
/* Deal with affinity. For now, just figure out the number of cores. */
#if defined(HAVE_SETAFFINITY)
const int nr_cores = sysconf(_SC_NPROCESSORS_ONLN);
cpu_set_t *entry_affinity = engine_entry_affinity();
......
......@@ -190,5 +190,6 @@ struct link *engine_addlink(struct engine *e, struct link *l, struct task *t);
void engine_print_policy(struct engine *e);
int engine_is_done(struct engine *e);
void engine_pin();
void engine_unpin();
#endif /* SWIFT_ENGINE_H */
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment