Commit 9a668d20 authored by Matthieu Schaller's avatar Matthieu Schaller
Browse files

Documentation of the extra functions.

parent b484ebee
...@@ -329,7 +329,7 @@ fi ...@@ -329,7 +329,7 @@ fi
AM_CONDITIONAL([HAVEPARALLELHDF5],[test "$have_parallel_hdf5" = "yes"]) AM_CONDITIONAL([HAVEPARALLELHDF5],[test "$have_parallel_hdf5" = "yes"])
# Check for setaffinity. # Check for setaffinity.
AC_CHECK_FUNC(pthread_setaffinity_np, AC_DEFINE([HAVE_SETAFFINITY],[true], AC_CHECK_FUNC(pthread_setaffinity_np, AC_DEFINE([HAVE_SETAFFINITY],[1],
[Defined if pthread_setaffinity_np exists.]) ) [Defined if pthread_setaffinity_np exists.]) )
AM_CONDITIONAL(HAVESETAFFINITY, AM_CONDITIONAL(HAVESETAFFINITY,
[test "$ac_cv_func_pthread_setaffinity_np" = "yes"]) [test "$ac_cv_func_pthread_setaffinity_np" = "yes"])
......
...@@ -116,15 +116,11 @@ int main(int argc, char *argv[]) { ...@@ -116,15 +116,11 @@ int main(int argc, char *argv[]) {
fflush(stdout); fflush(stdout);
#endif #endif
/* Let's pin the main thread */
#if defined(HAVE_SETAFFINITY) && defined(HAVE_LIBNUMA) #if defined(HAVE_SETAFFINITY) && defined(HAVE_LIBNUMA)
if (((ENGINE_POLICY) & if (((ENGINE_POLICY) & engine_policy_setaffinity) ==
engine_policy_setaffinity) == engine_policy_setaffinity) { engine_policy_setaffinity)
/* Ensure the NUMA node on which we initialise (first touch) everything
* doesn't change before engine_init allocates NUMA-local workers.
* Otherwise, we may be scheduled elsewhere between the two times.
*/
engine_pin(); engine_pin();
}
#endif #endif
/* Welcome to SWIFT, you made the right choice */ /* Welcome to SWIFT, you made the right choice */
......
...@@ -65,6 +65,11 @@ const char *engine_policy_names[13] = { ...@@ -65,6 +65,11 @@ const char *engine_policy_names[13] = {
/** The rank of the engine as a global variable (for messages). */ /** The rank of the engine as a global variable (for messages). */
int engine_rank; int engine_rank;
#ifdef HAVE_SETAFFINITY
/** The initial affinity of the main thread (set by engin_pin()) */
static cpu_set_t entry_affinity;
#endif
/** /**
* @brief Link a density/force task to a cell. * @brief Link a density/force task to a cell.
* *
...@@ -2298,20 +2303,30 @@ void engine_split(struct engine *e, struct partition *initial_partition) { ...@@ -2298,20 +2303,30 @@ void engine_split(struct engine *e, struct partition *initial_partition) {
} }
#ifdef HAVE_SETAFFINITY #ifdef HAVE_SETAFFINITY
static cpu_set_t entry_affinity; /**
static bool use_entry_affinity = false; * @brief Returns the initial affinity the main thread is using.
*/
static cpu_set_t *engine_entry_affinity() {
static int use_entry_affinity = 0;
static cpu_set_t *engine_entry_affinity(void) {
if (!use_entry_affinity) { if (!use_entry_affinity) {
pthread_t engine = pthread_self(); pthread_t engine = pthread_self();
pthread_getaffinity_np(engine, sizeof entry_affinity, &entry_affinity); pthread_getaffinity_np(engine, sizeof(entry_affinity), &entry_affinity);
use_entry_affinity = true; use_entry_affinity = 1;
} }
return &entry_affinity; return &entry_affinity;
} }
#endif
void engine_pin(void) { /**
* @brief Ensure the NUMA node on which we initialise (first touch) everything
* doesn't change before engine_init allocates NUMA-local workers.
*/
void engine_pin() {
#ifdef HAVE_SETAFFINITY
cpu_set_t *entry_affinity = engine_entry_affinity(); cpu_set_t *entry_affinity = engine_entry_affinity();
int pin; int pin;
...@@ -2321,16 +2336,25 @@ void engine_pin(void) { ...@@ -2321,16 +2336,25 @@ void engine_pin(void) {
cpu_set_t affinity; cpu_set_t affinity;
CPU_ZERO(&affinity); CPU_ZERO(&affinity);
CPU_SET(pin, &affinity); CPU_SET(pin, &affinity);
if (sched_setaffinity(0, sizeof affinity, &affinity) != 0) { if (sched_setaffinity(0, sizeof(affinity), &affinity) != 0) {
error("failed to set engine's affinity"); error("failed to set engine's affinity");
} }
#else
error("SWIFT was not compiled with support for pinning.");
#endif
} }
static void engine_unpin(void) { /**
* @brief Unpins the main thread.
*/
void engine_unpin() {
#ifdef HAVE_SETAFFINITY
pthread_t main_thread = pthread_self(); pthread_t main_thread = pthread_self();
pthread_setaffinity_np(main_thread, sizeof entry_affinity, &entry_affinity); pthread_setaffinity_np(main_thread, sizeof(entry_affinity), &entry_affinity);
} #else
error("SWIFT was not compiled with support for pinning.");
#endif #endif
}
/** /**
* @brief init an engine with the given number of threads, queues, and * @brief init an engine with the given number of threads, queues, and
...@@ -2388,7 +2412,7 @@ void engine_init(struct engine *e, struct space *s, ...@@ -2388,7 +2412,7 @@ void engine_init(struct engine *e, struct space *s,
if (nr_queues <= 0) nr_queues = e->nr_threads; if (nr_queues <= 0) nr_queues = e->nr_threads;
s->nr_queues = nr_queues; s->nr_queues = nr_queues;
/* Deal with affinity. */ /* Deal with affinity. For now, just figure out the number of cores. */
#if defined(HAVE_SETAFFINITY) #if defined(HAVE_SETAFFINITY)
const int nr_cores = sysconf(_SC_NPROCESSORS_ONLN); const int nr_cores = sysconf(_SC_NPROCESSORS_ONLN);
cpu_set_t *entry_affinity = engine_entry_affinity(); cpu_set_t *entry_affinity = engine_entry_affinity();
......
...@@ -190,5 +190,6 @@ struct link *engine_addlink(struct engine *e, struct link *l, struct task *t); ...@@ -190,5 +190,6 @@ struct link *engine_addlink(struct engine *e, struct link *l, struct task *t);
void engine_print_policy(struct engine *e); void engine_print_policy(struct engine *e);
int engine_is_done(struct engine *e); int engine_is_done(struct engine *e);
void engine_pin(); void engine_pin();
void engine_unpin();
#endif /* SWIFT_ENGINE_H */ #endif /* SWIFT_ENGINE_H */
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment