Documentation of the extra functions.

configure.ac

@@ -329,7 +329,7 @@ fi
 AM_CONDITIONAL([HAVEPARALLELHDF5],[test "$have_parallel_hdf5" = "yes"])
 
 # Check for setaffinity.
-AC_CHECK_FUNC(pthread_setaffinity_np, AC_DEFINE([HAVE_SETAFFINITY],[true],
+AC_CHECK_FUNC(pthread_setaffinity_np, AC_DEFINE([HAVE_SETAFFINITY],[1],
     [Defined if pthread_setaffinity_np exists.]) )
 AM_CONDITIONAL(HAVESETAFFINITY,
     [test "$ac_cv_func_pthread_setaffinity_np" = "yes"])

examples/main.c

@@ -116,15 +116,11 @@ int main(int argc, char *argv[]) {
   fflush(stdout);
 #endif
 
+/* Let's pin the main thread */
 #if defined(HAVE_SETAFFINITY) && defined(HAVE_LIBNUMA)
-  if (((ENGINE_POLICY) &
-       engine_policy_setaffinity) == engine_policy_setaffinity) {
-    /* Ensure the NUMA node on which we initialise (first touch) everything
-     * doesn't change before engine_init allocates NUMA-local workers.
-     * Otherwise, we may be scheduled elsewhere between the two times.
-     */
+  if (((ENGINE_POLICY) & engine_policy_setaffinity) ==
+      engine_policy_setaffinity)
     engine_pin();
-  }
 #endif
 
   /* Welcome to SWIFT, you made the right choice */

src/engine.c

@@ -65,6 +65,11 @@ const char *engine_policy_names[13] = {
 /** The rank of the engine as a global variable (for messages). */
 int engine_rank;
 
+#ifdef HAVE_SETAFFINITY
+/** The initial affinity of the main thread (set by engin_pin()) */
+static cpu_set_t entry_affinity;
+#endif
+
 /**
  * @brief Link a density/force task to a cell.
  *
@@ -2298,20 +2303,30 @@ void engine_split(struct engine *e, struct partition *initial_partition) {
 }
 
 #ifdef HAVE_SETAFFINITY
-static cpu_set_t entry_affinity;
-static bool use_entry_affinity = false;
+/**
+ * @brief Returns the initial affinity the main thread is using.
+ */
+static cpu_set_t *engine_entry_affinity() {
+
+  static int use_entry_affinity = 0;
 
-static cpu_set_t *engine_entry_affinity(void) {
   if (!use_entry_affinity) {
     pthread_t engine = pthread_self();
-    pthread_getaffinity_np(engine, sizeof entry_affinity, &entry_affinity);
-    use_entry_affinity = true;
+    pthread_getaffinity_np(engine, sizeof(entry_affinity), &entry_affinity);
+    use_entry_affinity = 1;
   }
 
   return &entry_affinity;
 }
+#endif
 
-void engine_pin(void) {
+/**
+ * @brief  Ensure the NUMA node on which we initialise (first touch) everything
+ *  doesn't change before engine_init allocates NUMA-local workers.
+ */
+void engine_pin() {
+
+#ifdef HAVE_SETAFFINITY
   cpu_set_t *entry_affinity = engine_entry_affinity();
 
   int pin;
@@ -2321,16 +2336,25 @@ void engine_pin(void) {
   cpu_set_t affinity;
   CPU_ZERO(&affinity);
   CPU_SET(pin, &affinity);
-  if (sched_setaffinity(0, sizeof affinity, &affinity) != 0) {
+  if (sched_setaffinity(0, sizeof(affinity), &affinity) != 0) {
     error("failed to set engine's affinity");
   }
+#else
+  error("SWIFT was not compiled with support for pinning.");
+#endif
 }
 
-static void engine_unpin(void) {
+/**
+ * @brief Unpins the main thread.
+ */
+void engine_unpin() {
+#ifdef HAVE_SETAFFINITY
   pthread_t main_thread = pthread_self();
-  pthread_setaffinity_np(main_thread, sizeof entry_affinity, &entry_affinity);
-}
+  pthread_setaffinity_np(main_thread, sizeof(entry_affinity), &entry_affinity);
+#else
+  error("SWIFT was not compiled with support for pinning.");
 #endif
+}
 
 /**
  * @brief init an engine with the given number of threads, queues, and
@@ -2388,7 +2412,7 @@ void engine_init(struct engine *e, struct space *s,
   if (nr_queues <= 0) nr_queues = e->nr_threads;
   s->nr_queues = nr_queues;
 
-/* Deal with affinity. */
+/* Deal with affinity. For now, just figure out the number of cores. */
 #if defined(HAVE_SETAFFINITY)
   const int nr_cores = sysconf(_SC_NPROCESSORS_ONLN);
   cpu_set_t *entry_affinity = engine_entry_affinity();

src/engine.h

@@ -190,5 +190,6 @@ struct link *engine_addlink(struct engine *e, struct link *l, struct task *t);
 void engine_print_policy(struct engine *e);
 int engine_is_done(struct engine *e);
 void engine_pin();
+void engine_unpin();
 
 #endif /* SWIFT_ENGINE_H */