Merge branch 'threadpool-affinity-fix' into 'master'

Keep affinity of threadpool threads the same as the main thread on entry

Closes #846

See merge request !1686

src/engine.c

@@ -2913,6 +2913,11 @@ void engine_pin(void) {
 
 #ifdef HAVE_SETAFFINITY
   cpu_set_t *entry_affinity = engine_entry_affinity();
+
+  /* Share this affinity with the threadpool, it will use this even when the
+   * main thread is otherwise pinned. */
+  threadpool_set_affinity_mask(entry_affinity);
+
   int pin;
   for (pin = 0; pin < CPU_SETSIZE && !CPU_ISSET(pin, entry_affinity); ++pin)
     ;

src/threadpool.c

@@ -24,6 +24,7 @@
 #include <float.h>
 #include <limits.h>
 #include <math.h>
+#include <sched.h>
 #include <stdlib.h>
 #include <string.h>
 #ifdef SWIFT_DEBUG_THREADPOOL
@@ -42,6 +43,13 @@
 /* Keys for thread specific data. */
 static pthread_key_t threadpool_tid;
 
+/* Affinity mask shared by all threads, and if set. */
+static cpu_set_t thread_affinity;
+static int thread_affinity_set = 0;
+
+/* Local declarations. */
+static void threadpool_apply_affinity_mask(void);
+
 #ifdef SWIFT_DEBUG_THREADPOOL
 /**
  * @brief Store a log entry of the given chunk.
@@ -178,11 +186,19 @@ static void threadpool_chomp(struct threadpool *tp, int tid) {
   }
 }
 
+/**
+ * @brief The thread start routine. Loops until told to exit.
+ *
+ * @param data the threadpool we are part of.
+ */
 static void *threadpool_runner(void *data) {
 
   /* Our threadpool. */
   struct threadpool *tp = (struct threadpool *)data;
 
+  /* Our affinity, if set. */
+  threadpool_apply_affinity_mask();
+
   /* Main loop. */
   while (1) {
 
@@ -412,3 +428,26 @@ int threadpool_gettid() {
   int *tid = (int *)pthread_getspecific(threadpool_tid);
   return *tid;
 }
+
+#ifdef HAVE_SETAFFINITY
+/**
+ * @brief set an affinity mask to be used for all threads.
+ *
+ * @param affinity the mask to use.
+ */
+void threadpool_set_affinity_mask(cpu_set_t *affinity) {
+  memcpy(&thread_affinity, affinity, sizeof(cpu_set_t));
+  thread_affinity_set = 1;
+}
+#endif
+
+/**
+ * @brief apply the affinity mask the current thread, if set.
+ *
+ */
+static void threadpool_apply_affinity_mask(void) {
+  if (thread_affinity_set) {
+    pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t),
+                           &thread_affinity);
+  }
+}

src/threadpool.h

@@ -101,6 +101,9 @@ void threadpool_map(struct threadpool *tp, threadpool_map_function map_function,
                     void *extra_data);
 int threadpool_gettid(void);
 void threadpool_clean(struct threadpool *tp);
+#ifdef HAVE_SETAFFINITY
+void threadpool_set_affinity_mask(cpu_set_t *entry_affinity);
+#endif
 #ifdef SWIFT_DEBUG_THREADPOOL
 void threadpool_reset_log(struct threadpool *tp);
 void threadpool_dump_log(struct threadpool *tp, const char *filename,