From 23d74bfefc8340433c827d09bb742704ea09debd Mon Sep 17 00:00:00 2001
From: Matthieu Schaller <matthieu.schaller@durham.ac.uk>
Date: Sun, 8 Oct 2017 16:59:12 +0100
Subject: [PATCH] Implement simple thread barriers to be used on systems where
 the POSIX implementation does not provide them. We default to the POSIX ones
 when detected at configure time.

---
 configure.ac     |   4 +-
 src/Makefile.am  |   2 +-
 src/barrier.h    | 170 +++++++++++++++++++++++++++++++++++++++++++++++
 src/engine.c     |  14 ++--
 src/engine.h     |   9 +--
 src/threadpool.c |  20 +++---
 src/threadpool.h |   7 +-
 7 files changed, 198 insertions(+), 28 deletions(-)
 create mode 100644 src/barrier.h

diff --git a/configure.ac b/configure.ac
index 7506459d85..279e41507b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -379,7 +379,9 @@ AX_PTHREAD([LIBS="$PTHREAD_LIBS $LIBS" CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
     or use CPPFLAGS and LDFLAGS if the library is installed in a
     non-standard location.]))
 
-AC_CHECK_LIB(pthread, pthread_barrier_init,,AC_MSG_ERROR(POSIX implementation does not have barriers!))
+AC_CHECK_LIB(pthread, pthread_barrier_init,
+	     AC_DEFINE([HAVE_PTHREAD_BARRIERS], [1], [The posix library implements barriers]),
+	     AC_MSG_WARN(POSIX implementation does not have barriers. SWIFT will use home-made ones.))		
 
 # Check for metis. Note AX_LIB_METIS exists, but cannot be configured
 # to be default off (i.e. given no option it tries to locate METIS), so we
diff --git a/src/Makefile.am b/src/Makefile.am
index ec01184928..ce056ce67b 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -60,7 +60,7 @@ AM_SOURCES = space.c runner.c queue.c task.c cell.c engine.c \
     collectgroup.c hydro_space.c
 
 # Include files for distribution, not installation.
-nobase_noinst_HEADERS = align.h approx_math.h atomic.h cycle.h error.h inline.h kernel_hydro.h kernel_gravity.h \
+nobase_noinst_HEADERS = align.h approx_math.h atomic.h barrier.h cycle.h error.h inline.h kernel_hydro.h kernel_gravity.h \
 		 kernel_long_gravity.h vector.h cache.h runner_doiact.h runner_doiact_vec.h runner_doiact_grav.h runner_doiact_fft.h \
                  runner_doiact_nosort.h units.h intrinsics.h minmax.h kick.h timestep.h drift.h adiabatic_index.h io_properties.h \
 		 dimension.h equation_of_state.h part_type.h periodic.h \
diff --git a/src/barrier.h b/src/barrier.h
new file mode 100644
index 0000000000..dbe856b402
--- /dev/null
+++ b/src/barrier.h
@@ -0,0 +1,170 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2017 Matthieu Schaller (matthieu.schaller@durham.ac.uk).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+#ifndef SWIFT_BARRIER_H
+#define SWIFT_BARRIER_H
+
+/**
+ * @file barrier.h
+ * @brief Define the thread barriers if the POSIX implementation on this system
+ * does not.
+ *
+ * The pthread barriers are only an option of the POSIX norm and they are not
+ * necessarily implemented. One example is OSX where all the rest of POSIX
+ * exists but not the barriers.
+ * We implement them here in a simple way to allow for SWIFT to run on such
+ * systems but this may lead to poorer performance.
+ *
+ * Note that we only define the three functions we need. This is a textbook
+ * implementation of a barrier that uses the common POSIX features (mutex,
+ * conditions and broadcasts).
+ *
+ * If the pthread barriers exist (Linux systems), we default to them.
+ */
+
+/* Config parameters. */
+#include "../config.h"
+
+/* Standard headers */
+#include <pthread.h>
+
+/* Does this POSIX implementation provide barriers? */
+#ifdef HAVE_PTHREAD_BARRIERS
+
+#define swift_barrier_t pthread_barrier_t
+#define swift_barrier_wait pthread_barrier_wait
+#define swift_barrier_init pthread_barrier_init
+#define swift_barrier_destroy pthread_barrier_destroy
+
+#else
+
+/* Local headers */
+#include "error.h"
+#include "inline.h"
+
+/**
+ * @brief An ersatz of POSIX barriers to be used on systems that don't provide
+ * the good ones.
+ */
+typedef struct {
+
+  /*! Barrier mutex */
+  pthread_mutex_t mutex;
+
+  /*! Condition to open the barrier */
+  pthread_cond_t condition;
+
+  /*! Total number of threads */
+  int limit;
+
+  /*! Number of threads that reached the barrier */
+  int count;
+
+} swift_barrier_t;
+
+/**
+ * @brief Initialise a barrier object.
+ *
+ * @param barrier The #swift_barrier_t to initialise
+ * @param unused Unused parameter (NULL) as we don't support barrier attributes.
+ * @param count The number of threads that will wait at the barrier.
+ */
+static INLINE int swift_barrier_init(swift_barrier_t *barrier, void *unused,
+                                     unsigned int count) {
+  /* Initialise the mutex */
+  if (pthread_mutex_init(&barrier->mutex, 0) != 0)
+    error("Error initializing the barrier mutex");
+
+  /* Initialise the condition */
+  if (pthread_cond_init(&barrier->condition, 0) != 0)
+    error("Error initializing the barrier condition");
+
+  barrier->limit = count;
+  barrier->count = 0;
+
+  /* All is good */
+  return 0;
+}
+
+/**
+ * @brief Make a set of threads wait at the barrier
+ *
+ * Note that once all threads have reached the barrier, we also
+ * reset the barrier to state where it is ready to be re-used
+ * without calling swift_barrier_init.
+ *
+ * @param barrier The (initialised) #swift_barrier_t to wait at.
+ */
+static INLINE int swift_barrier_wait(swift_barrier_t *barrier) {
+
+  /* Start by locking the barrier */
+  pthread_mutex_lock(&barrier->mutex);
+
+  /* One more thread has gone home*/
+  barrier->count++;
+
+  /* Are threads still running? */
+  if (barrier->count < barrier->limit) {
+
+    /* We need to make the thread wait until everyone is back */
+    pthread_cond_wait(&barrier->condition, &(barrier->mutex));
+
+    /* Release the mutex */
+    pthread_mutex_unlock(&barrier->mutex);
+
+    /* Say that this was not the last thread */
+    return 0;
+
+  } else { /* Everybody is home */
+
+    /* Open the barrier (i.e. release the threads blocked in the while loop) */
+    pthread_cond_broadcast(&barrier->condition);
+
+    /* Re-initialize the barrier */
+    barrier->count = 0;
+
+    /* Release the mutex */
+    pthread_mutex_unlock(&barrier->mutex);
+
+    /* Say that we are all done */
+    return 1;
+  }
+}
+
+/**
+ * @brief Destroy a barrier object
+ *
+ * Note that if destroy is called before a barrier is open, we return
+ * an error message and do not attempt to wait for the barrier to open
+ * before destroying it.
+ *
+ * @param barrier The #swift_barrier_t object to destroy.
+ */
+static INLINE int swift_barrier_destroy(swift_barrier_t *barrier) {
+
+  /* Destroy the pthread things */
+  pthread_cond_destroy(&barrier->condition);
+  pthread_mutex_destroy(&barrier->mutex);
+
+  /* All is good */
+  return 0;
+}
+
+#endif /* HAVE_PTHREAD_BARRIERS */
+
+#endif /* SWIFT_BARRIER_H */
diff --git a/src/engine.c b/src/engine.c
index 7a397e1a00..1e48514ba3 100644
--- a/src/engine.c
+++ b/src/engine.c
@@ -3140,10 +3140,10 @@ void engine_prepare(struct engine *e) {
 void engine_barrier(struct engine *e) {
 
   /* Wait at the wait barrier. */
-  pthread_barrier_wait(&e->wait_barrier);
+  swift_barrier_wait(&e->wait_barrier);
 
   /* Wait at the run barrier. */
-  pthread_barrier_wait(&e->run_barrier);
+  swift_barrier_wait(&e->run_barrier);
 }
 
 /**
@@ -3427,7 +3427,7 @@ void engine_launch(struct engine *e) {
   atomic_inc(&e->sched.waiting);
 
   /* Cry havoc and let loose the dogs of war. */
-  pthread_barrier_wait(&e->run_barrier);
+  swift_barrier_wait(&e->run_barrier);
 
   /* Load the tasks. */
   scheduler_start(&e->sched);
@@ -3439,7 +3439,7 @@ void engine_launch(struct engine *e) {
   pthread_mutex_unlock(&e->sched.sleep_mutex);
 
   /* Sit back and wait for the runners to come home. */
-  pthread_barrier_wait(&e->wait_barrier);
+  swift_barrier_wait(&e->wait_barrier);
 
   if (e->verbose)
     message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
@@ -4622,8 +4622,8 @@ void engine_init(struct engine *e, struct space *s,
   threadpool_init(&e->threadpool, e->nr_threads);
 
   /* First of all, init the barrier and lock it. */
-  if (pthread_barrier_init(&e->wait_barrier, NULL, e->nr_threads + 1) != 0 ||
-      pthread_barrier_init(&e->run_barrier, NULL, e->nr_threads + 1) != 0)
+  if (swift_barrier_init(&e->wait_barrier, NULL, e->nr_threads + 1) != 0 ||
+      swift_barrier_init(&e->run_barrier, NULL, e->nr_threads + 1) != 0)
     error("Failed to initialize barrier.");
 
   /* Expected average for tasks per cell. If set to zero we use a heuristic
@@ -4709,7 +4709,7 @@ void engine_init(struct engine *e, struct space *s,
 #endif
 
   /* Wait for the runner threads to be in place. */
-  pthread_barrier_wait(&e->wait_barrier);
+  swift_barrier_wait(&e->wait_barrier);
 }
 
 /**
diff --git a/src/engine.h b/src/engine.h
index 2b5a4eee7c..77f56d183a 100644
--- a/src/engine.h
+++ b/src/engine.h
@@ -32,11 +32,8 @@
 #include <mpi.h>
 #endif
 
-/* Some standard headers. */
-#include <pthread.h>
-#include <stdio.h>
-
 /* Includes. */
+#include "barrier.h"
 #include "clocks.h"
 #include "collectgroup.h"
 #include "cooling_struct.h"
@@ -175,8 +172,8 @@ struct engine {
   int count_step;
 
   /* Data for the threads' barrier. */
-  pthread_barrier_t wait_barrier;
-  pthread_barrier_t run_barrier;
+  swift_barrier_t wait_barrier;
+  swift_barrier_t run_barrier;
 
   /* ID of the node this engine lives on. */
   int nr_nodes, nodeID;
diff --git a/src/threadpool.c b/src/threadpool.c
index 465756f71d..0ef69f9895 100644
--- a/src/threadpool.c
+++ b/src/threadpool.c
@@ -169,10 +169,10 @@ void *threadpool_runner(void *data) {
   while (1) {
 
     /* Let the controller know that this thread is waiting. */
-    pthread_barrier_wait(&tp->wait_barrier);
+    swift_barrier_wait(&tp->wait_barrier);
 
     /* Wait for the controller. */
-    pthread_barrier_wait(&tp->run_barrier);
+    swift_barrier_wait(&tp->run_barrier);
 
     /* If no map function is specified, just die. We use this as a mechanism
        to shut down threads without leaving the barriers in an invalid state. */
@@ -212,8 +212,8 @@ void threadpool_init(struct threadpool *tp, int num_threads) {
   if (num_threads == 1) return;
 
   /* Init the barriers. */
-  if (pthread_barrier_init(&tp->wait_barrier, NULL, num_threads) != 0 ||
-      pthread_barrier_init(&tp->run_barrier, NULL, num_threads) != 0)
+  if (swift_barrier_init(&tp->wait_barrier, NULL, num_threads) != 0 ||
+      swift_barrier_init(&tp->run_barrier, NULL, num_threads) != 0)
     error("Failed to initialize barriers.");
 
   /* Set the task counter to zero. */
@@ -237,7 +237,7 @@ void threadpool_init(struct threadpool *tp, int num_threads) {
   }
 
   /* Wait for all the threads to be up and running. */
-  pthread_barrier_wait(&tp->wait_barrier);
+  swift_barrier_wait(&tp->wait_barrier);
 }
 
 /**
@@ -288,13 +288,13 @@ void threadpool_map(struct threadpool *tp, threadpool_map_function map_function,
   tp->num_threads_running = 0;
 
   /* Wait for all the threads to be up and running. */
-  pthread_barrier_wait(&tp->run_barrier);
+  swift_barrier_wait(&tp->run_barrier);
 
   /* Do some work while I'm at it. */
   threadpool_chomp(tp, tp->num_threads - 1);
 
   /* Wait for all threads to be done. */
-  pthread_barrier_wait(&tp->wait_barrier);
+  swift_barrier_wait(&tp->wait_barrier);
 
 #ifdef SWIFT_DEBUG_THREADPOOL
   /* Log the total call time to thread id -1. */
@@ -321,15 +321,15 @@ void threadpool_clean(struct threadpool *tp) {
      * and waiting for all the threads to terminate. This ensures that no
      * thread is still waiting at a barrier. */
     tp->map_function = NULL;
-    pthread_barrier_wait(&tp->run_barrier);
+    swift_barrier_wait(&tp->run_barrier);
     for (int k = 0; k < tp->num_threads - 1; k++) {
       void *retval;
       pthread_join(tp->threads[k], &retval);
     }
 
     /* Release the barriers. */
-    if (pthread_barrier_destroy(&tp->wait_barrier) != 0 ||
-        pthread_barrier_destroy(&tp->run_barrier) != 0)
+    if (swift_barrier_destroy(&tp->wait_barrier) != 0 ||
+        swift_barrier_destroy(&tp->run_barrier) != 0)
       error("Failed to destroy threadpool barriers.");
 
     /* Clean up memory. */
diff --git a/src/threadpool.h b/src/threadpool.h
index 019403f658..9d19b56836 100644
--- a/src/threadpool.h
+++ b/src/threadpool.h
@@ -22,10 +22,11 @@
 /* Config parameters. */
 #include "../config.h"
 
-/* Some standard headers. */
+/* Standard headers */
 #include <pthread.h>
 
 /* Local includes. */
+#include "barrier.h"
 #include "cycle.h"
 
 /* Local defines. */
@@ -70,8 +71,8 @@ struct threadpool {
   pthread_t *threads;
 
   /* This is where threads go to rest. */
-  pthread_barrier_t wait_barrier;
-  pthread_barrier_t run_barrier;
+  swift_barrier_t wait_barrier;
+  swift_barrier_t run_barrier;
 
   /* Current map data and count. */
   void *map_data, *map_extra_data;
-- 
GitLab