Implement simple thread barriers to be used on systems where the POSIX...

Implement simple thread barriers to be used on systems where the POSIX implementation does not provide them. We default to the POSIX ones when detected at configure time.

configure.ac

@@ -379,7 +379,9 @@ AX_PTHREAD([LIBS="$PTHREAD_LIBS $LIBS" CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
     or use CPPFLAGS and LDFLAGS if the library is installed in a
     non-standard location.]))
 
-AC_CHECK_LIB(pthread, pthread_barrier_init,,AC_MSG_ERROR(POSIX implementation does not have barriers!))
+AC_CHECK_LIB(pthread, pthread_barrier_init,
+	     AC_DEFINE([HAVE_PTHREAD_BARRIERS], [1], [The posix library implements barriers]),
+	     AC_MSG_WARN(POSIX implementation does not have barriers. SWIFT will use home-made ones.))		
 
 # Check for metis. Note AX_LIB_METIS exists, but cannot be configured
 # to be default off (i.e. given no option it tries to locate METIS), so we

src/Makefile.am

@@ -60,7 +60,7 @@ AM_SOURCES = space.c runner.c queue.c task.c cell.c engine.c \
     collectgroup.c hydro_space.c
 
 # Include files for distribution, not installation.
-nobase_noinst_HEADERS = align.h approx_math.h atomic.h cycle.h error.h inline.h kernel_hydro.h kernel_gravity.h \
+nobase_noinst_HEADERS = align.h approx_math.h atomic.h barrier.h cycle.h error.h inline.h kernel_hydro.h kernel_gravity.h \
 		 kernel_long_gravity.h vector.h cache.h runner_doiact.h runner_doiact_vec.h runner_doiact_grav.h runner_doiact_fft.h \
                  runner_doiact_nosort.h units.h intrinsics.h minmax.h kick.h timestep.h drift.h adiabatic_index.h io_properties.h \
 		 dimension.h equation_of_state.h part_type.h periodic.h \

src/barrier.h

+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2017 Matthieu Schaller (matthieu.schaller@durham.ac.uk).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+#ifndef SWIFT_BARRIER_H
+#define SWIFT_BARRIER_H
+
+/**
+ * @file barrier.h
+ * @brief Define the thread barriers if the POSIX implementation on this system
+ * does not.
+ *
+ * The pthread barriers are only an option of the POSIX norm and they are not
+ * necessarily implemented. One example is OSX where all the rest of POSIX
+ * exists but not the barriers.
+ * We implement them here in a simple way to allow for SWIFT to run on such
+ * systems but this may lead to poorer performance.
+ *
+ * Note that we only define the three functions we need. This is a textbook
+ * implementation of a barrier that uses the common POSIX features (mutex,
+ * conditions and broadcasts).
+ *
+ * If the pthread barriers exist (Linux systems), we default to them.
+ */
+
+/* Config parameters. */
+#include "../config.h"
+
+/* Standard headers */
+#include <pthread.h>
+
+/* Does this POSIX implementation provide barriers? */
+#ifdef HAVE_PTHREAD_BARRIERS
+
+#define swift_barrier_t pthread_barrier_t
+#define swift_barrier_wait pthread_barrier_wait
+#define swift_barrier_init pthread_barrier_init
+#define swift_barrier_destroy pthread_barrier_destroy
+
+#else
+
+/* Local headers */
+#include "error.h"
+#include "inline.h"
+
+/**
+ * @brief An ersatz of POSIX barriers to be used on systems that don't provide
+ * the good ones.
+ */
+typedef struct {
+
+  /*! Barrier mutex */
+  pthread_mutex_t mutex;
+
+  /*! Condition to open the barrier */
+  pthread_cond_t condition;
+
+  /*! Total number of threads */
+  int limit;
+
+  /*! Number of threads that reached the barrier */
+  int count;
+
+} swift_barrier_t;
+
+/**
+ * @brief Initialise a barrier object.
+ *
+ * @param barrier The #swift_barrier_t to initialise
+ * @param unused Unused parameter (NULL) as we don't support barrier attributes.
+ * @param count The number of threads that will wait at the barrier.
+ */
+static INLINE int swift_barrier_init(swift_barrier_t *barrier, void *unused,
+                                     unsigned int count) {
+  /* Initialise the mutex */
+  if (pthread_mutex_init(&barrier->mutex, 0) != 0)
+    error("Error initializing the barrier mutex");
+
+  /* Initialise the condition */
+  if (pthread_cond_init(&barrier->condition, 0) != 0)
+    error("Error initializing the barrier condition");
+
+  barrier->limit = count;
+  barrier->count = 0;
+
+  /* All is good */
+  return 0;
+}
+
+/**
+ * @brief Make a set of threads wait at the barrier
+ *
+ * Note that once all threads have reached the barrier, we also
+ * reset the barrier to state where it is ready to be re-used
+ * without calling swift_barrier_init.
+ *
+ * @param barrier The (initialised) #swift_barrier_t to wait at.
+ */
+static INLINE int swift_barrier_wait(swift_barrier_t *barrier) {
+
+  /* Start by locking the barrier */
+  pthread_mutex_lock(&barrier->mutex);
+
+  /* One more thread has gone home*/
+  barrier->count++;
+
+  /* Are threads still running? */
+  if (barrier->count < barrier->limit) {
+
+    /* We need to make the thread wait until everyone is back */
+    pthread_cond_wait(&barrier->condition, &(barrier->mutex));
+
+    /* Release the mutex */
+    pthread_mutex_unlock(&barrier->mutex);
+
+    /* Say that this was not the last thread */
+    return 0;
+
+  } else { /* Everybody is home */
+
+    /* Open the barrier (i.e. release the threads blocked in the while loop) */
+    pthread_cond_broadcast(&barrier->condition);
+
+    /* Re-initialize the barrier */
+    barrier->count = 0;
+
+    /* Release the mutex */
+    pthread_mutex_unlock(&barrier->mutex);
+
+    /* Say that we are all done */
+    return 1;
+  }
+}
+
+/**
+ * @brief Destroy a barrier object
+ *
+ * Note that if destroy is called before a barrier is open, we return
+ * an error message and do not attempt to wait for the barrier to open
+ * before destroying it.
+ *
+ * @param barrier The #swift_barrier_t object to destroy.
+ */
+static INLINE int swift_barrier_destroy(swift_barrier_t *barrier) {
+
+  /* Destroy the pthread things */
+  pthread_cond_destroy(&barrier->condition);
+  pthread_mutex_destroy(&barrier->mutex);
+
+  /* All is good */
+  return 0;
+}
+
+#endif /* HAVE_PTHREAD_BARRIERS */
+
+#endif /* SWIFT_BARRIER_H */

src/engine.c

@@ -3140,10 +3140,10 @@ void engine_prepare(struct engine *e) {
 void engine_barrier(struct engine *e) {
 
   /* Wait at the wait barrier. */
-  pthread_barrier_wait(&e->wait_barrier);
+  swift_barrier_wait(&e->wait_barrier);
 
   /* Wait at the run barrier. */
-  pthread_barrier_wait(&e->run_barrier);
+  swift_barrier_wait(&e->run_barrier);
 }
 
 /**
@@ -3427,7 +3427,7 @@ void engine_launch(struct engine *e) {
   atomic_inc(&e->sched.waiting);
 
   /* Cry havoc and let loose the dogs of war. */
-  pthread_barrier_wait(&e->run_barrier);
+  swift_barrier_wait(&e->run_barrier);
 
   /* Load the tasks. */
   scheduler_start(&e->sched);
@@ -3439,7 +3439,7 @@ void engine_launch(struct engine *e) {
   pthread_mutex_unlock(&e->sched.sleep_mutex);
 
   /* Sit back and wait for the runners to come home. */
-  pthread_barrier_wait(&e->wait_barrier);
+  swift_barrier_wait(&e->wait_barrier);
 
   if (e->verbose)
     message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
@@ -4622,8 +4622,8 @@ void engine_init(struct engine *e, struct space *s,
   threadpool_init(&e->threadpool, e->nr_threads);
 
   /* First of all, init the barrier and lock it. */
-  if (pthread_barrier_init(&e->wait_barrier, NULL, e->nr_threads + 1) != 0 ||
-      pthread_barrier_init(&e->run_barrier, NULL, e->nr_threads + 1) != 0)
+  if (swift_barrier_init(&e->wait_barrier, NULL, e->nr_threads + 1) != 0 ||
+      swift_barrier_init(&e->run_barrier, NULL, e->nr_threads + 1) != 0)
     error("Failed to initialize barrier.");
 
   /* Expected average for tasks per cell. If set to zero we use a heuristic
@@ -4709,7 +4709,7 @@ void engine_init(struct engine *e, struct space *s,
 #endif
 
   /* Wait for the runner threads to be in place. */
-  pthread_barrier_wait(&e->wait_barrier);
+  swift_barrier_wait(&e->wait_barrier);
 }
 
 /**

src/engine.h

@@ -32,11 +32,8 @@
 #include <mpi.h>
 #endif
 
-/* Some standard headers. */
-#include <pthread.h>
-#include <stdio.h>
-
 /* Includes. */
+#include "barrier.h"
 #include "clocks.h"
 #include "collectgroup.h"
 #include "cooling_struct.h"
@@ -175,8 +172,8 @@ struct engine {
   int count_step;
 
   /* Data for the threads' barrier. */
-  pthread_barrier_t wait_barrier;
-  pthread_barrier_t run_barrier;
+  swift_barrier_t wait_barrier;
+  swift_barrier_t run_barrier;
 
   /* ID of the node this engine lives on. */
   int nr_nodes, nodeID;

src/threadpool.c

@@ -169,10 +169,10 @@ void *threadpool_runner(void *data) {
   while (1) {
 
     /* Let the controller know that this thread is waiting. */
-    pthread_barrier_wait(&tp->wait_barrier);
+    swift_barrier_wait(&tp->wait_barrier);
 
     /* Wait for the controller. */
-    pthread_barrier_wait(&tp->run_barrier);
+    swift_barrier_wait(&tp->run_barrier);
 
     /* If no map function is specified, just die. We use this as a mechanism
        to shut down threads without leaving the barriers in an invalid state. */
@@ -212,8 +212,8 @@ void threadpool_init(struct threadpool *tp, int num_threads) {
   if (num_threads == 1) return;
 
   /* Init the barriers. */
-  if (pthread_barrier_init(&tp->wait_barrier, NULL, num_threads) != 0 ||
-      pthread_barrier_init(&tp->run_barrier, NULL, num_threads) != 0)
+  if (swift_barrier_init(&tp->wait_barrier, NULL, num_threads) != 0 ||
+      swift_barrier_init(&tp->run_barrier, NULL, num_threads) != 0)
     error("Failed to initialize barriers.");
 
   /* Set the task counter to zero. */
@@ -237,7 +237,7 @@ void threadpool_init(struct threadpool *tp, int num_threads) {
   }
 
   /* Wait for all the threads to be up and running. */
-  pthread_barrier_wait(&tp->wait_barrier);
+  swift_barrier_wait(&tp->wait_barrier);
 }
 
 /**
@@ -288,13 +288,13 @@ void threadpool_map(struct threadpool *tp, threadpool_map_function map_function,
   tp->num_threads_running = 0;
 
   /* Wait for all the threads to be up and running. */
-  pthread_barrier_wait(&tp->run_barrier);
+  swift_barrier_wait(&tp->run_barrier);
 
   /* Do some work while I'm at it. */
   threadpool_chomp(tp, tp->num_threads - 1);
 
   /* Wait for all threads to be done. */
-  pthread_barrier_wait(&tp->wait_barrier);
+  swift_barrier_wait(&tp->wait_barrier);
 
 #ifdef SWIFT_DEBUG_THREADPOOL
   /* Log the total call time to thread id -1. */
@@ -321,15 +321,15 @@ void threadpool_clean(struct threadpool *tp) {
      * and waiting for all the threads to terminate. This ensures that no
      * thread is still waiting at a barrier. */
     tp->map_function = NULL;
-    pthread_barrier_wait(&tp->run_barrier);
+    swift_barrier_wait(&tp->run_barrier);
     for (int k = 0; k < tp->num_threads - 1; k++) {
       void *retval;
       pthread_join(tp->threads[k], &retval);
     }
 
     /* Release the barriers. */
-    if (pthread_barrier_destroy(&tp->wait_barrier) != 0 ||
-        pthread_barrier_destroy(&tp->run_barrier) != 0)
+    if (swift_barrier_destroy(&tp->wait_barrier) != 0 ||
+        swift_barrier_destroy(&tp->run_barrier) != 0)
       error("Failed to destroy threadpool barriers.");
 
     /* Clean up memory. */

src/threadpool.h

@@ -22,10 +22,11 @@
 /* Config parameters. */
 #include "../config.h"
 
-/* Some standard headers. */
+/* Standard headers */
 #include <pthread.h>
 
 /* Local includes. */
+#include "barrier.h"
 #include "cycle.h"
 
 /* Local defines. */
@@ -70,8 +71,8 @@ struct threadpool {
   pthread_t *threads;
 
   /* This is where threads go to rest. */
-  pthread_barrier_t wait_barrier;
-  pthread_barrier_t run_barrier;
+  swift_barrier_t wait_barrier;
+  swift_barrier_t run_barrier;
 
   /* Current map data and count. */
   void *map_data, *map_extra_data;