Merge branch 'master' of gitlab.cosma.dur.ac.uk:swift/swiftsim into formatting

Conflicts:
	src/engine.c
	src/proxy.c
	src/runner.c
	src/scheduler.c
	src/space.c
	src/task.c

Former-commit-id: 97aa233cd52050e707d1aca5bec071e3f6f82515

AUTHORS

-Pedro Gonnet: Main author.
-Matthieu Schaller: TBA.
+Pedro Gonnet            gonnet@google.com	
+Matthieu Schaller       matthieu.schaller@durham.ac.uk
+Aidan Chalk             aidan.chalk@durham.ac.uk
+Peter W. Draper         p.w.draper@durham.ac.uk
+Bert Vandenbrouck       bert.vandenbroucke@gmail.com

INSTALL.swift

@@ -58,10 +58,36 @@ for instance. GCC address sanitizer flags can be included using the
 
 option. Note this requires a GCC compiler version of at least 4.8.
 
-Dependencies: needs to be filled in...
+SWIFT currently requires a compiler with OpenMP support.
 
 
-    
+                                 Dependencies
+                                 ============
 
+SWIFT depends on a number of thirdparty libraries that should be available
+before you can build it.
 
 
+HDF5: a HDF5 library is required to read and write particle data. One of the
+commands "h5cc" or "h5pcc" should be available. If "h5pcc" is located them a
+parallel HDF5 built for the version of MPI located should be provided. If
+the command is not available then it can be located using the "--with-hfd5"
+configure option. The value should be the full path to the "h5cc" or "h5pcc"
+commands.
+
+
+MPI: an optional MPI library that fully supports MPI_THREAD_MULTIPLE.  
+Before running configure the "mpirun" command should be available in the
+shell. If your command isn't called "mpirun" then define the "MPIRUN"
+environment variable, either in the shell or when running configure.
+
+
+METIS: a build of the METIS library can be optionally used to optimize the
+load between MPI nodes (requires an MPI library). This should be found in the
+standard installation directories, or pointed at using the "--with-metis"
+configuration option.  In this case the top-level installation directory of
+the METIS build should be given. Note to use METIS you should at least supply
+"--with-metis".
+
+
+DOXYGEN: the doxygen library is required to create the SWIFT API documentation.

README

+This is SWIFT. See INSTALL.swift for instructions.
+
+www.swiftsim.com

configure.ac

@@ -57,15 +57,21 @@ AC_ARG_ENABLE([mpi],
     [enable_mpi="$enableval"],
     [enable_mpi="yes"]
 )
+good_mpi="yes"
 if test "$enable_mpi" = "yes"; then
     AX_MPI([CC="$MPICC" AC_DEFINE(HAVE_MPI, 1, [Define if you have the MPI library.]) ])
 
     # Various MPI implementations require additional libraries when also using
     # threads. Use mpirun (on PATH) as that seems to be only command with
-    # version flag.
-    AC_PATH_PROG([MPIRUN],[mpirun],[notfound])
+    # version flag, allow MPIRUN to override for systems that insist on
+    # a non-standard name (PRACE).
+    : ${MPIRUN='mpirun'}
+    if test "$MPIRUN" = "mpirun"; then
+       AC_PATH_PROG([MPIRUN],[mpirun],[notfound])
+    fi
     if test "$MPIRUN" = "notfound"; then
        AC_MSG_WARN([Cannot find mpirun command on PATH, thread support may not be correct])
+       enable_mpi="no"
     else
        # Special options we know about.
        # Intel: -mt_mpi
@@ -86,6 +92,15 @@ if test "$enable_mpi" = "yes"; then
          *"Open MPI"*)
             MPI_THREAD_LIBS=""
             AC_MSG_RESULT([Open MPI])
+            #  OpenMPI should be 1.8.6 or later, if not complain.
+            #  Version is last word on first line of -version output.
+            revision=`mpirun -version 2>&1 | grep "Open MPI" | awk '{print $NF}'`
+            AX_COMPARE_VERSION( $revision, [ge], [1.8.6],,[good_mpi="no"] )
+            if test "$good_mpi" = "no"; then
+                AC_MSG_WARN([
+    Open MPI version should be at least 1.8.6 (is $revision)])
+                enable_mpi="yes (but with warning)"
+            fi
          ;;
          *)
             MPI_THREAD_LIBS=""
@@ -95,7 +110,10 @@ if test "$enable_mpi" = "yes"; then
        AC_SUBST([MPI_THREAD_LIBS])
     fi
 fi
-AM_CONDITIONAL([HAVEMPI],[test -n "$MPICC"])
+AM_CONDITIONAL([HAVEMPI],[test $enable_mpi = "yes"])
+
+# Indicate that MPIRUN can be modified by an environement variable
+AC_ARG_VAR(MPIRUN, Path to the mpirun command if non-standard)
 
 # Need C99 and inline support.
 AC_PROG_CC_C99
@@ -170,39 +188,31 @@ AX_PTHREAD([LIBS="$PTHREAD_LIBS $LIBS" CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
     or use CPPFLAGS and LDFLAGS if the library is installed in a
     non-standard location.]))
 
-# Check for OpenMP.
-AC_OPENMP
-AC_SUBST(OPENMP_CFLAGS)
-enable_openmp="no"
-if test -z "$OPENMP_CFLAGS"; then
-   echo $OPENMP_CFLAGS
-   AC_MSG_ERROR(Compiler does not support OpenMP, 1)
-else
-   CFLAGS="$CFLAGS $OPENMP_CFLAGS"
-   enable_openmp="yes"
-fi
 
 # Check for metis. Note AX_LIB_METIS exists, but cannot be configured
 # to be default off (i.e. given no option it tries to locate METIS), so we
 # don't use that.
 AC_ARG_WITH([metis],
     [AS_HELP_STRING([--with-metis=PATH],
-       [prefix where the metis library is installed @<:@default=yes@:>@]
+       [root directory where metis is installed @<:@default=yes@:>@]
     )],
     [],
     [with_metis="no"]
 )
 if test "x$with_metis" != "xno"; then
    if test "x$with_metis" != "xyes" -a "x$with_metis" != "x"; then
-      METIS_LIBS="-L$with_metis -lmetis"
+      METIS_LIBS="-L$with_metis/lib -lmetis"
+      METIS_INCS="-I$with_metis/include"
    else
       METIS_LIBS="-lmetis"
+      METIS_INCS=""
    fi
    AC_CHECK_LIB([metis],[METIS_PartGraphKway],
       AC_DEFINE([HAVE_METIS],1,[The metis library appears to be present.]),
       AC_MSG_ERROR(something is wrong with the metis library!),$METIS_LIBS)
 fi
 AC_SUBST([METIS_LIBS])
+AC_SUBST([METIS_INCS])
 AM_CONDITIONAL([HAVEMETIS],[test -n "$METIS_LIBS"])
 
 # Check for zlib.
@@ -219,7 +229,7 @@ AX_LIB_HDF5
 # The default is to use MPI support if it is available, i.e. this is
 # a parallel HDF5.
 # To do this need to ask the HDF5 compiler about its configuration,
-# -showconfig should have yes/no. 
+# -showconfig should have yes/no.
 have_parallel_hdf5="no"
 if test "$with_hdf5" = "yes"; then
     AC_ARG_ENABLE([parallel-hdf5],
@@ -284,7 +294,6 @@ AC_MSG_RESULT([
    MPI enabled: $enable_mpi
    HDF5 enabled: $with_hdf5
        parallel: $have_parallel_hdf5
-   OpenMP enabled: $enable_openmp
    Metis enabled: $with_metis
 ])
 

examples/Makefile.am

@@ -20,13 +20,13 @@
 MYFLAGS = -DTIMER
 
 # Add the source directory and debug to CFLAGS
-AM_CFLAGS = -Wall -Werror -I../src -DCPU_TPS=2.67e9
+AM_CFLAGS = -Wall -Werror -I../src -DCPU_TPS=2.67e9 $(HDF5_CPPFLAGS)
 
 AM_LDFLAGS =
 
-METIS_LIBS = @METIS_LIBS@
 MPI_THREAD_LIBS = @MPI_THREAD_LIBS@
 MPI_LIBS = $(METIS_LIBS) $(MPI_THREAD_LIBS)
+MPI_FLAGS = -DWITH_MPI $(METIS_INCS)
 
 # Set-up the library
 bin_PROGRAMS = test test_fixdt test_mindt test_single
@@ -53,17 +53,17 @@ test_mindt_LDADD =  ../src/.libs/libswiftsim.a $(HDF5_LDFLAGS) $(HDF5_LIBS)
 
 # Sources for test_mpi
 test_mpi_SOURCES = test.c
-test_mpi_CFLAGS = $(MYFLAGS) $(AM_CFLAGS) -DWITH_MPI -DENGINE_POLICY="engine_policy_multistep | engine_policy_keep"
+test_mpi_CFLAGS = $(MYFLAGS) $(AM_CFLAGS) $(MPI_FLAGS) -DENGINE_POLICY="engine_policy_multistep | engine_policy_keep"
 test_mpi_LDADD =  ../src/.libs/libswiftsim_mpi.a $(HDF5_LDFLAGS) $(HDF5_LIBS) $(MPI_LIBS)
 
 # Sources for test_fixdt_mpi
 test_fixdt_mpi_SOURCES = test.c
-test_fixdt_mpi_CFLAGS = $(MYFLAGS) $(AM_CFLAGS) -DWITH_MPI -DENGINE_POLICY="engine_policy_fixdt | engine_policy_keep"
+test_fixdt_mpi_CFLAGS = $(MYFLAGS) $(AM_CFLAGS) $(MPI_FLAGS) -DENGINE_POLICY="engine_policy_fixdt | engine_policy_keep"
 test_fixdt_mpi_LDADD =  ../src/.libs/libswiftsim_mpi.a $(HDF5_LDFLAGS) $(HDF5_LIBS) $(MPI_LIBS)
 
 # Sources for test_mindt_mpi
 test_mindt_mpi_SOURCES = test.c
-test_mindt_mpi_CFLAGS = $(MYFLAGS) $(AM_CFLAGS) -DWITH_MPI -DENGINE_POLICY="engine_policy_keep"
+test_mindt_mpi_CFLAGS = $(MYFLAGS) $(AM_CFLAGS) $(MPI_FLAGS) -DENGINE_POLICY="engine_policy_keep"
 test_mindt_mpi_LDADD =  ../src/.libs/libswiftsim_mpi.a $(HDF5_LDFLAGS) $(HDF5_LIBS) $(MPI_LIBS)
 
 # Sources for test_single

examples/test.c

 /*******************************************************************************
  * This file is part of SWIFT.
- * Coypright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk),
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk),
  *                    Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ *               2015 Peter W. Draper (p.w.draper@durham.ac.uk)
  * 
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Lesser General Public License as published
@@ -31,7 +32,6 @@
 #include <float.h>
 #include <limits.h>
 #include <fenv.h>
-#include <omp.h>
 
 /* Conditional headers. */
 #ifdef HAVE_LIBZ
@@ -270,7 +270,6 @@ void pairs_n2 ( double *dim , struct part *__restrict__ parts , int N , int peri
     double rho_max = 0.0, rho_min = 100;
     
     /* Loop over all particle pairs. */
-    #pragma omp parallel for schedule(dynamic), default(none), private(k,i,dx,r2), shared(periodic,parts,dim,N,stdout)
     for ( j = 0 ; j < N ; j++ ) {
         if ( j % 1000 == 0 ) {
             printf( "pairs_n2: j=%i.\n" , j );
@@ -290,13 +289,11 @@ void pairs_n2 ( double *dim , struct part *__restrict__ parts , int N , int peri
             if ( r2 < parts[j].h*parts[j].h || r2 < parts[k].h*parts[k].h ) {
                 runner_iact_density( r2 , NULL , parts[j].h , parts[k].h , &parts[j] , &parts[k] );
                 /* if ( parts[j].h / parts[k].h > maxratio )
-                    #pragma omp critical
                     {
                     maxratio = parts[j].h / parts[k].h;
                     mj = j; mk = k;
                     }
                 else if ( parts[k].h / parts[j].h > maxratio )
-                    #pragma omp critical
                     {
                     maxratio = parts[k].h / parts[j].h;
                     mj = j; mk = k;
@@ -524,6 +521,23 @@ void density_dump ( int N ) {
 
     }
 
+/**
+ *  Factorize a given integer, attempts to keep larger pair of factors.
+ */
+void factor( int value, int *f1, int *f2 ) {
+    int j;
+    int i;
+
+    j = (int) sqrt( value );
+    for ( i = j; i > 0; i-- ) {
+        if ( ( value % i ) == 0 ) {
+            *f1 = i;
+            *f2 = value / i;
+            break;
+            }
+        }
+    }
+
 
 /**
  * @brief Main routine that loads a few particles and generates some output.
@@ -570,14 +584,21 @@ int main ( int argc , char *argv[] ) {
     if ( myrank == 0 )
         message( "MPI is up and running with %i nodes." , nr_nodes );
     fflush(stdout);
+
+    /* Set a default grid so that grid[0]*grid[1]*grid[2] == nr_nodes. */
+    factor( nr_nodes, &grid[0], &grid[1] );
+    factor( nr_nodes / grid[1], &grid[0], &grid[2] );
+    factor( grid[0] * grid[1], &grid[1], &grid[0] );
 #endif
 
     /* Greeting message */
-    message( "This is %s\n", package_description() );
+    if ( myrank == 0 )
+        message( "This is %s\n", package_description() );
     
     /* Init the space. */
     bzero( &s , sizeof(struct space) );
 
+
     /* Parse the options */
     while ( ( c = getopt( argc , argv  , "a:c:d:f:g:m:q:r:s:t:w:z:" ) ) != -1 )
       switch( c )
@@ -609,8 +630,6 @@ int main ( int argc , char *argv[] ) {
 	case 'g':
 	  if ( sscanf( optarg , "%i %i %i" , &grid[0] , &grid[1] , &grid[2] ) != 3 )
 	    error( "Error parsing grid." );
-	  if ( myrank == 0 )
-        message( "grid set to [ %i %i %i ]." , grid[0] , grid[1] , grid[2] ); fflush(stdout);
 	  break;
 	case 'm':
 	  if ( sscanf( optarg , "%lf" , &h_max ) != 1 )
@@ -635,7 +654,6 @@ int main ( int argc , char *argv[] ) {
 	case 't':
 	  if ( sscanf( optarg , "%d" , &nr_threads ) != 1 )
 	    error( "Error parsing number of threads." );
-	  omp_set_num_threads( nr_threads );
 	  break;
 	case 'w':
 	  if ( sscanf( optarg , "%d" , &space_subsize ) != 1 )
@@ -654,7 +672,11 @@ int main ( int argc , char *argv[] ) {
 	  break;
 
 	}
-    
+
+#if defined( WITH_MPI )
+    if ( myrank == 0 )
+        message( "grid set to [ %i %i %i ]." , grid[0] , grid[1] , grid[2] ); fflush(stdout);
+#endif
 
     /* How large are the parts? */
     if ( myrank == 0 ) {

examples/test_single.c

@@ -31,7 +31,6 @@
 #include <float.h>
 #include <limits.h>
 #include <fenv.h>
-#include <omp.h>
 
 /* Conditional headers. */
 #ifdef HAVE_LIBZ
@@ -63,6 +62,9 @@ int main ( int argc , char *argv[] ) {
     struct part p1, p2;
     float x, w, dwdx, r2, dx[3] = { 0.0f , 0.0f , 0.0f }, gradw[3];
     
+    /* Greeting message */
+    printf( "This is %s\n", package_description() );
+
     /* Init the particles. */
     for ( k = 0 ; k < 3 ; k++ ) {
         p1.a[k] = 0.0f; p1.v[k] = 0.0f; p1.x[k] = 0.0;

src/Makefile.am

@@ -17,7 +17,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 # Add the debug flag to the whole thing
-AM_CFLAGS = -DTIMER -DCOUNTER -DCPU_TPS=2.30e9
+AM_CFLAGS = -DTIMER -DCOUNTER -DCPU_TPS=2.30e9 $(HDF5_CPPFLAGS)
 
 # Assign a "safe" version number
 AM_LDFLAGS = $(LAPACK_LIBS) $(BLAS_LIBS) $(HDF5_LDFLAGS) -version-info 0:0:0 # -fsanitize=address
@@ -52,17 +52,24 @@ libswiftsim_la_SOURCES = $(AM_SOURCES)
 
 # Sources and flags for MPI library
 libswiftsim_mpi_la_SOURCES = $(AM_SOURCES)
-libswiftsim_mpi_la_CFLAGS = $(AM_CFLAGS) -DWITH_MPI
+libswiftsim_mpi_la_CFLAGS = $(AM_CFLAGS) -DWITH_MPI $(METIS_INCS)
 libswiftsim_mpi_la_SHORTNAME = mpi
 
 
 # Versioning. If any sources change then update the version.h file with
 # the current git revision and package version.
+# May have a checkout without a version.h file and no git command (tar/zip
+# download), allow that, but make sure we know it.
 version.h: version.h.in $(AM_SOURCES) $(include_HEADERS) $(noinst_HEADERS)
 	if test "X$(GIT_CMD)" != "X"; then \
 	    GIT_REVISION=`git describe --abbrev=8  --always --tags --dirty`; \
 	    sed -e "s,@PACKAGE_VERSION\@,$(PACKAGE_VERSION)," \
 	        -e "s,@GIT_REVISION\@,$${GIT_REVISION}," version.h.in > version.h; \
+	else \
+	    if test ! -f version.h; then \
+	        sed -e "s,@PACKAGE_VERSION\@,$(PACKAGE_VERSION)," \
+	            -e "s,@GIT_REVISION\@,unknown," version.h.in > version.h; \
+	    fi; \
 	fi
 
 #  Make sure version.h is built first.

src/engine.c

@@ -23,8 +23,6 @@
 /* Some standard headers. */
 #include <float.h>
 #include <limits.h>
-#include <math.h>
-#include <omp.h>
 #include <sched.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -34,12 +32,11 @@
 /* MPI headers. */
 #ifdef WITH_MPI
 #include <mpi.h>
-#endif
-
-/* METIS headers. */
+/* METIS headers only used when MPI is also available. */
 #ifdef HAVE_METIS
 #include <metis.h>
 #endif
+#endif
 
 /* This object's header. */
 #include "engine.h"

src/proxy.c

@@ -23,9 +23,6 @@
 /* Some standard headers. */
 #include <float.h>
 #include <limits.h>
-#include <math.h>
-#include <omp.h>
-#include <pthread.h>
 #include <sched.h>
 #include <stdio.h>
 #include <stdlib.h>

src/runner.c

@@ -23,12 +23,6 @@
 /* Some standard headers. */
 #include <float.h>
 #include <limits.h>
-#include <math.h>
-#include <omp.h>
-#include <pthread.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
 
 /* MPI headers. */
 #ifdef WITH_MPI

src/scheduler.c

@@ -23,7 +23,6 @@
 /* Some standard headers. */
 #include <limits.h>
 #include <math.h>
-#include <omp.h>
 #include <pthread.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -803,7 +802,6 @@ void scheduler_reweight(struct scheduler *s) {
   /* Run throught the tasks backwards and set their waits and
      weights. */
   // tic = getticks();
-  // #pragma omp parallel for schedule(static) private(t,j)
   for (k = nr_tasks - 1; k >= 0; k--) {
     t = &tasks[tid[k]];
     t->weight = 0;
@@ -887,7 +885,6 @@ void scheduler_start(struct scheduler *s, unsigned int mask) {
 
   /* Run throught the tasks and set their waits. */
   // tic = getticks();
-  // #pragma omp parallel for schedule(static) private(t,j)
   for (k = nr_tasks - 1; k >= 0; k--) {
     t = &tasks[tid[k]];
     t->wait = 0;

src/task.c

@@ -23,9 +23,6 @@
 /* Some standard headers. */
 #include <float.h>
 #include <limits.h>
-#include <math.h>
-#include <omp.h>
-#include <pthread.h>
 #include <sched.h>
 #include <stdio.h>
 #include <stdlib.h>