Added a bunch of new stuff along with a tests folder

Makefile.am

@@ -19,4 +19,4 @@
 ACLOCAL_AMFLAGS = -I m4
 
 # Show the way...
-SUBDIRS = src examples doc
+SUBDIRS = src examples doc tests

configure.ac

@@ -97,6 +97,31 @@ if test "$enable_mpi" = "yes"; then
 fi
 AM_CONDITIONAL([HAVEMPI],[test -n "$MPICC"])
 
+# Check for metis. Note AX_LIB_METIS exists, but cannot be configured
+# to be default off (i.e. given no option it tries to locate METIS), so we
+# don't use that.
+AC_ARG_WITH([metis],
+    [AS_HELP_STRING([--with-metis=PATH],
+       [prefix where the metis library is installed @<:@default=yes@:>@]
+    )],
+    [],
+    [with_metis="no"]
+)
+if test "x$with_metis" != "xno"; then
+   if test "x$with_metis" != "xyes" -a "x$with_metis" != "x"; then
+      METIS_LIBS="-L$with_metis -lmetis"
+   else
+      METIS_LIBS="-lmetis"
+   fi
+   AC_CHECK_LIB([metis],[METIS_PartGraphKway],
+      AC_DEFINE([HAVE_METIS],1,[The metis library appears to be present.]),
+      AC_MSG_ERROR(something is wrong with the metis library!),$METIS_LIBS)
+fi
+AC_SUBST([METIS_LIBS])
+AM_CONDITIONAL([HAVEMETIS],[test -n "$METIS_LIBS"])
+
+
+
 # autoconf stuff
 AC_PROG_INSTALL
 AC_PROG_MAKE_SET
@@ -138,7 +163,7 @@ AC_MSG_RESULT($rtc_ok)
 DX_INIT_DOXYGEN(libquicksched,doc/Doxyfile,doc/)
 
 # .in files.
-AC_CONFIG_FILES([Makefile src/Makefile examples/Makefile doc/Makefile doc/Doxyfile])
+AC_CONFIG_FILES([Makefile src/Makefile examples/Makefile doc/Makefile doc/Doxyfile tests/Makefile])
 
 # generate output.
 AC_OUTPUT

examples/Makefile.am

@@ -20,7 +20,7 @@
 AM_CFLAGS = -g -O3 -Wall -Werror -I../src -ffast-math -fstrict-aliasing \
     -ftree-vectorize -funroll-loops $(SIMD_FLAGS) $(OPENMP_CFLAGS) \
     -DCPU_TPS=2.67e9 -DTIMERS -std=gnu99 \
-    # -fsanitize=address -fno-omit-frame-pointer
+     -fsanitize=address -fno-omit-frame-pointer
 # AM_CFLAGS = -g -O0 -Wall -Werror -I../src \
 #     -DCPU_TPS=2.67e9 -DTIMERS $(OPENMP_CFLAGS) \
 #     -fsanitize=address -fno-omit-frame-pointer

src/Makefile.am

@@ -18,7 +18,7 @@
 # Add the debug flag to the whole thing
 AM_CFLAGS = -g -O3 -Wall -Werror -ffast-math -fstrict-aliasing -ftree-vectorize \
     -funroll-loops $(SIMD_FLAGS) $(OPENMP_CFLAGS) -DTIMERS -std=gnu99 \
-    # -fsanitize=address -fno-omit-frame-pointer
+     -fsanitize=address -fno-omit-frame-pointer
 
 # Assign a "safe" version number
 AM_LDFLAGS = -version-info 0:0:0
@@ -26,10 +26,15 @@ AM_LDFLAGS = -version-info 0:0:0
 # Build the libquicksched library
 lib_LTLIBRARIES = libquicksched.la
 
+METIS_LIBS = @METIS_LIBS@
+MPI_THREAD_LIBS = @MPI_THREAD_LIBS@
+MPI_LIBS = $(METIS_LIBS) $(MPI_THREAD_LIBS)
+
 if HAVEMPI
 lib_LTLIBRARIES += libquickschedMPI.la
 endif
 
+libquicksched_la_LDFLAGS = -lmetis
 libquicksched_la_SOURCES = qsched.c queue.c
 
 #libquickschedMPI_la_CC = mpicc

src/qsched.h

@@ -26,6 +26,7 @@
 #define qsched_flag_noreown              16
 #define qsched_flag_norecost             32
 #define qsched_flag_mpidirty             64
+#define qsched_flag_ressync              128
 
 /* Some sched-specific constants. */
 #define qsched_stretch                   2
@@ -183,7 +184,7 @@ struct qsched {
     long long int *users_key;
     int size_users;
     int count_users;
-    /* MPICOMM TODO*/
+    MPI_Comm comm;
     #endif
     
     };
@@ -212,20 +213,28 @@ void qsched_sort_rec ( long long int *data , long long int *ind , int N , long l
 struct task *qsched_gettask ( struct qsched *s , int qid );
 void qsched_done ( struct qsched *s , struct task *t );
 void *qsched_getdata( struct qsched *s , struct task *t );
-int qsched_lockres_local ( struct qsched *s , long long int rid );
-void qsched_unlockres_local ( struct qsched *s , long long int rid );
-int qsched_locktask_local ( struct qsched *s , long long int tid );
-void qsched_unlocktask_local ( struct qsched *s , long long int tid );
+int qsched_lockres ( struct qsched *s , long long int rid );
+void qsched_unlockres ( struct qsched *s , long long int rid );
+int qsched_locktask ( struct qsched *s , long long int tid );
+void qsched_unlocktask ( struct qsched *s , long long int tid );
 void qsched_prepare ( struct qsched *s );
 void qsched_enqueue ( struct qsched *s , struct task *t );
 
 /* External functions for MPI. */
-qsched_res_t qsched_address (struct qsched *s, int owner, int size, void **data );
+qsched_res_t qsched_addres (struct qsched *s, int owner, int size, void **data );
 qsched_res_t qsched_addchildres( struct qsched *s, long long int parent, int owner, int size, int position, void **data);
 qsched_task_t qsched_addtask ( struct qsched *s , int type , unsigned int flags , void *data , int data_size , int cost );
+void qsched_sync_resources(struct qsched *s);
+void qsched_sync_schedulers( struct qsched *s);
+void qsched_prepare_mpi( struct qsched *s);
+
 
 /* External functions. */
+#ifdef WITH_MPI
+void qsched_init ( struct qsched *s , int nr_queues , int flags , MPI_Comm comms);
+#else
 void qsched_init ( struct qsched *s , int nr_queues , int flags );
+#endif
 qsched_res_t qsched_addres_local ( struct qsched *s , int owner , qsched_res_t parent );
 void qsched_addlock ( struct qsched *s , qsched_task_t t , qsched_res_t res );
 void qsched_addunlock ( struct qsched *s , qsched_task_t ta , qsched_task_t tb );

src/queue.c

@@ -80,7 +80,7 @@ int queue_get ( struct queue *q , struct qsched *s , int insist ) {
         tid = inds[k];
         
         /* If the task can be locked, break. */
-        if ( qsched_locktask_local( s , tid ) )
+        if ( qsched_locktask( s , tid ) )
             break;
     
         }

src/res.h

@@ -49,10 +49,10 @@ struct res {
     int offset;
 
     /* The pointer to the tasks that use this resource.*/
-    int *users;
+    long long int *users;
 
     /* The pointer to the tasks that lock this resource.*/
-    int *lockers;
+    long long int *lockers;
 
     /* The number of tasks that use/lock this resource. */    
     int num_users, num_lockers; 

tests/Makefile.am

+# This file is part of QuickSched.
+# Coypright (c) 2015 Aidan Chalk (aidan.chalk@durham.ac.uk),
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+AM_CFLAGS = -g -O3 -Wall -Werror -I../src -ffast-math -fstrict-aliasing \
+    -ftree-vectorize -funroll-loops $(SIMD_FLAGS) $(OPENMP_CFLAGS) \
+    -DCPU_TPS=2.67e9 -DTIMERS -std=gnu99 \
+     -fsanitize=address -fno-omit-frame-pointer
+
+AM_LDFLAGS = -lm  # -fsanitize=address
+
+METIS_LIBS = @METIS_LIBS@
+MPI_THREAD_LIBS = @MPI_THREAD_LIBS@
+MPI_LIBS = $(METIS_LIBS) $(MPI_THREAD_LIBS)
+
+
+bin_PROGRAMS = test_mpi #test_mpi_synchronize
+
+# Sources for test
+test_mpi_LDADD =  ../src/.libs/libquickschedMPI.a $(METIS_LIBS)
+test_mpi_LDFLAGS = $(MPI_THREAD_LIBS) 
+test_mpi_SOURCES = test_mpi.c
+test_mpi_CFLAGS = $(AM_CFLAGS) -DWITH_MPI
+
+
+#test_mpi_synchronize_LDFLAGS = $(MPI_THREAD_LIBS)
+#test_mpi_synchronize_SOURCES = test_mpi_synchronize.c
+#test_mpi_synchronize_CFLAGS = $(AM_CFLAGS) -DWITH_MPI
+test_mpi_synchronize_LDADD =  ../src/.libs/libquickschedMPI.a

tests/test_mpi.c

+
+/* Config parameters. */
+#include "../config.h"
+
+/* Some standard headers. */
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <pthread.h>
+#include <math.h>
+#include <float.h>
+#include <limits.h>
+#include <omp.h>
+#include <sched.h>
+#include <mpi.h>
+#include <metis.h>
+
+/* Local includes. */
+#include "quicksched.h"
+#include "res.h"
+
+
+int main(int argc, char *argv[]) {
+    // Initialize the MPI environment
+    MPI_Init(NULL, NULL);
+
+    struct qsched s;
+    
+    qsched_init(&s, 0, 0, MPI_COMM_WORLD);
+
+    // Get the name of the processor
+    char processor_name[MPI_MAX_PROCESSOR_NAME];
+    int name_len;
+    MPI_Get_processor_name(processor_name, &name_len);
+
+    int *data;
+    long long int resid;
+    resid = qsched_addres( &s, qsched_owner_none, 8*sizeof(int), (void**)&data);
+    
+    qsched_sync_resources(&s);
+
+    long long int taskid;
+    taskid = qsched_addtask(&s, 2, 0, NULL, 0, 25);
+    qsched_addlock(&s, taskid, resid);
+    qsched_adduse(&s, taskid, resid);
+
+
+//    qsched_sync_schedulers(&s);
+    qsched_prepare_mpi(&s);
+    int i,j;
+    for(i = 0; i < s.task_ranks[s.count_ranks]; i++)
+    {
+        printf("s->tasks[i].id = %lli\n", s.tasks[i].id);
+    }
+
+    for(i = 0; i < s.count_lockers; i++)
+    {
+        printf("s->tasks[i].id = %lli\n", s.lockers[i]);
+    }
+
+    for(i = 0; i < s.res_ranks[s.count_ranks]; i++)
+    {
+        printf("resource[i] = %lli, %i %i\n", s.res[i].ID, s.res[i].num_lockers, s.res[i].num_users);
+        printf("users are");
+        for(j = 0; j < s.res[i].num_users; j++)
+        {
+            printf(", %lli", s.res[i].users[j]);
+        }
+        printf("\n");
+
+        printf("lockers are");
+        for(j = 0; j < s.res[i].num_lockers; j++)
+        {
+            printf(", %lli", s.res[i].lockers[j]);
+        }
+        printf("\n");
+    }
+
+    // Print off a hello world message
+    printf("Hello world from processor %s, rank = %i, count_ranks = %i\n",
+           processor_name, s.rank, s.count_ranks);
+
+    // Finalize the MPI environment.
+    MPI_Finalize();
+}

tests/test_mpi_synchronize.c

+
+/* Config parameters. */
+#include "../config.h"
+
+/* Some standard headers. */
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <pthread.h>
+#include <math.h>
+#include <float.h>
+#include <limits.h>
+#include <omp.h>
+#include <sched.h>
+#include <mpi.h>
+
+/* Local includes. */
+#include "quicksched.h"
+
+
+int main(int argc, char *argv[]) {
+    // Initialize the MPI environment
+    MPI_Init(NULL, NULL);
+
+    struct qsched s;
+    
+    qsched_init(&s, 0, 0, MPI_COMM_WORLD);
+
+    // Get the name of the processor
+    char processor_name[MPI_MAX_PROCESSOR_NAME];
+    int name_len;
+    MPI_Get_processor_name(processor_name, &name_len);
+
+    int *data;
+    qsched_addres( &s, qsched_owner_none, 8*sizeof(int), (void**)&data);
+    
+    qsched_sync_resources(&s);
+
+    qsched_addres( &s, qsched_owner_none, 8*sizeof(int), (void**)&data);
+
+    // Print off a hello world message
+    printf("Hello world from processor %s, rank = %i, count_ranks = %i\n",
+           processor_name, s.rank, s.count_ranks);
+
+    // Finalize the MPI environment.
+    MPI_Finalize();
+}