From 4b62acfe6cbfc0eaf895d99bc14d35f0bc534c74 Mon Sep 17 00:00:00 2001
From: aidan <aidan@gtx690.dur.ac.uk>
Date: Tue, 26 May 2015 14:07:15 +0100
Subject: [PATCH] Some updates, first MPI functions added

---
 autogen.sh                 |   1 -
 configure.ac               |  56 ++++++
 examples/Makefile.am       |   2 +-
 examples/test.c            |   4 +-
 examples/test_bh.c         |  12 +-
 examples/test_bh_sorted.c  |  12 +-
 examples/test_fmm_sorted.c |   8 +-
 src/Makefile.am            |  14 +-
 src/error.h                |   2 +-
 src/qsched.c               | 375 ++++++++++++++++++++++++++++++++++---
 src/qsched.h               |  14 +-
 src/queue.c                |   5 +
 src/res.h                  |   3 +
 src/task.h                 |   6 +
 14 files changed, 462 insertions(+), 52 deletions(-)

diff --git a/autogen.sh b/autogen.sh
index 2143e39..f4999f9 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -6,4 +6,3 @@
 autoreconf --install --symlink 
 
 exit
-
diff --git a/configure.ac b/configure.ac
index 9511f8d..c5f61aa 100644
--- a/configure.ac
+++ b/configure.ac
@@ -32,6 +32,8 @@ AM_CONFIG_HEADER(config.h)
 # compiler settings
 #CFLAGS="-Wall $(CFLAGS)"
 
+
+
 # find and test the compiler
 AM_PROG_CC_C_O
 AC_PROG_CC_C99
@@ -41,6 +43,60 @@ AX_FUNC_POSIX_MEMALIGN
 AX_GCC_ARCHFLAG([no])
 AX_EXT
 
+
+# Check for MPI. Need to do this before characterising the compiler (C99 mode),
+# as this changes the compiler.
+# We should consider using AX_PROG_CC_MPI to replace AC_PROG_CC when compiling
+# whole applications. There are issues with mixing compilers when using this
+# macro. See
+# http://lists.gnu.org/archive/html/autoconf-archive-maintainers/2011-05/msg00004.html.
+AC_ARG_ENABLE([mpi],
+    [AS_HELP_STRING([--enable-mpi],
+      [Compile with functionality for distributed-memory parallelism using MPI @<:@default=yes@:>@]
+    )],
+    [enable_mpi="$enableval"],
+    [enable_mpi="yes"]
+)
+if test "$enable_mpi" = "yes"; then
+    AX_MPI([CC="$MPICC" AC_DEFINE(HAVE_MPI, 1, [Define if you have the MPI library.]) ])
+
+    # Various MPI implementations require additional libraries when also using
+    # threads. Use mpirun (on PATH) as that seems to be only command with
+    # version flag.
+    AC_PATH_PROG([MPIRUN],[mpirun],[notfound])
+    if test "$MPIRUN" = "notfound"; then
+       AC_MSG_WARN([Cannot find mpirun command on PATH, thread support may not be correct])
+    else
+       # Special options we know about.
+       # Intel: -mt_mpi
+       # PLATFORM: -lmtmpi
+       # OpenMPI: nothing, but library should be built correctly.
+       # Set MPI_THREAD_LIBS and add to linker commands as necessary.
+       AC_MSG_CHECKING([MPI threads options])
+       version=`$MPIRUN -version 2>&1`
+       case "$version" in
+         *Intel*MPI*)
+            MPI_THREAD_LIBS="-mt_mpi"
+            AC_MSG_RESULT([Intel MPI])
+         ;;
+         *Platform*)
+            MPI_THREAD_LIBS="-lmtmpi"
+            AC_MSG_RESULT([PLATFORM MPI])
+         ;;
+         *"Open MPI"*)
+            MPI_THREAD_LIBS=""
+            AC_MSG_RESULT([Open MPI])
+         ;;
+         *)
+            MPI_THREAD_LIBS=""
+            AC_MSG_RESULT([unknown])
+         ;;
+       esac
+       AC_SUBST([MPI_THREAD_LIBS])
+    fi
+fi
+AM_CONDITIONAL([HAVEMPI],[test -n "$MPICC"])
+
 # autoconf stuff
 AC_PROG_INSTALL
 AC_PROG_MAKE_SET
diff --git a/examples/Makefile.am b/examples/Makefile.am
index 792c596..5d9d3f8 100644
--- a/examples/Makefile.am
+++ b/examples/Makefile.am
@@ -19,7 +19,7 @@
 # Add the source directory and debug to CFLAGS
 AM_CFLAGS = -g -O3 -Wall -Werror -I../src -ffast-math -fstrict-aliasing \
     -ftree-vectorize -funroll-loops $(SIMD_FLAGS) $(OPENMP_CFLAGS) \
-    -DCPU_TPS=2.67e9 -DTIMERS \
+    -DCPU_TPS=2.67e9 -DTIMERS -std=gnu99 \
     # -fsanitize=address -fno-omit-frame-pointer
 # AM_CFLAGS = -g -O0 -Wall -Werror -I../src \
 #     -DCPU_TPS=2.67e9 -DTIMERS $(OPENMP_CFLAGS) \
diff --git a/examples/test.c b/examples/test.c
index b065504..6c5d641 100644
--- a/examples/test.c
+++ b/examples/test.c
@@ -123,7 +123,7 @@ void test2 ( int m , int n , int k , int nr_threads ) {
             for ( kk = 0 ; kk < k ; kk++ ) {
                 data[2] = kk;
                 tid = qsched_addtask_local( &s , 1 , task_flag_none , data , 3*sizeof(int) , 1 );
-                qsched_addlock_local( &s , tid , rid );
+                qsched_addlock( &s , tid , rid );
                 }
             }
             
@@ -224,7 +224,7 @@ void test1 ( int m , int n , int k , int nr_threads ) {
             data[0] = i; data[1] = j;
             rid = qsched_addres_local( &s , qsched_owner_none , qsched_res_none );
             tid = qsched_addtask_local( &s , 1 , task_flag_none , data , 2*sizeof(int) , 1 );
-            qsched_addlock_local( &s , tid , rid );
+            qsched_addlock( &s , tid , rid );
             printf("%lli, %lli\n", tid, rid);
             }
             
diff --git a/examples/test_bh.c b/examples/test_bh.c
index 1eab06c..afb42f4 100644
--- a/examples/test_bh.c
+++ b/examples/test_bh.c
@@ -367,9 +367,9 @@ void cell_split(struct cell *c, struct qsched *s) {
     struct cell *data[2] = {root, c};
     int tid = qsched_addtask_local(s, task_type_self_pc, task_flag_none, data,
                              2 * sizeof(struct cell *), 1);
-    qsched_addlock_local(s, tid, c->res);
+    qsched_addlock(s, tid, c->res);
 #ifdef COM_AS_TASK
-    qsched_addunlock_local(s, root->com_tid, tid);
+    qsched_addunlock(s, root->com_tid, tid);
 #endif
   } /* does the cell need to be split? */
 
@@ -1065,12 +1065,12 @@ void create_tasks(struct qsched *s, struct cell *ci, struct cell *cj) {
                          sizeof(struct cell *) * 2, ci->count * ci->count / 2);
 
       /* Add the resource (i.e. the cell) to the new task. */
-      qsched_addlock_local(s, tid, ci->res);
+      qsched_addlock(s, tid, ci->res);
 
 /* If this call might recurse, add a dependency on the cell's COM
    task. */
 #ifdef COM_AS_TASK
-      if (ci->split) qsched_addunlock_local(s, ci->com_tid, tid);
+      if (ci->split) qsched_addunlock(s, ci->com_tid, tid);
 #endif
     }
 
@@ -1105,8 +1105,8 @@ void create_tasks(struct qsched *s, struct cell *ci, struct cell *cj) {
                              sizeof(struct cell *) * 2, ci->count * cj->count);
 
         /* Add the resources. */
-        qsched_addlock_local(s, tid, ci->res);
-        qsched_addlock_local(s, tid, cj->res);
+        qsched_addlock(s, tid, ci->res);
+        qsched_addlock(s, tid, cj->res);
       }
     } /* Cells are direct neighbours */
   }   /* Otherwise it's a pair */
diff --git a/examples/test_bh_sorted.c b/examples/test_bh_sorted.c
index a0448a5..1d0bc72 100644
--- a/examples/test_bh_sorted.c
+++ b/examples/test_bh_sorted.c
@@ -730,9 +730,9 @@ void cell_split(struct cell *c, struct qsched *s) {
     struct cell *data[2] = {root, c};
     int tid = qsched_addtask_local(s, task_type_self_pc, task_flag_none, data,
                              2 * sizeof(struct cell *), 1);
-    qsched_addlock_local(s, tid, c->res);
+    qsched_addlock(s, tid, c->res);
 #ifdef COM_AS_TASK
-    qsched_addunlock_local(s, root->com_tid, tid);
+    qsched_addunlock(s, root->com_tid, tid);
 #endif
   } /* does the cell need to be split? */
 
@@ -1543,12 +1543,12 @@ void create_tasks(struct qsched *s, struct cell *ci, struct cell *cj) {
                          sizeof(struct cell *) * 2, ci->count * ci->count / 2);
 
       /* Add the resource (i.e. the cell) to the new task. */
-      qsched_addlock_local(s, tid, ci->res);
+      qsched_addlock(s, tid, ci->res);
 
 /* If this call might recurse, add a dependency on the cell's COM
    task. */
 #ifdef COM_AS_TASK
-      if (ci->split) qsched_addunlock_local(s, ci->com_tid, tid);
+      if (ci->split) qsched_addunlock(s, ci->com_tid, tid);
 #endif
     }
 
@@ -1583,8 +1583,8 @@ void create_tasks(struct qsched *s, struct cell *ci, struct cell *cj) {
                              sizeof(struct cell *) * 2, ci->count * cj->count);
 
         /* Add the resources. */
-        qsched_addlock_local(s, tid, ci->res);
-        qsched_addlock_local(s, tid, cj->res);
+        qsched_addlock(s, tid, ci->res);
+        qsched_addlock(s, tid, cj->res);
       }
     } /* Cells are direct neighbours */
   }   /* Otherwise it's a pair */
diff --git a/examples/test_fmm_sorted.c b/examples/test_fmm_sorted.c
index 9ffd3d9..3adc3c6 100644
--- a/examples/test_fmm_sorted.c
+++ b/examples/test_fmm_sorted.c
@@ -1496,12 +1496,12 @@ void create_tasks(struct qsched *s, struct cell *ci, struct cell *cj) {
                          sizeof(struct cell *) * 2, ci->count * ci->count / 2);
 
       /* Add the resource (i.e. the cell) to the new task. */
-      qsched_addlock_local(s, tid, ci->res);
+      qsched_addlock(s, tid, ci->res);
 
 /* If this call might recurse, add a dependency on the cell's COM
    task. */
 #ifdef COM_AS_TASK
-      if (ci->split) qsched_addunlock_local(s, ci->com_tid, tid);
+      if (ci->split) qsched_addunlock(s, ci->com_tid, tid);
 #endif
     }
 
@@ -1536,8 +1536,8 @@ void create_tasks(struct qsched *s, struct cell *ci, struct cell *cj) {
                              sizeof(struct cell *) * 2, ci->count * cj->count);
 
         /* Add the resources. */
-        qsched_addlock_local(s, tid, ci->res);
-        qsched_addlock_local(s, tid, cj->res);
+        qsched_addlock(s, tid, ci->res);
+        qsched_addlock(s, tid, cj->res);
       }
     } /* Cells are direct neighbours */
   }   /* Otherwise it's a pair */
diff --git a/src/Makefile.am b/src/Makefile.am
index c953a49..6573232 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -17,7 +17,7 @@
 
 # Add the debug flag to the whole thing
 AM_CFLAGS = -g -O3 -Wall -Werror -ffast-math -fstrict-aliasing -ftree-vectorize \
-    -funroll-loops $(SIMD_FLAGS) $(OPENMP_CFLAGS) -DTIMERS \
+    -funroll-loops $(SIMD_FLAGS) $(OPENMP_CFLAGS) -DTIMERS -std=gnu99 \
     # -fsanitize=address -fno-omit-frame-pointer
 
 # Assign a "safe" version number
@@ -25,8 +25,20 @@ AM_LDFLAGS = -version-info 0:0:0
 
 # Build the libquicksched library
 lib_LTLIBRARIES = libquicksched.la
+
+if HAVEMPI
+lib_LTLIBRARIES += libquickschedMPI.la
+endif
+
 libquicksched_la_SOURCES = qsched.c queue.c
 
+#libquickschedMPI_la_CC = mpicc
+libquickschedMPI_la_LDFLAGS = $(MPI_THREAD_LIBS)
+libquickschedMPI_la_CFLAGS = -g -O3 -Wall -Werror -ffast-math -fstrict-aliasing -ftree-vectorize \
+    -funroll-loops $(SIMD_FLAGS) $(OPENMP_CFLAGS) -DTIMERS -std=gnu99 \
+    -DWITH_MPI
+libquickschedMPI_la_SOURCES = qsched.c queue.c
+
 # List required headers
 include_HEADERS = atomic.h lock.h queue.h qsched.h task.h res.h error.h
 
diff --git a/src/error.h b/src/error.h
index d0ca19d..b6b3b5a 100644
--- a/src/error.h
+++ b/src/error.h
@@ -20,7 +20,7 @@
 
 /* Error macro. */
 #ifdef WITH_MPI
-#define error(s, ...) { fprintf( stderr , "%s:%s():%i: " s "\n" , __FILE__ , __FUNCTION__ , __LINE__ , ##__VA_ARGS__ ); MPI_FINALIZE(); abort(); }
+#define error(s, ...) { fprintf( stderr , "%s:%s():%i: " s "\n" , __FILE__ , __FUNCTION__ , __LINE__ , ##__VA_ARGS__ ); MPI_Finalize(); abort(); }
 #else
 #define error(s, ...) { fprintf( stderr , "%s:%s():%i: " s "\n" , __FILE__ , __FUNCTION__ , __LINE__ , ##__VA_ARGS__ ); abort(); }
 #endif
diff --git a/src/qsched.c b/src/qsched.c
index 50dac33..eb7e2ff 100644
--- a/src/qsched.c
+++ b/src/qsched.c
@@ -30,6 +30,10 @@
     #include <omp.h>
 #endif
 
+#ifdef WITH_MPI
+    #include <mpi.h>
+#endif
+
 // /* Pthread headers, only if available. */
 #ifdef HAVE_PTHREAD
     #include <pthread.h>
@@ -45,6 +49,36 @@
 #include "qsched.h"
 #include "queue.h"
 
+#ifdef WITH_MPI
+inline int getindex(long long int    id , struct qsched *s)
+{
+    return s->res_ranks[id>>48] + (id & 0xFFFFFFFFFFFFFF);
+}
+
+inline struct res* getres(long long int id, struct qsched *s)
+{
+    return &s->res[ getindex(id, s) ];
+}
+
+inline int gettaskindex(long long int id, struct qsched *s)
+{
+    return s->task_ranks[id>>48] + (id & 0xFFFFFFFFFFFFFF);
+}
+#else
+inline int getindex(long long int id, struct qsched *s){
+    return id;
+}
+
+inline struct res* getres(long long int id, struct qsched *s)
+{
+    return &s->res[id];
+}
+
+inline int gettaskindex(long long int id, struct qsched *s)
+{
+    return id;
+}
+#endif
 
 /** Timer names. */
 char *qsched_timer_names[ qsched_timer_count ] = 
@@ -645,7 +679,7 @@ void qsched_done ( struct qsched *s , struct task *t ) {
     
     /* Careful, this may pick up duplicate timers if virtual
        tasks are used. */
-    TIMER_TOC( s , qsched_timer_done )
+    TIMER_TOC( s , qsched_timer_done );
                 
     }
     
@@ -841,7 +875,7 @@ struct task *qsched_gettask ( struct qsched *s , int qid ) {
         {
             TIMER_TIC
             tid = queue_get( &s->queues[qid] , s , 1 );
-            TIMER_TOC( s , qsched_timer_queue )
+            TIMER_TOC( s , qsched_timer_queue );
             if ( tid < 0 ) {
 
                 /* Otherwise, hit the other queues. */
@@ -852,7 +886,7 @@ struct task *qsched_gettask ( struct qsched *s , int qid ) {
                     k = rand() % naq;
                     TIMER_TIC2
                     tid = queue_get( &s->queues[ qids[k] ] , s , 0 );
-                    TIMER_TOC( s , qsched_timer_queue )
+                    TIMER_TOC( s , qsched_timer_queue );
                     if ( tid < 0 )
                         qids[k] = qids[ --naq ];
                     else
@@ -881,7 +915,7 @@ struct task *qsched_gettask ( struct qsched *s , int qid ) {
             t->qid = qid;
             
             /* Return the task. */
-            TIMER_TOC( s , qsched_timer_gettask )
+            TIMER_TOC( s , qsched_timer_gettask );
             return t;
             
             }
@@ -889,7 +923,7 @@ struct task *qsched_gettask ( struct qsched *s , int qid ) {
         /* Otherwise, take a nap? */
         #ifdef HAVE_PTHREAD
         else if ( s->flags & qsched_flag_yield ) {
-            TIMER_TOC( s , qsched_timer_gettask )
+            TIMER_TOC( s , qsched_timer_gettask );
             pthread_mutex_lock( &s->mutex );
             if ( s->waiting )
                 pthread_cond_wait( &s->cond , &s->mutex );
@@ -1173,7 +1207,7 @@ void qsched_prepare ( struct qsched *s ) {
     /* Unlock the sched. */
     lock_unlock_blind( &s->lock );
     
-    TIMER_TOC( s , qsched_timer_prepare )
+    TIMER_TOC( s , qsched_timer_prepare );
 
     }
 
@@ -1240,6 +1274,265 @@ long long int qsched_addres_local ( struct qsched *s , int owner , long long int
 #endif
     }
 
+/**
+ * @brief Add a new resource to the #qsched.
+ *
+ * @param s Pointer to the #qsched.
+ * @param owner ID of the ower
+ * @param size Size of the data associated with the resource in bytes.
+ * @param data Pointer to the pointer which should hold the data.
+ *
+ * @return The ID of the new shared resource.
+ */
+
+long long int qsched_addres ( struct qsched *s, int owner, int size, void **data) {
+#ifndef WITH_MPI
+    error("Quicksched wasn't compiled with MPI support.");
+#else
+struct res *res_new;
+    long long int id;
+    
+    lock_lock( &s->lock);
+
+    //Reallocate res array if neccessary.
+    /* Do the deps need to be re-allocated? */
+    if ( s->count_res == s->size_res ) {
+
+        /* Scale the res list size. */
+        s->size_res *= 2;
+        
+        /* Allocate a new task list. */
+        if ( ( res_new = malloc( sizeof(struct res) * s->size_res ) ) == NULL )
+            error( "Failed to allocate new res lists." );
+            
+        /* Copy the res and owners over to the new list. */
+        memcpy( res_new , s->res , sizeof(struct res) * s->count_res );
+        
+        /* Free the old res lists. */
+        free( s->res );
+        
+        /* Set the new res lists. */
+        s->res = res_new;
+    
+        }
+
+    id = s->count_res;
+    id |= ((long long int)s->rank << 48);
+    s->count_res += 1;
+    int index = getindex(id, s);
+
+    lock_init( &s->res[ getindex(id, s) ]. lock);
+    s->res[ index ].hold = 0;
+    s->res[ index ].owner = owner;
+    s->res[ index ].parent = -1;
+    s->res[ index ].offset = 0;
+    s->res[ index ].node = s->rank;
+    s->res[ index ].data = calloc(size, 1);
+    s->res[ index ].size = size;
+    //Users and locks are set after res_sync, when task creation begins.
+    s->res[ index ].users = NULL;
+    s->res[ index ].lockers = NULL;
+    s->res[ index ].ID = id;
+    s->res[ index ].num_lockers = 0;
+    s->res[ index ].num_users = 0;
+
+    *data = s->res[index].data;
+
+ /* The sched is now dirty. */
+    s->flags |= qsched_flag_dirty;
+    s->flags |= qsched_flag_mpidirty;
+
+    lock_unlock_blind( &s->lock);
+
+   
+
+
+    return id;
+#endif
+}
+
+/**
+ * @brief Add a new child resource to the #qsched.
+ *
+ * @param s Pointer to the #qsched.
+ * @param parent ID of the parent resource. This must be accurate.
+ * @param owner ID of the ower
+ * @param size Size of the data associated with the resource in bytes.
+ * @param position Position in the data array of the parent that this resource's data is located.
+ * @param data Pointer to the pointer which should hold the data.
+ *
+ * @return The ID of the new shared resource.
+ */
+long long int qsched_addchildres( struct qsched *s, long long int parent, int owner, int size, int position, void **data)
+{
+#ifndef WITH_MPI
+    error("Quicksched wasn't compiled with MPI support.");
+#else
+    struct res *res_new;
+    long long int id;
+    
+    lock_lock( &s->lock);
+    //Reallocate res array if neccessary.
+    /* Do the deps need to be re-allocated? */
+    if ( s->count_res == s->size_res ) {
+    
+        /* Scale the res list size. */
+        s->size_res *= 2;
+        
+        /* Allocate a new task list. */
+        if ( ( res_new = malloc( sizeof(struct res) * s->size_res ) ) == NULL )
+            error( "Failed to allocate new res lists." );
+            
+        /* Copy the res and owners over to the new list. */
+        memcpy( res_new , s->res , sizeof(struct res) * s->count_res );
+        
+        /* Free the old res lists. */
+        free( s->res );
+        
+        /* Set the new res lists. */
+        s->res = res_new;
+    
+        }
+
+    id = s->count_res;
+    id |= ((long long int)s->rank << 48);
+int index = getindex(id,s);
+    s->count_res += 1;
+   
+
+    //TODO lock_init( &s->res[ getindex(id,s) ]. lock);
+    s->res[ index ].hold = 0;
+    s->res[ index ].owner = owner;
+    s->res[ index ].parent = parent;
+    s->res[ index ].offset = position;
+    s->res[ index ].node = s->rank;
+    int *temp = (int*) s->res[getindex(parent,s)].data;
+    s->res[ index ].data = &temp[position];
+    s->res[ index ].size = size;
+    s->res[ index ].users = NULL;
+    s->res[ index ].lockers = NULL;
+    s->res[ index ].ID = id;
+
+    *data = s->res[ index ].data;
+
+    //TODO Check for non-overlapping memory in children? I don’t know how best to do this.
+
+ /* The sched is now dirty. */
+    s->flags |= qsched_flag_dirty;
+    s->flags |= qsched_flag_mpidirty;
+    
+    lock_unlock_blind(&s->lock);
+
+    return id;
+
+#endif
+}
+
+/**
+ * @brief Add a task to the #qsched.
+ *
+ * @param s Pointer to the #qsched.
+ * @param type Task type.
+ * @param flags Any flags needed by the task.
+ * @param *data Pointer to the data required to decode the task.
+ * @param data_size Size of the data associated with the task.
+ * @param cost Estimate of the computational cost associated with the task.
+ *
+ * @return The ID of the new shared resource.
+ */
+long long int qsched_addtask ( struct qsched *s , int type , unsigned int flags , void *data , int data_size , int cost ){
+#ifndef WITH_MPI
+    error("Quicksched wasn't compiled with MPI support.");
+#else
+
+void *temp;
+    struct task *t;
+    long long int id;
+    int data_size2;
+
+    /* Lock the sched. */
+        lock_lock( &s->lock );
+    
+    /* Do the tasks need to be re-allocated? */
+    if ( s->count == s->size ) {
+    
+        /* Scale the task list size. */
+        s->size *= 2;
+        
+        /* Allocate a new task list. */
+        if ( ( temp = malloc( sizeof(struct task) * s->size ) ) == NULL )
+            error( "Failed to allocate new task list." );
+            
+        /* Copy the tasks over to the new list. */
+        memcpy( temp , s->tasks , sizeof(struct task) * s->count );
+        
+        /* Free the old task list. */
+        free( s->tasks );
+        
+        /* Set the new task list. */
+        s->tasks = (struct task *)temp;
+    
+        }
+        
+    /* Round-up the data size. */
+    data_size2 = ( data_size + (qsched_data_round-1) ) & ~(qsched_data_round-1);
+        
+    /* Do the task data need to be re-allocated? */
+    if ( s->count_data + data_size2 > s->size_data ) {
+    
+        /* Scale the task list size. */
+        s->size_data *= 2;
+        
+        /* Allocate a new task list. */
+        if ( ( temp = malloc( s->size_data ) ) == NULL )
+            error( "Failed to allocate new task list." );
+            
+        /* Copy the tasks over to the new list. */
+        memcpy( temp , s->data , s->count_data );
+        
+        /* Free the old task list. */
+        free( s->data );
+        
+        /* Set the new task list. */
+        s->data = temp;
+    
+        }
+        
+    /* Store the new task ID. */
+    id = s->count;
+    id |= ((long long int) s->rank << 48);    
+    
+    /* Init the new task. */
+    t = &s->tasks[ s->count ];
+    t->type = type;
+    t->flags = flags;
+    t->cost = cost;
+    t->wait = 0;
+    t->nr_conflicts = 0;
+    t->nr_unlocks = 0;
+    t->nr_locks = 0;
+    t->nr_uses = 0;
+    t->id = id;
+    
+    /* Add a relative pointer to the data. */
+    memcpy( &s->data[ s->count_data ] , data , data_size );
+    t->data = &s->data[ s->count_data ] - s->data;
+    s->count_data += data_size2;
+    
+    /* Increase the task counter. */
+    s->count += 1;
+    
+    t->node = s->rank;
+
+    /* The sched is now dirty. */
+    s->flags |= qsched_flag_dirty;
+    /* Unlock the sched. */
+    lock_unlock_blind( &s->lock );
+    
+    /* Return the task ID. */
+    return id;
+#endif
+}
 
 /**
  * @brief Add a resource requirement to a task.
@@ -1249,11 +1542,8 @@ long long int qsched_addres_local ( struct qsched *s , int owner , long long int
  * @param res ID of the resource.
  */
  
-void qsched_addlock_local ( struct qsched *s , long long int t , long long int res ) {
+void qsched_addlock ( struct qsched *s , long long int t , long long int res ) {
 
-#ifdef WITH_MPI
-    error("Don't use local functions with MPI setup.");
-#else
 
     void *temp1, *temp2;
 
@@ -1288,7 +1578,31 @@ void qsched_addlock_local ( struct qsched *s , long long int t , long long int r
     /* Add the new dependency. */
     s->locks[ s->count_locks ] = res;
     s->locks_key[ s->count_locks ] = t;
-    s->tasks[t].nr_locks += 1;
+    s->tasks[gettaskindex(t,s)].nr_locks += 1;
+
+    #ifdef WITH_MPI
+    /* Do the lockers need to be reallocated? */
+    if(s->count_lockers == s->size_lockers) {
+        s->size_lockers *= qsched_stretch;
+
+        if ( ( temp1 = malloc( sizeof(long long int) * s->size_lockers ) ) == NULL ||
+             ( temp2 = malloc( sizeof(long long int) * s->size_lockers ) ) == NULL )
+            error("Failed to allocate new lockers lists." );
+
+        memcpy( temp1, s->lockers, sizeof(long long int) * s->count_lockers );
+        memcpy( temp2, s->lockers_key, sizeof(long long int) * s->count_lockers );
+
+        /* Free the old lockers lists. */
+        free( s->lockers );
+        free( s->lockers_key );
+        
+        s->lockers = (long long int *) temp1;
+        s->lockers_key = (long long int *) temp2;
+    }
+    s->lockers[ s->count_lockers ] = t;
+    s->lockers_key[ s->count_lockers ] = res;
+    s->count_lockers += 1;
+    #endif
     
     /* Increase the locks counter. */
     s->count_locks += 1;
@@ -1298,7 +1612,6 @@ void qsched_addlock_local ( struct qsched *s , long long int t , long long int r
     
     /* Unlock the sched. */
     lock_unlock_blind( &s->lock );
-#endif
     }
 
 
@@ -1310,11 +1623,8 @@ void qsched_addlock_local ( struct qsched *s , long long int t , long long int r
  * @param res ID of the resource.
  */
  
-void qsched_adduse_local ( struct qsched *s , long long int t , long long int res ) {
+void qsched_adduse ( struct qsched *s , long long int t , long long int res ) {
 
-#ifdef WITH_MPI
-    error("Don't use local functions with MPI setup.");
-#else
 
     void *temp1, *temp2;
 
@@ -1349,7 +1659,7 @@ void qsched_adduse_local ( struct qsched *s , long long int t , long long int re
     /* Add the new dependency. */
     s->uses[ s->count_uses ] = res;
     s->uses_key[ s->count_uses ] = t;
-    s->tasks[t].nr_uses += 1;
+    s->tasks[gettaskindex(t, s)].nr_uses += 1;
     
     /* Increase the uses counter. */
     s->count_uses += 1;
@@ -1360,7 +1670,6 @@ void qsched_adduse_local ( struct qsched *s , long long int t , long long int re
     /* Unlock the sched. */
     lock_unlock_blind( &s->lock );
 
-#endif
     }
 
 
@@ -1374,10 +1683,7 @@ void qsched_adduse_local ( struct qsched *s , long long int t , long long int re
  * A dependency is added such that @c tb depends on @c ta.
  */
  
-void qsched_addunlock_local ( struct qsched *s , long long int ta , long long int tb ) {
-#ifdef WITH_MPI
-    error("Don't use local functions with MPI setup.");
-#else
+void qsched_addunlock ( struct qsched *s , long long int ta , long long int tb ) {
 
     void *temp1, *temp2;
 
@@ -1412,7 +1718,7 @@ void qsched_addunlock_local ( struct qsched *s , long long int ta , long long in
     /* Add the new dependency. */
     s->deps[ s->count_deps ] = tb;
     s->deps_key[ s->count_deps ] = ta;
-    s->tasks[ta].nr_unlocks += 1;
+    s->tasks[gettaskindex(ta,s)].nr_unlocks += 1;
     
     /* Increase the deps counter. */
     s->count_deps += 1;
@@ -1422,7 +1728,7 @@ void qsched_addunlock_local ( struct qsched *s , long long int ta , long long in
     
     /* Unlock the sched. */
     lock_unlock_blind( &s->lock );
-#endif
+
     }
 
 
@@ -1495,8 +1801,7 @@ long long int qsched_addtask_local ( struct qsched *s , int type , unsigned int
         }
         
     /* Store the new task ID. */
-    id = s->count;
-    printf("id = %lli\n", id);    
+    id = s->count; 
     /* Init the new task. */
     t = &s->tasks[ id ];
     t->type = type;
@@ -1524,7 +1829,8 @@ long long int qsched_addtask_local ( struct qsched *s , int type , unsigned int
 #endif
     }
     
-    
+
+
 /**
  * @brief Clean up a #qsched, free all associated memory.
  *
@@ -1642,7 +1948,20 @@ void qsched_init ( struct qsched *s , int nr_queues , int flags ) {
          ( s->uses_key = (long long int *)malloc( sizeof(long long int) * s->size_uses ) ) == NULL )
         error( "Failed to allocate memory for uses." );
     s->count_uses = 0;
-    
+   
+#ifdef WITH_MPI
+    s->size_users = qsched_init_usespertask * s->count_res;
+    if ( ( s->users = (long long int *)malloc( sizeof(long long int) * s->size_users ) ) == NULL ||
+         ( s->users_key = (long long int *)malloc( sizeof(long long int) * s->size_users ) ) == NULL )
+        error( "Failed to allocate memory for users." );
+    s->count_users = 0;
+
+    s->size_lockers = qsched_init_lockspertask * s->count_res;
+    if ( ( s->lockers = (long long int *)malloc( sizeof(long long int) * s->size_lockers ) ) == NULL ||
+         ( s->lockers = (long long int *)malloc( sizeof(long long int) * s->size_lockers ) ) == NULL )
+        error( "Failed to allocate memory for lockers." );
+#endif
+
     /* Allocate the initial data. */
     s->size_data = qsched_init_datapertask * s->size;
     if ( ( s->data = malloc( s->size_data ) ) == NULL )
diff --git a/src/qsched.h b/src/qsched.h
index 9848967..6797c7b 100644
--- a/src/qsched.h
+++ b/src/qsched.h
@@ -25,6 +25,7 @@
 #define qsched_flag_pthread              8
 #define qsched_flag_noreown              16
 #define qsched_flag_norecost             32
+#define qsched_flag_mpidirty             64
 
 /* Some sched-specific constants. */
 #define qsched_stretch                   2
@@ -175,9 +176,13 @@ struct qsched {
 
     long long int *lockers;
     long long int *lockers_key;
+    int size_lockers;
+    int count_lockers;
         
     long long int *users;
     long long int *users_key;
+    int size_users;
+    int count_users;
     /* MPICOMM TODO*/
     #endif
     
@@ -214,11 +219,16 @@ void qsched_unlocktask_local ( struct qsched *s , long long int tid );
 void qsched_prepare ( struct qsched *s );
 void qsched_enqueue ( struct qsched *s , struct task *t );
 
+/* External functions for MPI. */
+qsched_res_t qsched_address (struct qsched *s, int owner, int size, void **data );
+qsched_res_t qsched_addchildres( struct qsched *s, long long int parent, int owner, int size, int position, void **data);
+qsched_task_t qsched_addtask ( struct qsched *s , int type , unsigned int flags , void *data , int data_size , int cost );
+
 /* External functions. */
 void qsched_init ( struct qsched *s , int nr_queues , int flags );
 qsched_res_t qsched_addres_local ( struct qsched *s , int owner , qsched_res_t parent );
-void qsched_addlock_local ( struct qsched *s , qsched_task_t t , qsched_res_t res );
-void qsched_addunlock_local ( struct qsched *s , qsched_task_t ta , qsched_task_t tb );
+void qsched_addlock ( struct qsched *s , qsched_task_t t , qsched_res_t res );
+void qsched_addunlock ( struct qsched *s , qsched_task_t ta , qsched_task_t tb );
 qsched_task_t qsched_addtask_local ( struct qsched *s , int type , unsigned int flags , void *data , int data_size , int cost );
 void qsched_adduse ( struct qsched *s , qsched_task_t t , qsched_res_t res );
 void qsched_free ( struct qsched *s );
diff --git a/src/queue.c b/src/queue.c
index 062ffeb..ed9fbd3 100644
--- a/src/queue.c
+++ b/src/queue.c
@@ -25,6 +25,11 @@
 #include <stdlib.h>
 #include <string.h>
 
+#ifdef WITH_MPI
+    #include <mpi.h>
+
+#endif
+
 /* Local includes. */
 #include "error.h"
 #include "cycle.h"
diff --git a/src/res.h b/src/res.h
index 5fa5f68..cdea3f4 100644
--- a/src/res.h
+++ b/src/res.h
@@ -45,6 +45,9 @@ struct res {
     /* The ID of the resource. >>48 is node ID (where the resource was created), &0xFFFFFFFFFFFF is resource index.*/
     long long int ID;
 
+    /* The data offset in the parents data array. */
+    int offset;
+
     /* The pointer to the tasks that use this resource.*/
     int *users;
 
diff --git a/src/task.h b/src/task.h
index 42e5836..7976eae 100644
--- a/src/task.h
+++ b/src/task.h
@@ -60,4 +60,10 @@ struct task {
     /* Task weight for queue selection. */
     int cost, weight; 
 
+    #ifdef WITH_MPI
+    long long int id;
+
+    int node;
+    #endif
+
     };
-- 
GitLab