From 4b62acfe6cbfc0eaf895d99bc14d35f0bc534c74 Mon Sep 17 00:00:00 2001 From: aidan <aidan@gtx690.dur.ac.uk> Date: Tue, 26 May 2015 14:07:15 +0100 Subject: [PATCH] Some updates, first MPI functions added --- autogen.sh | 1 - configure.ac | 56 ++++++ examples/Makefile.am | 2 +- examples/test.c | 4 +- examples/test_bh.c | 12 +- examples/test_bh_sorted.c | 12 +- examples/test_fmm_sorted.c | 8 +- src/Makefile.am | 14 +- src/error.h | 2 +- src/qsched.c | 375 ++++++++++++++++++++++++++++++++++--- src/qsched.h | 14 +- src/queue.c | 5 + src/res.h | 3 + src/task.h | 6 + 14 files changed, 462 insertions(+), 52 deletions(-) diff --git a/autogen.sh b/autogen.sh index 2143e39..f4999f9 100755 --- a/autogen.sh +++ b/autogen.sh @@ -6,4 +6,3 @@ autoreconf --install --symlink exit - diff --git a/configure.ac b/configure.ac index 9511f8d..c5f61aa 100644 --- a/configure.ac +++ b/configure.ac @@ -32,6 +32,8 @@ AM_CONFIG_HEADER(config.h) # compiler settings #CFLAGS="-Wall $(CFLAGS)" + + # find and test the compiler AM_PROG_CC_C_O AC_PROG_CC_C99 @@ -41,6 +43,60 @@ AX_FUNC_POSIX_MEMALIGN AX_GCC_ARCHFLAG([no]) AX_EXT + +# Check for MPI. Need to do this before characterising the compiler (C99 mode), +# as this changes the compiler. +# We should consider using AX_PROG_CC_MPI to replace AC_PROG_CC when compiling +# whole applications. There are issues with mixing compilers when using this +# macro. See +# http://lists.gnu.org/archive/html/autoconf-archive-maintainers/2011-05/msg00004.html. +AC_ARG_ENABLE([mpi], + [AS_HELP_STRING([--enable-mpi], + [Compile with functionality for distributed-memory parallelism using MPI @<:@default=yes@:>@] + )], + [enable_mpi="$enableval"], + [enable_mpi="yes"] +) +if test "$enable_mpi" = "yes"; then + AX_MPI([CC="$MPICC" AC_DEFINE(HAVE_MPI, 1, [Define if you have the MPI library.]) ]) + + # Various MPI implementations require additional libraries when also using + # threads. Use mpirun (on PATH) as that seems to be only command with + # version flag. + AC_PATH_PROG([MPIRUN],[mpirun],[notfound]) + if test "$MPIRUN" = "notfound"; then + AC_MSG_WARN([Cannot find mpirun command on PATH, thread support may not be correct]) + else + # Special options we know about. + # Intel: -mt_mpi + # PLATFORM: -lmtmpi + # OpenMPI: nothing, but library should be built correctly. + # Set MPI_THREAD_LIBS and add to linker commands as necessary. + AC_MSG_CHECKING([MPI threads options]) + version=`$MPIRUN -version 2>&1` + case "$version" in + *Intel*MPI*) + MPI_THREAD_LIBS="-mt_mpi" + AC_MSG_RESULT([Intel MPI]) + ;; + *Platform*) + MPI_THREAD_LIBS="-lmtmpi" + AC_MSG_RESULT([PLATFORM MPI]) + ;; + *"Open MPI"*) + MPI_THREAD_LIBS="" + AC_MSG_RESULT([Open MPI]) + ;; + *) + MPI_THREAD_LIBS="" + AC_MSG_RESULT([unknown]) + ;; + esac + AC_SUBST([MPI_THREAD_LIBS]) + fi +fi +AM_CONDITIONAL([HAVEMPI],[test -n "$MPICC"]) + # autoconf stuff AC_PROG_INSTALL AC_PROG_MAKE_SET diff --git a/examples/Makefile.am b/examples/Makefile.am index 792c596..5d9d3f8 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -19,7 +19,7 @@ # Add the source directory and debug to CFLAGS AM_CFLAGS = -g -O3 -Wall -Werror -I../src -ffast-math -fstrict-aliasing \ -ftree-vectorize -funroll-loops $(SIMD_FLAGS) $(OPENMP_CFLAGS) \ - -DCPU_TPS=2.67e9 -DTIMERS \ + -DCPU_TPS=2.67e9 -DTIMERS -std=gnu99 \ # -fsanitize=address -fno-omit-frame-pointer # AM_CFLAGS = -g -O0 -Wall -Werror -I../src \ # -DCPU_TPS=2.67e9 -DTIMERS $(OPENMP_CFLAGS) \ diff --git a/examples/test.c b/examples/test.c index b065504..6c5d641 100644 --- a/examples/test.c +++ b/examples/test.c @@ -123,7 +123,7 @@ void test2 ( int m , int n , int k , int nr_threads ) { for ( kk = 0 ; kk < k ; kk++ ) { data[2] = kk; tid = qsched_addtask_local( &s , 1 , task_flag_none , data , 3*sizeof(int) , 1 ); - qsched_addlock_local( &s , tid , rid ); + qsched_addlock( &s , tid , rid ); } } @@ -224,7 +224,7 @@ void test1 ( int m , int n , int k , int nr_threads ) { data[0] = i; data[1] = j; rid = qsched_addres_local( &s , qsched_owner_none , qsched_res_none ); tid = qsched_addtask_local( &s , 1 , task_flag_none , data , 2*sizeof(int) , 1 ); - qsched_addlock_local( &s , tid , rid ); + qsched_addlock( &s , tid , rid ); printf("%lli, %lli\n", tid, rid); } diff --git a/examples/test_bh.c b/examples/test_bh.c index 1eab06c..afb42f4 100644 --- a/examples/test_bh.c +++ b/examples/test_bh.c @@ -367,9 +367,9 @@ void cell_split(struct cell *c, struct qsched *s) { struct cell *data[2] = {root, c}; int tid = qsched_addtask_local(s, task_type_self_pc, task_flag_none, data, 2 * sizeof(struct cell *), 1); - qsched_addlock_local(s, tid, c->res); + qsched_addlock(s, tid, c->res); #ifdef COM_AS_TASK - qsched_addunlock_local(s, root->com_tid, tid); + qsched_addunlock(s, root->com_tid, tid); #endif } /* does the cell need to be split? */ @@ -1065,12 +1065,12 @@ void create_tasks(struct qsched *s, struct cell *ci, struct cell *cj) { sizeof(struct cell *) * 2, ci->count * ci->count / 2); /* Add the resource (i.e. the cell) to the new task. */ - qsched_addlock_local(s, tid, ci->res); + qsched_addlock(s, tid, ci->res); /* If this call might recurse, add a dependency on the cell's COM task. */ #ifdef COM_AS_TASK - if (ci->split) qsched_addunlock_local(s, ci->com_tid, tid); + if (ci->split) qsched_addunlock(s, ci->com_tid, tid); #endif } @@ -1105,8 +1105,8 @@ void create_tasks(struct qsched *s, struct cell *ci, struct cell *cj) { sizeof(struct cell *) * 2, ci->count * cj->count); /* Add the resources. */ - qsched_addlock_local(s, tid, ci->res); - qsched_addlock_local(s, tid, cj->res); + qsched_addlock(s, tid, ci->res); + qsched_addlock(s, tid, cj->res); } } /* Cells are direct neighbours */ } /* Otherwise it's a pair */ diff --git a/examples/test_bh_sorted.c b/examples/test_bh_sorted.c index a0448a5..1d0bc72 100644 --- a/examples/test_bh_sorted.c +++ b/examples/test_bh_sorted.c @@ -730,9 +730,9 @@ void cell_split(struct cell *c, struct qsched *s) { struct cell *data[2] = {root, c}; int tid = qsched_addtask_local(s, task_type_self_pc, task_flag_none, data, 2 * sizeof(struct cell *), 1); - qsched_addlock_local(s, tid, c->res); + qsched_addlock(s, tid, c->res); #ifdef COM_AS_TASK - qsched_addunlock_local(s, root->com_tid, tid); + qsched_addunlock(s, root->com_tid, tid); #endif } /* does the cell need to be split? */ @@ -1543,12 +1543,12 @@ void create_tasks(struct qsched *s, struct cell *ci, struct cell *cj) { sizeof(struct cell *) * 2, ci->count * ci->count / 2); /* Add the resource (i.e. the cell) to the new task. */ - qsched_addlock_local(s, tid, ci->res); + qsched_addlock(s, tid, ci->res); /* If this call might recurse, add a dependency on the cell's COM task. */ #ifdef COM_AS_TASK - if (ci->split) qsched_addunlock_local(s, ci->com_tid, tid); + if (ci->split) qsched_addunlock(s, ci->com_tid, tid); #endif } @@ -1583,8 +1583,8 @@ void create_tasks(struct qsched *s, struct cell *ci, struct cell *cj) { sizeof(struct cell *) * 2, ci->count * cj->count); /* Add the resources. */ - qsched_addlock_local(s, tid, ci->res); - qsched_addlock_local(s, tid, cj->res); + qsched_addlock(s, tid, ci->res); + qsched_addlock(s, tid, cj->res); } } /* Cells are direct neighbours */ } /* Otherwise it's a pair */ diff --git a/examples/test_fmm_sorted.c b/examples/test_fmm_sorted.c index 9ffd3d9..3adc3c6 100644 --- a/examples/test_fmm_sorted.c +++ b/examples/test_fmm_sorted.c @@ -1496,12 +1496,12 @@ void create_tasks(struct qsched *s, struct cell *ci, struct cell *cj) { sizeof(struct cell *) * 2, ci->count * ci->count / 2); /* Add the resource (i.e. the cell) to the new task. */ - qsched_addlock_local(s, tid, ci->res); + qsched_addlock(s, tid, ci->res); /* If this call might recurse, add a dependency on the cell's COM task. */ #ifdef COM_AS_TASK - if (ci->split) qsched_addunlock_local(s, ci->com_tid, tid); + if (ci->split) qsched_addunlock(s, ci->com_tid, tid); #endif } @@ -1536,8 +1536,8 @@ void create_tasks(struct qsched *s, struct cell *ci, struct cell *cj) { sizeof(struct cell *) * 2, ci->count * cj->count); /* Add the resources. */ - qsched_addlock_local(s, tid, ci->res); - qsched_addlock_local(s, tid, cj->res); + qsched_addlock(s, tid, ci->res); + qsched_addlock(s, tid, cj->res); } } /* Cells are direct neighbours */ } /* Otherwise it's a pair */ diff --git a/src/Makefile.am b/src/Makefile.am index c953a49..6573232 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -17,7 +17,7 @@ # Add the debug flag to the whole thing AM_CFLAGS = -g -O3 -Wall -Werror -ffast-math -fstrict-aliasing -ftree-vectorize \ - -funroll-loops $(SIMD_FLAGS) $(OPENMP_CFLAGS) -DTIMERS \ + -funroll-loops $(SIMD_FLAGS) $(OPENMP_CFLAGS) -DTIMERS -std=gnu99 \ # -fsanitize=address -fno-omit-frame-pointer # Assign a "safe" version number @@ -25,8 +25,20 @@ AM_LDFLAGS = -version-info 0:0:0 # Build the libquicksched library lib_LTLIBRARIES = libquicksched.la + +if HAVEMPI +lib_LTLIBRARIES += libquickschedMPI.la +endif + libquicksched_la_SOURCES = qsched.c queue.c +#libquickschedMPI_la_CC = mpicc +libquickschedMPI_la_LDFLAGS = $(MPI_THREAD_LIBS) +libquickschedMPI_la_CFLAGS = -g -O3 -Wall -Werror -ffast-math -fstrict-aliasing -ftree-vectorize \ + -funroll-loops $(SIMD_FLAGS) $(OPENMP_CFLAGS) -DTIMERS -std=gnu99 \ + -DWITH_MPI +libquickschedMPI_la_SOURCES = qsched.c queue.c + # List required headers include_HEADERS = atomic.h lock.h queue.h qsched.h task.h res.h error.h diff --git a/src/error.h b/src/error.h index d0ca19d..b6b3b5a 100644 --- a/src/error.h +++ b/src/error.h @@ -20,7 +20,7 @@ /* Error macro. */ #ifdef WITH_MPI -#define error(s, ...) { fprintf( stderr , "%s:%s():%i: " s "\n" , __FILE__ , __FUNCTION__ , __LINE__ , ##__VA_ARGS__ ); MPI_FINALIZE(); abort(); } +#define error(s, ...) { fprintf( stderr , "%s:%s():%i: " s "\n" , __FILE__ , __FUNCTION__ , __LINE__ , ##__VA_ARGS__ ); MPI_Finalize(); abort(); } #else #define error(s, ...) { fprintf( stderr , "%s:%s():%i: " s "\n" , __FILE__ , __FUNCTION__ , __LINE__ , ##__VA_ARGS__ ); abort(); } #endif diff --git a/src/qsched.c b/src/qsched.c index 50dac33..eb7e2ff 100644 --- a/src/qsched.c +++ b/src/qsched.c @@ -30,6 +30,10 @@ #include <omp.h> #endif +#ifdef WITH_MPI + #include <mpi.h> +#endif + // /* Pthread headers, only if available. */ #ifdef HAVE_PTHREAD #include <pthread.h> @@ -45,6 +49,36 @@ #include "qsched.h" #include "queue.h" +#ifdef WITH_MPI +inline int getindex(long long int id , struct qsched *s) +{ + return s->res_ranks[id>>48] + (id & 0xFFFFFFFFFFFFFF); +} + +inline struct res* getres(long long int id, struct qsched *s) +{ + return &s->res[ getindex(id, s) ]; +} + +inline int gettaskindex(long long int id, struct qsched *s) +{ + return s->task_ranks[id>>48] + (id & 0xFFFFFFFFFFFFFF); +} +#else +inline int getindex(long long int id, struct qsched *s){ + return id; +} + +inline struct res* getres(long long int id, struct qsched *s) +{ + return &s->res[id]; +} + +inline int gettaskindex(long long int id, struct qsched *s) +{ + return id; +} +#endif /** Timer names. */ char *qsched_timer_names[ qsched_timer_count ] = @@ -645,7 +679,7 @@ void qsched_done ( struct qsched *s , struct task *t ) { /* Careful, this may pick up duplicate timers if virtual tasks are used. */ - TIMER_TOC( s , qsched_timer_done ) + TIMER_TOC( s , qsched_timer_done ); } @@ -841,7 +875,7 @@ struct task *qsched_gettask ( struct qsched *s , int qid ) { { TIMER_TIC tid = queue_get( &s->queues[qid] , s , 1 ); - TIMER_TOC( s , qsched_timer_queue ) + TIMER_TOC( s , qsched_timer_queue ); if ( tid < 0 ) { /* Otherwise, hit the other queues. */ @@ -852,7 +886,7 @@ struct task *qsched_gettask ( struct qsched *s , int qid ) { k = rand() % naq; TIMER_TIC2 tid = queue_get( &s->queues[ qids[k] ] , s , 0 ); - TIMER_TOC( s , qsched_timer_queue ) + TIMER_TOC( s , qsched_timer_queue ); if ( tid < 0 ) qids[k] = qids[ --naq ]; else @@ -881,7 +915,7 @@ struct task *qsched_gettask ( struct qsched *s , int qid ) { t->qid = qid; /* Return the task. */ - TIMER_TOC( s , qsched_timer_gettask ) + TIMER_TOC( s , qsched_timer_gettask ); return t; } @@ -889,7 +923,7 @@ struct task *qsched_gettask ( struct qsched *s , int qid ) { /* Otherwise, take a nap? */ #ifdef HAVE_PTHREAD else if ( s->flags & qsched_flag_yield ) { - TIMER_TOC( s , qsched_timer_gettask ) + TIMER_TOC( s , qsched_timer_gettask ); pthread_mutex_lock( &s->mutex ); if ( s->waiting ) pthread_cond_wait( &s->cond , &s->mutex ); @@ -1173,7 +1207,7 @@ void qsched_prepare ( struct qsched *s ) { /* Unlock the sched. */ lock_unlock_blind( &s->lock ); - TIMER_TOC( s , qsched_timer_prepare ) + TIMER_TOC( s , qsched_timer_prepare ); } @@ -1240,6 +1274,265 @@ long long int qsched_addres_local ( struct qsched *s , int owner , long long int #endif } +/** + * @brief Add a new resource to the #qsched. + * + * @param s Pointer to the #qsched. + * @param owner ID of the ower + * @param size Size of the data associated with the resource in bytes. + * @param data Pointer to the pointer which should hold the data. + * + * @return The ID of the new shared resource. + */ + +long long int qsched_addres ( struct qsched *s, int owner, int size, void **data) { +#ifndef WITH_MPI + error("Quicksched wasn't compiled with MPI support."); +#else +struct res *res_new; + long long int id; + + lock_lock( &s->lock); + + //Reallocate res array if neccessary. + /* Do the deps need to be re-allocated? */ + if ( s->count_res == s->size_res ) { + + /* Scale the res list size. */ + s->size_res *= 2; + + /* Allocate a new task list. */ + if ( ( res_new = malloc( sizeof(struct res) * s->size_res ) ) == NULL ) + error( "Failed to allocate new res lists." ); + + /* Copy the res and owners over to the new list. */ + memcpy( res_new , s->res , sizeof(struct res) * s->count_res ); + + /* Free the old res lists. */ + free( s->res ); + + /* Set the new res lists. */ + s->res = res_new; + + } + + id = s->count_res; + id |= ((long long int)s->rank << 48); + s->count_res += 1; + int index = getindex(id, s); + + lock_init( &s->res[ getindex(id, s) ]. lock); + s->res[ index ].hold = 0; + s->res[ index ].owner = owner; + s->res[ index ].parent = -1; + s->res[ index ].offset = 0; + s->res[ index ].node = s->rank; + s->res[ index ].data = calloc(size, 1); + s->res[ index ].size = size; + //Users and locks are set after res_sync, when task creation begins. + s->res[ index ].users = NULL; + s->res[ index ].lockers = NULL; + s->res[ index ].ID = id; + s->res[ index ].num_lockers = 0; + s->res[ index ].num_users = 0; + + *data = s->res[index].data; + + /* The sched is now dirty. */ + s->flags |= qsched_flag_dirty; + s->flags |= qsched_flag_mpidirty; + + lock_unlock_blind( &s->lock); + + + + + return id; +#endif +} + +/** + * @brief Add a new child resource to the #qsched. + * + * @param s Pointer to the #qsched. + * @param parent ID of the parent resource. This must be accurate. + * @param owner ID of the ower + * @param size Size of the data associated with the resource in bytes. + * @param position Position in the data array of the parent that this resource's data is located. + * @param data Pointer to the pointer which should hold the data. + * + * @return The ID of the new shared resource. + */ +long long int qsched_addchildres( struct qsched *s, long long int parent, int owner, int size, int position, void **data) +{ +#ifndef WITH_MPI + error("Quicksched wasn't compiled with MPI support."); +#else + struct res *res_new; + long long int id; + + lock_lock( &s->lock); + //Reallocate res array if neccessary. + /* Do the deps need to be re-allocated? */ + if ( s->count_res == s->size_res ) { + + /* Scale the res list size. */ + s->size_res *= 2; + + /* Allocate a new task list. */ + if ( ( res_new = malloc( sizeof(struct res) * s->size_res ) ) == NULL ) + error( "Failed to allocate new res lists." ); + + /* Copy the res and owners over to the new list. */ + memcpy( res_new , s->res , sizeof(struct res) * s->count_res ); + + /* Free the old res lists. */ + free( s->res ); + + /* Set the new res lists. */ + s->res = res_new; + + } + + id = s->count_res; + id |= ((long long int)s->rank << 48); +int index = getindex(id,s); + s->count_res += 1; + + + //TODO lock_init( &s->res[ getindex(id,s) ]. lock); + s->res[ index ].hold = 0; + s->res[ index ].owner = owner; + s->res[ index ].parent = parent; + s->res[ index ].offset = position; + s->res[ index ].node = s->rank; + int *temp = (int*) s->res[getindex(parent,s)].data; + s->res[ index ].data = &temp[position]; + s->res[ index ].size = size; + s->res[ index ].users = NULL; + s->res[ index ].lockers = NULL; + s->res[ index ].ID = id; + + *data = s->res[ index ].data; + + //TODO Check for non-overlapping memory in children? I don’t know how best to do this. + + /* The sched is now dirty. */ + s->flags |= qsched_flag_dirty; + s->flags |= qsched_flag_mpidirty; + + lock_unlock_blind(&s->lock); + + return id; + +#endif +} + +/** + * @brief Add a task to the #qsched. + * + * @param s Pointer to the #qsched. + * @param type Task type. + * @param flags Any flags needed by the task. + * @param *data Pointer to the data required to decode the task. + * @param data_size Size of the data associated with the task. + * @param cost Estimate of the computational cost associated with the task. + * + * @return The ID of the new shared resource. + */ +long long int qsched_addtask ( struct qsched *s , int type , unsigned int flags , void *data , int data_size , int cost ){ +#ifndef WITH_MPI + error("Quicksched wasn't compiled with MPI support."); +#else + +void *temp; + struct task *t; + long long int id; + int data_size2; + + /* Lock the sched. */ + lock_lock( &s->lock ); + + /* Do the tasks need to be re-allocated? */ + if ( s->count == s->size ) { + + /* Scale the task list size. */ + s->size *= 2; + + /* Allocate a new task list. */ + if ( ( temp = malloc( sizeof(struct task) * s->size ) ) == NULL ) + error( "Failed to allocate new task list." ); + + /* Copy the tasks over to the new list. */ + memcpy( temp , s->tasks , sizeof(struct task) * s->count ); + + /* Free the old task list. */ + free( s->tasks ); + + /* Set the new task list. */ + s->tasks = (struct task *)temp; + + } + + /* Round-up the data size. */ + data_size2 = ( data_size + (qsched_data_round-1) ) & ~(qsched_data_round-1); + + /* Do the task data need to be re-allocated? */ + if ( s->count_data + data_size2 > s->size_data ) { + + /* Scale the task list size. */ + s->size_data *= 2; + + /* Allocate a new task list. */ + if ( ( temp = malloc( s->size_data ) ) == NULL ) + error( "Failed to allocate new task list." ); + + /* Copy the tasks over to the new list. */ + memcpy( temp , s->data , s->count_data ); + + /* Free the old task list. */ + free( s->data ); + + /* Set the new task list. */ + s->data = temp; + + } + + /* Store the new task ID. */ + id = s->count; + id |= ((long long int) s->rank << 48); + + /* Init the new task. */ + t = &s->tasks[ s->count ]; + t->type = type; + t->flags = flags; + t->cost = cost; + t->wait = 0; + t->nr_conflicts = 0; + t->nr_unlocks = 0; + t->nr_locks = 0; + t->nr_uses = 0; + t->id = id; + + /* Add a relative pointer to the data. */ + memcpy( &s->data[ s->count_data ] , data , data_size ); + t->data = &s->data[ s->count_data ] - s->data; + s->count_data += data_size2; + + /* Increase the task counter. */ + s->count += 1; + + t->node = s->rank; + + /* The sched is now dirty. */ + s->flags |= qsched_flag_dirty; + /* Unlock the sched. */ + lock_unlock_blind( &s->lock ); + + /* Return the task ID. */ + return id; +#endif +} /** * @brief Add a resource requirement to a task. @@ -1249,11 +1542,8 @@ long long int qsched_addres_local ( struct qsched *s , int owner , long long int * @param res ID of the resource. */ -void qsched_addlock_local ( struct qsched *s , long long int t , long long int res ) { +void qsched_addlock ( struct qsched *s , long long int t , long long int res ) { -#ifdef WITH_MPI - error("Don't use local functions with MPI setup."); -#else void *temp1, *temp2; @@ -1288,7 +1578,31 @@ void qsched_addlock_local ( struct qsched *s , long long int t , long long int r /* Add the new dependency. */ s->locks[ s->count_locks ] = res; s->locks_key[ s->count_locks ] = t; - s->tasks[t].nr_locks += 1; + s->tasks[gettaskindex(t,s)].nr_locks += 1; + + #ifdef WITH_MPI + /* Do the lockers need to be reallocated? */ + if(s->count_lockers == s->size_lockers) { + s->size_lockers *= qsched_stretch; + + if ( ( temp1 = malloc( sizeof(long long int) * s->size_lockers ) ) == NULL || + ( temp2 = malloc( sizeof(long long int) * s->size_lockers ) ) == NULL ) + error("Failed to allocate new lockers lists." ); + + memcpy( temp1, s->lockers, sizeof(long long int) * s->count_lockers ); + memcpy( temp2, s->lockers_key, sizeof(long long int) * s->count_lockers ); + + /* Free the old lockers lists. */ + free( s->lockers ); + free( s->lockers_key ); + + s->lockers = (long long int *) temp1; + s->lockers_key = (long long int *) temp2; + } + s->lockers[ s->count_lockers ] = t; + s->lockers_key[ s->count_lockers ] = res; + s->count_lockers += 1; + #endif /* Increase the locks counter. */ s->count_locks += 1; @@ -1298,7 +1612,6 @@ void qsched_addlock_local ( struct qsched *s , long long int t , long long int r /* Unlock the sched. */ lock_unlock_blind( &s->lock ); -#endif } @@ -1310,11 +1623,8 @@ void qsched_addlock_local ( struct qsched *s , long long int t , long long int r * @param res ID of the resource. */ -void qsched_adduse_local ( struct qsched *s , long long int t , long long int res ) { +void qsched_adduse ( struct qsched *s , long long int t , long long int res ) { -#ifdef WITH_MPI - error("Don't use local functions with MPI setup."); -#else void *temp1, *temp2; @@ -1349,7 +1659,7 @@ void qsched_adduse_local ( struct qsched *s , long long int t , long long int re /* Add the new dependency. */ s->uses[ s->count_uses ] = res; s->uses_key[ s->count_uses ] = t; - s->tasks[t].nr_uses += 1; + s->tasks[gettaskindex(t, s)].nr_uses += 1; /* Increase the uses counter. */ s->count_uses += 1; @@ -1360,7 +1670,6 @@ void qsched_adduse_local ( struct qsched *s , long long int t , long long int re /* Unlock the sched. */ lock_unlock_blind( &s->lock ); -#endif } @@ -1374,10 +1683,7 @@ void qsched_adduse_local ( struct qsched *s , long long int t , long long int re * A dependency is added such that @c tb depends on @c ta. */ -void qsched_addunlock_local ( struct qsched *s , long long int ta , long long int tb ) { -#ifdef WITH_MPI - error("Don't use local functions with MPI setup."); -#else +void qsched_addunlock ( struct qsched *s , long long int ta , long long int tb ) { void *temp1, *temp2; @@ -1412,7 +1718,7 @@ void qsched_addunlock_local ( struct qsched *s , long long int ta , long long in /* Add the new dependency. */ s->deps[ s->count_deps ] = tb; s->deps_key[ s->count_deps ] = ta; - s->tasks[ta].nr_unlocks += 1; + s->tasks[gettaskindex(ta,s)].nr_unlocks += 1; /* Increase the deps counter. */ s->count_deps += 1; @@ -1422,7 +1728,7 @@ void qsched_addunlock_local ( struct qsched *s , long long int ta , long long in /* Unlock the sched. */ lock_unlock_blind( &s->lock ); -#endif + } @@ -1495,8 +1801,7 @@ long long int qsched_addtask_local ( struct qsched *s , int type , unsigned int } /* Store the new task ID. */ - id = s->count; - printf("id = %lli\n", id); + id = s->count; /* Init the new task. */ t = &s->tasks[ id ]; t->type = type; @@ -1524,7 +1829,8 @@ long long int qsched_addtask_local ( struct qsched *s , int type , unsigned int #endif } - + + /** * @brief Clean up a #qsched, free all associated memory. * @@ -1642,7 +1948,20 @@ void qsched_init ( struct qsched *s , int nr_queues , int flags ) { ( s->uses_key = (long long int *)malloc( sizeof(long long int) * s->size_uses ) ) == NULL ) error( "Failed to allocate memory for uses." ); s->count_uses = 0; - + +#ifdef WITH_MPI + s->size_users = qsched_init_usespertask * s->count_res; + if ( ( s->users = (long long int *)malloc( sizeof(long long int) * s->size_users ) ) == NULL || + ( s->users_key = (long long int *)malloc( sizeof(long long int) * s->size_users ) ) == NULL ) + error( "Failed to allocate memory for users." ); + s->count_users = 0; + + s->size_lockers = qsched_init_lockspertask * s->count_res; + if ( ( s->lockers = (long long int *)malloc( sizeof(long long int) * s->size_lockers ) ) == NULL || + ( s->lockers = (long long int *)malloc( sizeof(long long int) * s->size_lockers ) ) == NULL ) + error( "Failed to allocate memory for lockers." ); +#endif + /* Allocate the initial data. */ s->size_data = qsched_init_datapertask * s->size; if ( ( s->data = malloc( s->size_data ) ) == NULL ) diff --git a/src/qsched.h b/src/qsched.h index 9848967..6797c7b 100644 --- a/src/qsched.h +++ b/src/qsched.h @@ -25,6 +25,7 @@ #define qsched_flag_pthread 8 #define qsched_flag_noreown 16 #define qsched_flag_norecost 32 +#define qsched_flag_mpidirty 64 /* Some sched-specific constants. */ #define qsched_stretch 2 @@ -175,9 +176,13 @@ struct qsched { long long int *lockers; long long int *lockers_key; + int size_lockers; + int count_lockers; long long int *users; long long int *users_key; + int size_users; + int count_users; /* MPICOMM TODO*/ #endif @@ -214,11 +219,16 @@ void qsched_unlocktask_local ( struct qsched *s , long long int tid ); void qsched_prepare ( struct qsched *s ); void qsched_enqueue ( struct qsched *s , struct task *t ); +/* External functions for MPI. */ +qsched_res_t qsched_address (struct qsched *s, int owner, int size, void **data ); +qsched_res_t qsched_addchildres( struct qsched *s, long long int parent, int owner, int size, int position, void **data); +qsched_task_t qsched_addtask ( struct qsched *s , int type , unsigned int flags , void *data , int data_size , int cost ); + /* External functions. */ void qsched_init ( struct qsched *s , int nr_queues , int flags ); qsched_res_t qsched_addres_local ( struct qsched *s , int owner , qsched_res_t parent ); -void qsched_addlock_local ( struct qsched *s , qsched_task_t t , qsched_res_t res ); -void qsched_addunlock_local ( struct qsched *s , qsched_task_t ta , qsched_task_t tb ); +void qsched_addlock ( struct qsched *s , qsched_task_t t , qsched_res_t res ); +void qsched_addunlock ( struct qsched *s , qsched_task_t ta , qsched_task_t tb ); qsched_task_t qsched_addtask_local ( struct qsched *s , int type , unsigned int flags , void *data , int data_size , int cost ); void qsched_adduse ( struct qsched *s , qsched_task_t t , qsched_res_t res ); void qsched_free ( struct qsched *s ); diff --git a/src/queue.c b/src/queue.c index 062ffeb..ed9fbd3 100644 --- a/src/queue.c +++ b/src/queue.c @@ -25,6 +25,11 @@ #include <stdlib.h> #include <string.h> +#ifdef WITH_MPI + #include <mpi.h> + +#endif + /* Local includes. */ #include "error.h" #include "cycle.h" diff --git a/src/res.h b/src/res.h index 5fa5f68..cdea3f4 100644 --- a/src/res.h +++ b/src/res.h @@ -45,6 +45,9 @@ struct res { /* The ID of the resource. >>48 is node ID (where the resource was created), &0xFFFFFFFFFFFF is resource index.*/ long long int ID; + /* The data offset in the parents data array. */ + int offset; + /* The pointer to the tasks that use this resource.*/ int *users; diff --git a/src/task.h b/src/task.h index 42e5836..7976eae 100644 --- a/src/task.h +++ b/src/task.h @@ -60,4 +60,10 @@ struct task { /* Task weight for queue selection. */ int cost, weight; + #ifdef WITH_MPI + long long int id; + + int node; + #endif + }; -- GitLab