diff --git a/src/cell.h b/src/cell.h
index 10454a9abb1a12730aa6d7af5406c4d543f1136c..3ae9617e821ded4ad80a5cfa7a2bd5dd3e726516 100644
--- a/src/cell.h
+++ b/src/cell.h
@@ -96,7 +96,7 @@ struct cell {
int sortsize;
/* The tasks computing this cell's density. */
- struct task *density[27], *force[27];
+ struct task *density[8*27], *force[8*27];
int nr_density, nr_force;
/* The ghost task to link density to interactions. */
diff --git a/src/engine.c b/src/engine.c
index 920545dfa7c20d032e4ea927bee6a73704d94d50..f0fab9fa04cc59dd90d8f1691613177d08b820d3 100644
--- a/src/engine.c
+++ b/src/engine.c
@@ -463,6 +463,7 @@ void engine_repartition ( struct engine *e ) {
void engine_addtasks_send ( struct engine *e , struct cell *ci , struct cell *cj ) {
int k;
+ struct scheduler *s = &e->sched;
/* Check if any of the density tasks are for the target node. */
for ( k = 0 ; k < ci->nr_density ; k++ )
@@ -478,13 +479,13 @@ void engine_addtasks_send ( struct engine *e , struct cell *ci , struct cell *cj
struct task *t_rho = scheduler_addtask( &e->sched , task_type_send_rho , task_subtype_none , 2*ci->tag + 1 , 0 , ci , cj , 0 );
/* The send_rho task depends on the cell's ghost task. */
- task_addunlock( ci->ghost , t_rho );
+ scheduler_addunlock( s , ci->ghost , t_rho );
/* The send_rho task should unlock the super-cell's kick2 task. */
- task_addunlock( t_rho , ci->super->kick2 );
+ scheduler_addunlock( s , t_rho , ci->super->kick2 );
/* The send_xv task should unlock the super-cell's ghost task. */
- task_addunlock( t_xv , ci->super->ghost );
+ scheduler_addunlock( s , t_xv , ci->super->ghost );
}
@@ -509,6 +510,7 @@ void engine_addtasks_send ( struct engine *e , struct cell *ci , struct cell *cj
void engine_addtasks_recv ( struct engine *e , struct cell *c , struct task *t_xv , struct task *t_rho ) {
int k;
+ struct scheduler *s = &e->sched;
/* Do we need to construct a recv task? */
if ( t_xv != NULL || c->nr_density > 0 ) {
@@ -520,8 +522,8 @@ void engine_addtasks_recv ( struct engine *e , struct cell *c , struct task *t_x
/* If there has been a higher-up recv task, then these tasks
are implicit and depend on the higher-up task. */
if ( t_xv != NULL ) {
- task_addunlock( c->parent->recv_xv , c->recv_xv );
- task_addunlock( c->parent->recv_rho , c->recv_rho );
+ scheduler_addunlock( s , c->parent->recv_xv , c->recv_xv );
+ scheduler_addunlock( s , c->parent->recv_rho , c->recv_rho );
c->recv_xv->implicit = 1;
c->recv_rho->implicit = 1;
}
@@ -532,13 +534,13 @@ void engine_addtasks_recv ( struct engine *e , struct cell *c , struct task *t_x
/* Add dependencies if there are density/force tasks. */
for ( k = 0 ; k < c->nr_density ; k++ ) {
- task_addunlock( c->recv_xv , c->density[k] );
- task_addunlock( c->density[k] , t_rho );
+ scheduler_addunlock( s , c->recv_xv , c->density[k] );
+ scheduler_addunlock( s , c->density[k] , t_rho );
}
for ( k = 0 ; k < c->nr_force ; k++ )
- task_addunlock( c->recv_rho , c->force[k] );
+ scheduler_addunlock( s , c->recv_rho , c->force[k] );
if ( c->sorts != NULL )
- task_addunlock( c->recv_xv , c->sorts );
+ scheduler_addunlock( s , c->recv_xv , c->sorts );
}
@@ -850,7 +852,7 @@ void engine_maketasks ( struct engine *e ) {
for ( j = 0 ; j < 8 ; j++ )
if ( t->ci->progeny[j] != NULL && t->ci->progeny[j]->sorts != NULL ) {
t->ci->progeny[j]->sorts->skip = 0;
- task_addunlock( t->ci->progeny[j]->sorts , t );
+ scheduler_addunlock( sched , t->ci->progeny[j]->sorts , t );
}
if ( t->type == task_type_self ) {
atomic_inc( &t->ci->nr_tasks );
@@ -900,10 +902,10 @@ void engine_maketasks ( struct engine *e ) {
/* Self-interaction? */
if ( t->type == task_type_self && t->subtype == task_subtype_density ) {
- task_addunlock( t , t->ci->super->ghost );
+ scheduler_addunlock( sched , t , t->ci->super->ghost );
t2 = scheduler_addtask( sched , task_type_self , task_subtype_force , 0 , 0 , t->ci , NULL , 0 );
- task_addunlock( t->ci->ghost , t2 );
- task_addunlock( t2 , t->ci->super->kick2 );
+ scheduler_addunlock( sched , t->ci->ghost , t2 );
+ scheduler_addunlock( sched , t2 , t->ci->super->kick2 );
t->ci->force[ atomic_inc( &t->ci->nr_force ) ] = t2;
}
@@ -911,15 +913,15 @@ void engine_maketasks ( struct engine *e ) {
else if ( t->type == task_type_pair && t->subtype == task_subtype_density ) {
t2 = scheduler_addtask( sched , task_type_pair , task_subtype_force , 0 , 0 , t->ci , t->cj , 0 );
if ( t->ci->nodeID == e->nodeID ) {
- task_addunlock( t->ci->ghost , t2 );
- task_addunlock( t , t->ci->super->ghost );
- task_addunlock( t2 , t->ci->super->kick2 );
+ scheduler_addunlock( sched , t->ci->ghost , t2 );
+ scheduler_addunlock( sched , t , t->ci->super->ghost );
+ scheduler_addunlock( sched , t2 , t->ci->super->kick2 );
}
if ( t->cj->nodeID == e->nodeID ) {
- task_addunlock( t->cj->ghost , t2 );
+ scheduler_addunlock( sched , t->cj->ghost , t2 );
if ( t->ci->super != t->cj->super ) {
- task_addunlock( t , t->cj->super->ghost );
- task_addunlock( t2 , t->cj->super->kick2 );
+ scheduler_addunlock( sched , t , t->cj->super->ghost );
+ scheduler_addunlock( sched , t2 , t->cj->super->kick2 );
}
}
t->ci->force[ atomic_inc( &t->ci->nr_force ) ] = t2;
@@ -930,15 +932,15 @@ void engine_maketasks ( struct engine *e ) {
else if ( t->type == task_type_sub && t->subtype == task_subtype_density ) {
t2 = scheduler_addtask( sched , task_type_sub , task_subtype_force , t->flags , 0 , t->ci , t->cj , 0 );
if ( t->ci->nodeID == e->nodeID ) {
- task_addunlock( t , t->ci->super->ghost );
- task_addunlock( t->ci->ghost , t2 );
- task_addunlock( t2 , t->ci->super->kick2 );
+ scheduler_addunlock( sched , t , t->ci->super->ghost );
+ scheduler_addunlock( sched , t->ci->ghost , t2 );
+ scheduler_addunlock( sched , t2 , t->ci->super->kick2 );
}
if ( t->cj != NULL && t->cj->nodeID == e->nodeID ) {
- task_addunlock( t->cj->ghost , t2 );
+ scheduler_addunlock( sched , t->cj->ghost , t2 );
if ( t->ci->super != t->cj->super ) {
- task_addunlock( t , t->cj->super->ghost );
- task_addunlock( t2 , t->cj->super->kick2 );
+ scheduler_addunlock( sched , t , t->cj->super->ghost );
+ scheduler_addunlock( sched , t2 , t->cj->super->kick2 );
}
}
t->ci->force[ atomic_inc( &t->ci->nr_force ) ] = t2;
@@ -1219,7 +1221,8 @@ void engine_prepare ( struct engine *e ) {
(1 << task_type_send_xv) |
(1 << task_type_recv_xv) |
(1 << task_type_send_rho) |
- (1 << task_type_recv_rho) );
+ (1 << task_type_recv_rho) |
+ (1 << task_type_link) );
// message( "scheduler_start took %.3f ms." , (double)(getticks() - tic2)/CPU_TPS*1000 );
TIMER_TOC( timer_prepare );
@@ -1493,7 +1496,7 @@ void engine_step ( struct engine *e ) {
/* First kick. */
if ( e->step == 0 || !( e->policy & engine_policy_fixdt ) ) {
TIMER_TIC
- scheduler_start( &e->sched , (1 << task_type_kick1) );
+ scheduler_start( &e->sched , (1 << task_type_kick1) | (1 << task_type_link) );
engine_launch( e , ( e->nr_threads > 8 ) ? 8 : e->nr_threads );
TIMER_TOC( timer_kick1 );
}
diff --git a/src/scheduler.c b/src/scheduler.c
index 64084571d7db1d16c4d373cfb522405759bb8d33..bda22f07346d8a7adf00045e65bee9533daf3947 100644
--- a/src/scheduler.c
+++ b/src/scheduler.c
@@ -50,6 +50,53 @@
#include "scheduler.h"
+/**
+ * @brief Add an unlock_task to the given task.
+ *
+ * @param s The #scheduler.
+ * @param ta The unlocking #task.
+ * @param tb The #task that will be unlocked.
+ */
+
+void scheduler_addunlock ( struct scheduler *s , struct task *ta , struct task *tb ) {
+
+ /* Main loop. */
+ while ( 1 ) {
+
+ /* Follow the links. */
+ while ( ta->link != NULL )
+ ta = ta->link;
+
+ /* Get the index of the next free task. */
+ int ind = atomic_inc( &ta->nr_unlock_tasks );
+
+ /* Is there room in this task? */
+ if ( ind < task_maxunlock ) {
+ ta->unlock_tasks[ ind ] = tb;
+ break;
+ }
+
+ /* Otherwise, generate a link task. */
+ else {
+
+ /* Only one thread should have to do this. */
+ if ( ind == task_maxunlock ) {
+ ta->link = scheduler_addtask( s , task_type_link , task_subtype_none , ta->flags , 0 , NULL , NULL , 0 );
+ ta->link->implicit = 1;
+ ta->unlock_tasks[ ind ] = ta->link;
+ }
+
+ /* Otherwise, reduce the count. */
+ else
+ atomic_dec( &ta->nr_unlock_tasks );
+
+ }
+
+ }
+
+ }
+
+
/**
* @brief Mapping function to append a ghost task to each cell.
*
@@ -84,7 +131,7 @@ void scheduler_map_mkghosts ( struct cell *c , void *data ) {
if (c->parent == NULL || c->parent->count >= space_subsize ) {
c->kick1 = scheduler_addtask( s , task_type_kick1 , task_subtype_none , 0 , 0 , c , NULL , 0 );
if ( c->parent != NULL )
- task_addunlock( c->kick1 , c->parent->kick1 );
+ scheduler_addunlock( s , c->kick1 , c->parent->kick1 );
}
/* Append a kick2 task if we are local and the active super cell. */
@@ -94,7 +141,7 @@ void scheduler_map_mkghosts ( struct cell *c , void *data ) {
/* If we are not the super cell ourselves, make our ghost depend
on our parent cell. */
if ( c->ghost != NULL && c->super != c ) {
- task_addunlock( c->parent->ghost , c->ghost );
+ scheduler_addunlock( s , c->parent->ghost , c->ghost );
c->ghost->implicit = 1;
}
@@ -128,7 +175,7 @@ void scheduler_map_mkghosts_nokick1 ( struct cell *c , void *data ) {
/* If we are not the super cell ourselves, make our ghost depend
on our parent cell. */
if ( c->super != c ) {
- task_addunlock( c->parent->ghost , c->ghost );
+ scheduler_addunlock( s , c->parent->ghost , c->ghost );
c->ghost->implicit = 1;
}
@@ -149,7 +196,7 @@ void scheduler_map_mkkick1 ( struct cell *c , void *data ) {
/* Append a kick1 task and make sure the parent depends on it. */
c->kick1 = scheduler_addtask( s , task_type_kick1 , task_subtype_none , 0 , 0 , c , NULL , 0 );
if ( c->parent != NULL )
- task_addunlock( c->kick1 , c->parent->kick1 );
+ scheduler_addunlock( s , c->kick1 , c->parent->kick1 );
/* Set a bogus super cell. */
for ( finger = c ; finger->parent != NULL ; finger = finger->parent );
@@ -199,11 +246,11 @@ void scheduler_splittasks ( struct scheduler *s ) {
}
/* Empty task? */
- if ( t->ci == NULL || ( t->type == task_type_pair && t->cj == NULL ) ) {
+ /* if ( t->ci == NULL || ( t->type == task_type_pair && t->cj == NULL ) ) {
t->type = task_type_none;
t->skip = 1;
continue;
- }
+ } */
/* Non-local kick task? */
if ( (t->type == task_type_kick1 || t->type == task_type_kick2 ) &&
@@ -454,7 +501,7 @@ void scheduler_splittasks ( struct scheduler *s ) {
else
ci->sorts->flags |= (1 << sid);
// lock_unlock_blind( &ci->lock );
- task_addunlock( ci->sorts , t );
+ scheduler_addunlock( s , ci->sorts , t );
/* Create the sort for cj. */
// lock_lock( &cj->lock );
@@ -463,7 +510,7 @@ void scheduler_splittasks ( struct scheduler *s ) {
else
cj->sorts->flags |= (1 << sid);
// lock_unlock_blind( &cj->lock );
- task_addunlock( cj->sorts , t );
+ scheduler_addunlock( s , cj->sorts , t );
}
@@ -511,6 +558,7 @@ struct task *scheduler_addtask ( struct scheduler *s , int type , int subtype ,
t->tic = 0;
t->toc = 0;
t->nr_unlock_tasks = 0;
+ t->link = NULL;
/* Init the lock. */
lock_init( &t->lock );
@@ -613,6 +661,9 @@ void scheduler_reset ( struct scheduler *s , int size ) {
}
+ /* Reset the task data. */
+ bzero( s->tasks , sizeof(struct task) * size );
+
/* Reset the counters. */
s->size = size;
s->nr_tasks = 0;
diff --git a/src/scheduler.h b/src/scheduler.h
index e7f4f3df98e058f667c9c793b1e2672c891e21a4..3032c2c97ee8a60d511f495712a45e92deef87cc 100644
--- a/src/scheduler.h
+++ b/src/scheduler.h
@@ -86,3 +86,4 @@ void scheduler_map_mkghosts_nokick1 ( struct cell *c , void *data );
void scheduler_map_mkkick1 ( struct cell *c , void *data );
struct task *scheduler_done ( struct scheduler *s , struct task *t );
struct task *scheduler_unlock ( struct scheduler *s , struct task *t );
+void scheduler_addunlock ( struct scheduler *s , struct task *ta , struct task *tb );
diff --git a/src/space.h b/src/space.h
index 1768ecefbc86e517236813d33248d8310bdf5fa4..e840cfb6ee2e628115a8219c2a026ab25ecf41a1 100644
--- a/src/space.h
+++ b/src/space.h
@@ -25,7 +25,7 @@
#define space_cellallocchunk 1000
#define space_splitratio 0.875f
#define space_splitsize_default 400
-#define space_maxsize_default 3000
+#define space_maxsize_default 2000
#define space_subsize_default 5000
#define space_stretch 1.10f
#define space_maxreldx 0.25f
diff --git a/src/task.c b/src/task.c
index 12161a8b60286a453041303a35fcf476f9f50915..a8d2f125734833fbd9e788ccfd8ff393e3b64613 100644
--- a/src/task.c
+++ b/src/task.c
@@ -50,7 +50,7 @@
const char *taskID_names[task_type_count] = {
"none" , "sort" , "self" , "pair" , "sub" , "ghost" ,
"kick1" , "kick2" , "send_xv" , "recv_xv" , "send_rho" ,
- "recv_rho" };
+ "recv_rho" , "link" };
/**
@@ -222,6 +222,8 @@ void task_rmunlock_blind ( struct task *ta , struct task *tb ) {
void task_addunlock ( struct task *ta , struct task *tb ) {
+ error( "Use sched_addunlock instead." );
+
/* Add the lock atomically. */
ta->unlock_tasks[ atomic_inc( &ta->nr_unlock_tasks ) ] = tb;
diff --git a/src/task.h b/src/task.h
index c616f22a49fe5abfa6ca8c236256f2a232c04d19..300ce57b3293135554ccb4eeccf3a5af5c7134db 100644
--- a/src/task.h
+++ b/src/task.h
@@ -20,7 +20,7 @@
/* Some constants. */
#define task_maxwait 3
-#define task_maxunlock 64
+#define task_maxunlock 31
/* The different task types. */
@@ -37,6 +37,7 @@ enum task_types {
task_type_recv_xv,
task_type_send_rho,
task_type_recv_rho,
+ task_type_link,
task_type_count
};
@@ -72,7 +73,8 @@ struct task {
ticks tic, toc;
int nr_unlock_tasks;
- struct task *unlock_tasks[ task_maxunlock ];
+ struct task *unlock_tasks[ task_maxunlock + 1 ];
+ struct task *link;
};