Created a super pointer for the hydro and gravity tasks. Still need to verify all the task links.

src/cell.c

@@ -1391,13 +1391,13 @@ void cell_activate_drift_part(struct cell *c, struct scheduler *s) {
 
   /* Set the do_sub_drifts all the way up and activate the super drift
      if this has not yet been done. */
-  if (c == c->super) {
+  if (c == c->super_hydro) {
     scheduler_activate(s, c->drift_part);
   } else {
     for (struct cell *parent = c->parent;
          parent != NULL && !parent->do_sub_drift; parent = parent->parent) {
       parent->do_sub_drift = 1;
-      if (parent == c->super) {
+      if (parent == c->super_hydro) {
         scheduler_activate(s, parent->drift_part);
         break;
       }
@@ -1418,14 +1418,14 @@ void cell_activate_drift_gpart(struct cell *c, struct scheduler *s) {
 
   /* Set the do_grav_sub_drifts all the way up and activate the super drift
      if this has not yet been done. */
-  if (c == c->super) {
+  if (c == c->super_gravity) {
     scheduler_activate(s, c->drift_gpart);
   } else {
     for (struct cell *parent = c->parent;
          parent != NULL && !parent->do_grav_sub_drift;
          parent = parent->parent) {
       parent->do_grav_sub_drift = 1;
-      if (parent == c->super) {
+      if (parent == c->super_gravity) {
         scheduler_activate(s, parent->drift_gpart);
         break;
       }
@@ -1437,14 +1437,14 @@ void cell_activate_drift_gpart(struct cell *c, struct scheduler *s) {
  * @brief Activate the sorts up a cell hierarchy.
  */
 void cell_activate_sorts_up(struct cell *c, struct scheduler *s) {
-  if (c == c->super) {
+  if (c == c->super_hydro) {
     scheduler_activate(s, c->sorts);
     if (c->nodeID == engine_rank) cell_activate_drift_part(c, s);
   } else {
     for (struct cell *parent = c->parent;
          parent != NULL && !parent->do_sub_sort; parent = parent->parent) {
       parent->do_sub_sort = 1;
-      if (parent == c->super) {
+      if (parent == c->super_hydro) {
         scheduler_activate(s, parent->sorts);
         if (parent->nodeID == engine_rank) cell_activate_drift_part(parent, s);
         break;
@@ -1992,6 +1992,9 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) {
           scheduler_activate(s, ci->recv_xv);
           if (ci_active) {
             scheduler_activate(s, ci->recv_rho);
+	    if(e->step == 32)
+	      message("recv_rho: cj->cellID=%ld ci->cellID=%ld", cj-e->s->cells_top, ci-e->s->cells_top);
+
 #ifdef EXTRA_HYDRO_LOOP
             scheduler_activate(s, ci->recv_gradient);
 #endif
@@ -2013,6 +2016,8 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) {
           /* If the local cell is also active, more stuff will be needed. */
           if (cj_active) {
             scheduler_activate_send(s, cj->send_rho, ci->nodeID);
+	    //if(e->step == 32)
+	    //  message("send_rho: cj->cellID=%ld ci->cellID=%ld", cj-e->s->cells_top, ci-e->s->cells_top);
 
 #ifdef EXTRA_HYDRO_LOOP
             scheduler_activate_send(s, cj->send_gradient, ci->nodeID);
@@ -2030,6 +2035,10 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) {
           scheduler_activate(s, cj->recv_xv);
           if (cj_active) {
             scheduler_activate(s, cj->recv_rho);
+	    if(e->step == 32)
+	      message("recv_rho: ci->cellID=%ld cj->cellID=%ld", ci-e->s->cells_top, cj-e->s->cells_top);
+
+
 #ifdef EXTRA_HYDRO_LOOP
             scheduler_activate(s, cj->recv_gradient);
 #endif
@@ -2052,6 +2061,8 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) {
           if (ci_active) {
 
             scheduler_activate_send(s, ci->send_rho, cj->nodeID);
+	    //if(e->step == 32)
+	    //  message("send_rho: ci->cellID=%ld cj->cellID=%ld", ci-e->s->cells_top, cj-e->s->cells_top);
 
 #ifdef EXTRA_HYDRO_LOOP
             scheduler_activate_send(s, ci->send_gradient, cj->nodeID);
@@ -2193,6 +2204,46 @@ void cell_set_super(struct cell *c, struct cell *super) {
       if (c->progeny[k] != NULL) cell_set_super(c->progeny[k], super);
 }
 
+/**
+ * @brief Set the super-cell pointers for all cells in a hierarchy.
+ *
+ * @param c The top-level #cell to play with.
+ * @param super Pointer to the deepest cell with tasks in this part of the tree.
+ */
+void cell_set_super_hydro(struct cell *c, struct cell *super_hydro) {
+
+  /* Are we in a cell with some kind of self/pair task ? */
+  if (super_hydro == NULL && c->density != NULL) super_hydro = c;
+
+  /* Set the super-cell */
+  c->super_hydro = super_hydro;
+
+  /* Recurse */
+  if (c->split)
+    for (int k = 0; k < 8; k++)
+      if (c->progeny[k] != NULL) cell_set_super_hydro(c->progeny[k], super_hydro);
+}
+
+/**
+ * @brief Set the super-cell pointers for all cells in a hierarchy.
+ *
+ * @param c The top-level #cell to play with.
+ * @param super Pointer to the deepest cell with tasks in this part of the tree.
+ */
+void cell_set_super_gravity(struct cell *c, struct cell *super_gravity) {
+
+  /* Are we in a cell with some kind of self/pair task ? */
+  if (super_gravity == NULL && c->grav != NULL) super_gravity = c;
+
+  /* Set the super-cell */
+  c->super_gravity = super_gravity;
+
+  /* Recurse */
+  if (c->split)
+    for (int k = 0; k < 8; k++)
+      if (c->progeny[k] != NULL) cell_set_super_gravity(c->progeny[k], super_gravity);
+}
+
 /**
  * @brief Mapper function to set the super pointer of the cells.
  *
@@ -2201,8 +2252,15 @@ void cell_set_super(struct cell *c, struct cell *super) {
  * @param extra_data Unused parameter.
  */
 void cell_set_super_mapper(void *map_data, int num_elements, void *extra_data) {
+
+  const struct engine *e = (const struct engine*) extra_data;
+
   for (int ind = 0; ind < num_elements; ind++) {
     struct cell *c = &((struct cell *)map_data)[ind];
+    if(e->policy & engine_policy_hydro)
+      cell_set_super_hydro(c, NULL);
+    if(e->policy & (engine_policy_self_gravity | engine_policy_external_gravity))
+      cell_set_super_gravity(c, NULL);
     cell_set_super(c, NULL);
   }
 }
@@ -2216,7 +2274,7 @@ void cell_set_super_mapper(void *map_data, int num_elements, void *extra_data) {
  * @param c The #cell to probe.
  */
 int cell_has_tasks(struct cell *c) {
-
+  
 #ifdef WITH_MPI
   if (c->timestep != NULL || c->recv_ti != NULL) return 1;
 #else

src/cell.h

@@ -178,9 +178,15 @@ struct cell {
   /*! Parent cell. */
   struct cell *parent;
 
-  /*! Super cell, i.e. the highest-level parent cell that has pair/self tasks */
+  /*! Super cell, i.e. the highest-level parent cell with *any* task */
   struct cell *super;
 
+  /*! Super cell, i.e. the highest-level parent cell that has a hydro pair/self tasks */
+  struct cell *super_hydro;
+
+  /*! Super cell, i.e. the highest-level parent cell that has a grav pair/self tasks */
+  struct cell *super_gravity;
+
   /*! Linked list of the tasks computing this cell's hydro density. */
   struct link *density;
 

src/scheduler.c

@@ -976,21 +976,21 @@ void scheduler_set_unlocks(struct scheduler *s) {
     t->unlock_tasks = &s->unlocks[offsets[k]];
   }
 
-#ifdef SWIFT_DEBUG_CHECKS
-  /* Verify that there are no duplicate unlocks. */
-  for (int k = 0; k < s->nr_tasks; k++) {
-    struct task *t = &s->tasks[k];
-    for (int i = 0; i < t->nr_unlock_tasks; i++) {
-      for (int j = i + 1; j < t->nr_unlock_tasks; j++) {
-        if (t->unlock_tasks[i] == t->unlock_tasks[j])
-          error("duplicate unlock! t->type=%s/%s unlocking type=%s/%s",
-                taskID_names[t->type], subtaskID_names[t->subtype],
-                taskID_names[t->unlock_tasks[i]->type],
-                subtaskID_names[t->unlock_tasks[i]->subtype]);
-      }
-    }
-  }
-#endif
+/* #ifdef SWIFT_DEBUG_CHECKS */
+/*   /\* Verify that there are no duplicate unlocks. *\/ */
+/*   for (int k = 0; k < s->nr_tasks; k++) { */
+/*     struct task *t = &s->tasks[k]; */
+/*     for (int i = 0; i < t->nr_unlock_tasks; i++) { */
+/*       for (int j = i + 1; j < t->nr_unlock_tasks; j++) { */
+/*         if (t->unlock_tasks[i] == t->unlock_tasks[j]) */
+/*           error("duplicate unlock! t->type=%s/%s unlocking type=%s/%s", */
+/*                 taskID_names[t->type], subtaskID_names[t->subtype], */
+/*                 taskID_names[t->unlock_tasks[i]->type], */
+/*                 subtaskID_names[t->unlock_tasks[i]->subtype]); */
+/*       } */
+/*     } */
+/*   } */
+/* #endif */
 
   /* Clean up. */
   free(counts);
@@ -1174,10 +1174,10 @@ void scheduler_reweight(struct scheduler *s, int verbose) {
         cost = 1 * wscale * t->ci->count * t->ci->count;
         break;
       case task_type_ghost:
-        if (t->ci == t->ci->super) cost = wscale * t->ci->count;
+        if (t->ci == t->ci->super_hydro) cost = wscale * t->ci->count;
         break;
       case task_type_extra_ghost:
-        if (t->ci == t->ci->super) cost = wscale * t->ci->count;
+        if (t->ci == t->ci->super_hydro) cost = wscale * t->ci->count;
         break;
       case task_type_drift_part:
         cost = wscale * t->ci->count;
@@ -1427,12 +1427,21 @@ void scheduler_enqueue(struct scheduler *s, struct task *t) {
     switch (t->type) {
       case task_type_self:
       case task_type_sub_self:
+	if(t->subtype == task_subtype_grav)
+	  qid = t->ci->super_gravity->owner;
+	else
+	  qid = t->ci->super_hydro->owner;
+	break;
       case task_type_sort:
       case task_type_ghost:
-      case task_type_kick1:
-      case task_type_kick2:
       case task_type_drift_part:
+        qid = t->ci->super_hydro->owner;
+        break;
       case task_type_drift_gpart:
+        qid = t->ci->super_gravity->owner;
+        break;
+      case task_type_kick1:
+      case task_type_kick2:
       case task_type_timestep:
         qid = t->ci->super->owner;
         break;

src/space.c

@@ -233,6 +233,8 @@ void space_rebuild_recycle_mapper(void *map_data, int num_elements,
     c->grav_long_range = NULL;
     c->grav_down = NULL;
     c->super = c;
+    c->super_hydro = c;
+    c->super_gravity = c;
     c->parts = NULL;
     c->xparts = NULL;
     c->gparts = NULL;
@@ -462,7 +464,9 @@ void space_regrid(struct space *s, int verbose) {
           c->count = 0;
           c->gcount = 0;
           c->scount = 0;
-          c->super = c;
+	  c->super = c;
+          c->super_hydro = c;
+          c->super_gravity = c;
           c->ti_old_part = ti_old;
           c->ti_old_gpart = ti_old;
           c->ti_old_multipole = ti_old;
@@ -2085,6 +2089,8 @@ void space_split_recursive(struct space *s, struct cell *c,
       cp->nodeID = c->nodeID;
       cp->parent = c;
       cp->super = NULL;
+      cp->super_hydro = NULL;
+      cp->super_gravity = NULL;
 #ifdef SWIFT_DEBUG_CHECKS
       cp->cellID = last_cell_id++;
 #endif