From cbdd02930f1ca3320c00f73c290dc1517db7d53b Mon Sep 17 00:00:00 2001
From: Matthieu Schaller <schaller@strw.leidenuniv.nl>
Date: Mon, 5 Nov 2018 13:10:03 +0100
Subject: [PATCH 01/12] Replace the absolute depth criterion for gravity tasks
 with a relative criterion based on the local depth of the tree.

---
 examples/parameter_example.yml | 6 +++---
 src/cell.h                     | 8 ++++----
 src/engine_maketasks.c         | 5 +++--
 src/runner.c                   | 2 +-
 src/space.c                    | 9 +++++----
 src/space.h                    | 4 ++--
 6 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/examples/parameter_example.yml b/examples/parameter_example.yml
index 63a61af3e..9ae5fef66 100644
--- a/examples/parameter_example.yml
+++ b/examples/parameter_example.yml
@@ -59,10 +59,10 @@ Scheduler:
   cell_sub_size_self_hydro:  32000     # (Optional) Maximal number of interactions per sub-self hydro task  (this is the default value).
   cell_sub_size_pair_grav:   256000000 # (Optional) Maximal number of interactions per sub-pair gravity task  (this is the default value).
   cell_sub_size_self_grav:   32000     # (Optional) Maximal number of interactions per sub-self gravity task  (this is the default value).
-  cell_sub_size_pair_stars:   256000000 # (Optional) Maximal number of interactions per sub-pair stars task  (this is the default value).
-  cell_sub_size_self_stars:   32000     # (Optional) Maximal number of interactions per sub-self stars task  (this is the default value).
+  cell_sub_size_pair_stars:  256000000 # (Optional) Maximal number of interactions per sub-pair stars task  (this is the default value).
+  cell_sub_size_self_stars:  32000     # (Optional) Maximal number of interactions per sub-self stars task  (this is the default value).
   cell_split_size:           400       # (Optional) Maximal number of particles per cell (this is the default value).
-  cell_subdepth_grav:        2         # (Optional) Maximal depth the gravity tasks can be pushed down (this is the default value).
+  cell_subdepth_diff_grav:   4         # (Optional) Maximal depth difference between leaves and a cell that gravity tasks can be pushed down to (this is the default value).
   max_top_level_cells:       12        # (Optional) Maximal number of top-level cells in any dimension. The number of top-level cells will be the cube of this (this is the default value).
   tasks_per_cell:            0         # (Optional) The average number of tasks per cell. If not large enough the simulation will fail (means guess...).
   mpi_message_limit:         4096      # (Optional) Maximum MPI task message size to send non-buffered, KB.
diff --git a/src/cell.h b/src/cell.h
index 89f7c954c..6203c0ec5 100644
--- a/src/cell.h
+++ b/src/cell.h
@@ -808,8 +808,8 @@ __attribute__((always_inline)) INLINE static int cell_can_split_self_stars_task(
 __attribute__((always_inline)) INLINE static int
 cell_can_split_pair_gravity_task(const struct cell *c) {
 
-  /* Is the cell split ? */
-  return c->split && c->depth < space_subdepth_grav;
+  /* Is the cell split and still far from the leaves ? */
+  return c->split && ((c->maxdepth - c->depth) > space_subdepth_diff_grav);
 }
 
 /**
@@ -821,8 +821,8 @@ cell_can_split_pair_gravity_task(const struct cell *c) {
 __attribute__((always_inline)) INLINE static int
 cell_can_split_self_gravity_task(const struct cell *c) {
 
-  /* Is the cell split ? */
-  return c->split && c->depth < space_subdepth_grav;
+  /* Is the cell split and still far from the leaves ? */
+  return c->split && ((c->maxdepth - c->depth) > space_subdepth_diff_grav);
 }
 
 /**
diff --git a/src/engine_maketasks.c b/src/engine_maketasks.c
index e4431a8cf..270b1d386 100644
--- a/src/engine_maketasks.c
+++ b/src/engine_maketasks.c
@@ -568,7 +568,8 @@ void engine_make_hierarchical_tasks_gravity(struct engine *e, struct cell *c) {
   }
 
   /* We are below the super-cell but not below the maximal splitting depth */
-  else if (c->grav.super != NULL && c->depth < space_subdepth_grav) {
+  else if ((c->grav.super != NULL) &&
+           ((c->maxdepth - c->depth) >= space_subdepth_diff_grav)) {
 
     /* Local tasks only... */
     if (c->nodeID == e->nodeID) {
@@ -588,7 +589,7 @@ void engine_make_hierarchical_tasks_gravity(struct engine *e, struct cell *c) {
   }
 
   /* Recurse but not below the maximal splitting depth */
-  if (c->split && c->depth <= space_subdepth_grav)
+  if (c->split && ((c->maxdepth - c->depth) >= space_subdepth_diff_grav))
     for (int k = 0; k < 8; k++)
       if (c->progeny[k] != NULL)
         engine_make_hierarchical_tasks_gravity(e, c->progeny[k]);
diff --git a/src/runner.c b/src/runner.c
index cf5492dad..0026d833b 100644
--- a/src/runner.c
+++ b/src/runner.c
@@ -1353,7 +1353,7 @@ static void runner_do_unskip_gravity(struct cell *c, struct engine *e) {
   if (!cell_is_active_gravity(c, e)) return;
 
   /* Recurse */
-  if (c->split && c->depth < space_subdepth_grav) {
+  if (c->split && ((c->maxdepth - c->depth) >= space_subdepth_diff_grav)) {
     for (int k = 0; k < 8; k++) {
       if (c->progeny[k] != NULL) {
         struct cell *cp = c->progeny[k];
diff --git a/src/space.c b/src/space.c
index bda2f4882..32b7718c2 100644
--- a/src/space.c
+++ b/src/space.c
@@ -68,7 +68,7 @@ int space_subsize_pair_grav = space_subsize_pair_grav_default;
 int space_subsize_self_grav = space_subsize_self_grav_default;
 int space_subsize_pair_stars = space_subsize_pair_stars_default;
 int space_subsize_self_stars = space_subsize_self_stars_default;
-int space_subdepth_grav = space_subdepth_grav_default;
+int space_subdepth_diff_grav = space_subdepth_diff_grav_default;
 int space_maxsize = space_maxsize_default;
 #ifdef SWIFT_DEBUG_CHECKS
 int last_cell_id;
@@ -3159,13 +3159,14 @@ void space_init(struct space *s, struct swift_params *params,
                                space_subsize_self_stars_default);
   space_splitsize = parser_get_opt_param_int(
       params, "Scheduler:cell_split_size", space_splitsize_default);
-  space_subdepth_grav = parser_get_opt_param_int(
-      params, "Scheduler:cell_subdepth_grav", space_subdepth_grav_default);
+  space_subdepth_diff_grav =
+      parser_get_opt_param_int(params, "Scheduler:cell_subdepth_diff_grav",
+                               space_subdepth_diff_grav_default);
 
   if (verbose) {
     message("max_size set to %d split_size set to %d", space_maxsize,
             space_splitsize);
-    message("subdepth_grav set to %d", space_subdepth_grav);
+    message("subdepth_grav set to %d", space_subdepth_diff_grav);
     message("sub_size_pair_hydro set to %d, sub_size_self_hydro set to %d",
             space_subsize_pair_hydro, space_subsize_self_hydro);
     message("sub_size_pair_grav set to %d, sub_size_self_grav set to %d",
diff --git a/src/space.h b/src/space.h
index d0ec372d6..e6d774200 100644
--- a/src/space.h
+++ b/src/space.h
@@ -50,7 +50,7 @@ struct cosmology;
 #define space_subsize_self_grav_default 32000
 #define space_subsize_pair_stars_default 256000000
 #define space_subsize_self_stars_default 32000
-#define space_subdepth_grav_default 2
+#define space_subdepth_diff_grav_default 4
 #define space_max_top_level_cells_default 12
 #define space_stretch 1.10f
 #define space_maxreldx 0.1f
@@ -67,7 +67,7 @@ extern int space_subsize_pair_grav;
 extern int space_subsize_self_grav;
 extern int space_subsize_pair_stars;
 extern int space_subsize_self_stars;
-extern int space_subdepth_grav;
+extern int space_subdepth_diff_grav;
 
 /**
  * @brief The space in which the cells and particles reside.
-- 
GitLab


From 4ba1728581135524458b24dc836c366b1413d44d Mon Sep 17 00:00:00 2001
From: Matthieu Schaller <schaller@strw.leidenuniv.nl>
Date: Mon, 5 Nov 2018 14:23:43 +0100
Subject: [PATCH 02/12] Added implicit tasks to walk up and down the tree for
 the end_force and drift_grav tasks.

---
 examples/analyse_tasks.py |  4 +-
 examples/plot_tasks.py    |  4 +-
 src/cell.c                | 21 +++++++---
 src/cell.h                |  6 +++
 src/engine_maketasks.c    | 82 +++++++++++++++++++++++++--------------
 src/engine_marktasks.c    |  3 +-
 src/task.c                | 46 ++++++++++++++++------
 src/task.h                |  2 +
 8 files changed, 116 insertions(+), 52 deletions(-)

diff --git a/examples/analyse_tasks.py b/examples/analyse_tasks.py
index 48a00a63e..343814d58 100755
--- a/examples/analyse_tasks.py
+++ b/examples/analyse_tasks.py
@@ -52,8 +52,8 @@ infile = args.input
 
 #  Tasks and subtypes. Indexed as in tasks.h.
 TASKTYPES = ["none", "sort", "self", "pair", "sub_self", "sub_pair",
-             "init_grav", "init_grav_out", "ghost_in", "ghost", "ghost_out", "extra_ghost", "drift_part", "drift_gpart",
-             "end_force", "kick1", "kick2", "timestep", "send", "recv", "grav_long_range", "grav_mm", "grav_down_in", 
+             "init_grav", "init_grav_out", "ghost_in", "ghost", "ghost_out", "extra_ghost", "drift_part", "drift_gpart", "drift_gpart_out",
+             "end_force", "end_force_in", "kick1", "kick2", "timestep", "send", "recv", "grav_long_range", "grav_mm", "grav_down_in", 
              "grav_down", "grav_mesh", "cooling", "star_formation", "sourceterms",
              "stars_ghost_in", "stars_ghost",   "stars_ghost_out",
              "count"]
diff --git a/examples/plot_tasks.py b/examples/plot_tasks.py
index 76128ac66..588061cd4 100755
--- a/examples/plot_tasks.py
+++ b/examples/plot_tasks.py
@@ -110,8 +110,8 @@ pl.rcParams.update(PLOT_PARAMS)
 
 #  Tasks and subtypes. Indexed as in tasks.h.
 TASKTYPES = ["none", "sort", "self", "pair", "sub_self", "sub_pair",
-             "init_grav", "init_grav_out", "ghost_in", "ghost", "ghost_out", "extra_ghost", "drift_part", "drift_gpart",
-             "end_force", "kick1", "kick2", "timestep", "send", "recv", "grav_long_range", "grav_mm", "grav_down_in", 
+             "init_grav", "init_grav_out", "ghost_in", "ghost", "ghost_out", "extra_ghost", "drift_part", "drift_gpart", "drift_gpart_out",
+             "end_force", "end_force_in", "kick1", "kick2", "timestep", "send", "recv", "grav_long_range", "grav_mm", "grav_down_in", 
              "grav_down", "grav_mesh", "cooling", "star_formation", "sourceterms",
              "stars_ghost_in", "stars_ghost",   "stars_ghost_out",
              "count"]
diff --git a/src/cell.c b/src/cell.c
index 37b7be42f..b869803f1 100644
--- a/src/cell.c
+++ b/src/cell.c
@@ -1626,6 +1626,8 @@ void cell_activate_drift_gpart(struct cell *c, struct scheduler *s) {
   /* Mark this cell for drifting. */
   c->grav.do_drift = 1;
 
+  if (c->grav.drift_out != NULL) scheduler_activate(s, c->grav.drift_out);
+
   /* Set the do_grav_sub_drifts all the way up and activate the super drift
      if this has not yet been done. */
   if (c == c->grav.super) {
@@ -1639,6 +1641,11 @@ void cell_activate_drift_gpart(struct cell *c, struct scheduler *s) {
          parent != NULL && !parent->grav.do_sub_drift;
          parent = parent->parent) {
       parent->grav.do_sub_drift = 1;
+
+      if (parent->grav.drift_out) {
+        scheduler_activate(s, parent->grav.drift_out);
+      }
+
       if (parent == c->grav.super) {
 #ifdef SWIFT_DEBUG_CHECKS
         if (parent->grav.drift == NULL)
@@ -2760,6 +2767,7 @@ int cell_unskip_hydro_tasks(struct cell *c, struct scheduler *s) {
     if (c->kick2 != NULL) scheduler_activate(s, c->kick2);
     if (c->timestep != NULL) scheduler_activate(s, c->timestep);
     if (c->end_force != NULL) scheduler_activate(s, c->end_force);
+    if (c->end_force_in != NULL) scheduler_activate(s, c->end_force_in);
     if (c->hydro.cooling != NULL) scheduler_activate(s, c->hydro.cooling);
     if (c->hydro.star_formation != NULL)
       scheduler_activate(s, c->hydro.star_formation);
@@ -2909,16 +2917,19 @@ int cell_unskip_gravity_tasks(struct cell *c, struct scheduler *s) {
     if (c->kick2 != NULL) scheduler_activate(s, c->kick2);
     if (c->timestep != NULL) scheduler_activate(s, c->timestep);
     if (c->end_force != NULL) scheduler_activate(s, c->end_force);
-    if ((e->policy & engine_policy_cooling) && c->hydro.cooling != NULL)
-      scheduler_activate(s, c->hydro.cooling);
-    if ((e->policy & engine_policy_star_formation) &&
-        c->hydro.star_formation != NULL)
-      scheduler_activate(s, c->hydro.star_formation);
+    if (c->end_force_in != NULL) scheduler_activate(s, c->end_force_in);
     if (c->grav.down != NULL) scheduler_activate(s, c->grav.down);
     if (c->grav.down_in != NULL) scheduler_activate(s, c->grav.down_in);
     if (c->grav.mesh != NULL) scheduler_activate(s, c->grav.mesh);
     if (c->grav.long_range != NULL) scheduler_activate(s, c->grav.long_range);
     if (c->logger != NULL) scheduler_activate(s, c->logger);
+
+    /* Subgrid tasks */
+    if ((e->policy & engine_policy_cooling) && c->hydro.cooling != NULL)
+      scheduler_activate(s, c->hydro.cooling);
+    if ((e->policy & engine_policy_star_formation) &&
+        c->hydro.star_formation != NULL)
+      scheduler_activate(s, c->hydro.star_formation);
   }
 
   return rebuild;
diff --git a/src/cell.h b/src/cell.h
index 6203c0ec5..575b764b9 100644
--- a/src/cell.h
+++ b/src/cell.h
@@ -404,6 +404,9 @@ struct cell {
     /*! The drift task for gparts */
     struct task *drift;
 
+    /*! Implicit task (going up- and down the tree) for the #gpart drifts */
+    struct task *drift_out;
+
     /*! Linked list of the tasks computing this cell's gravity forces. */
     struct link *grav;
 
@@ -538,6 +541,9 @@ struct cell {
   /*! The task to end the force calculation */
   struct task *end_force;
 
+  /*! Implicit task (going up- and down the tree) for the end force */
+  struct task *end_force_in;
+
   /*! The first kick task */
   struct task *kick1;
 
diff --git a/src/engine_maketasks.c b/src/engine_maketasks.c
index 270b1d386..f971aa003 100644
--- a/src/engine_maketasks.c
+++ b/src/engine_maketasks.c
@@ -169,7 +169,7 @@ void engine_addtasks_send_hydro(struct engine *e, struct cell *ci,
 
 #else
       /* The send_rho task should unlock the super_hydro-cell's kick task. */
-      scheduler_addunlock(s, t_rho, ci->super->end_force);
+      scheduler_addunlock(s, t_rho, ci->super->end_force_in);
 
       /* The send_rho task depends on the cell's ghost task. */
       scheduler_addunlock(s, ci->hydro.super->hydro.ghost_out, t_rho);
@@ -462,6 +462,11 @@ void engine_make_hierarchical_tasks_common(struct engine *e, struct cell *c) {
       c->end_force = scheduler_addtask(s, task_type_end_force,
                                        task_subtype_none, 0, 0, c, NULL);
 
+      /* Implicit task for the up pass */
+      c->end_force_in = scheduler_addtask(s, task_type_end_force_in,
+                                          task_subtype_none, 0, 1, c, NULL);
+
+      /* Subgrid tasks */
       if (is_with_cooling) {
 
         c->hydro.cooling = scheduler_addtask(s, task_type_cooling,
@@ -485,21 +490,35 @@ void engine_make_hierarchical_tasks_common(struct engine *e, struct cell *c) {
       } else {
         scheduler_addunlock(s, c->kick2, c->timestep);
       }
+
+      scheduler_addunlock(s, c->end_force_in, c->end_force);
       scheduler_addunlock(s, c->timestep, c->kick1);
 
 #if defined(WITH_LOGGER)
       scheduler_addunlock(s, c->kick1, c->logger);
 #endif
     }
+  }
 
-  } else { /* We are above the super-cell so need to go deeper */
+  /* We are below the super-cell but not below the maximal splitting depth */
+  else if ((c->super != NULL) &&
+           ((c->maxdepth - c->depth) >= space_subdepth_diff_grav)) {
 
-    /* Recurse. */
-    if (c->split)
-      for (int k = 0; k < 8; k++)
-        if (c->progeny[k] != NULL)
-          engine_make_hierarchical_tasks_common(e, c->progeny[k]);
+    /* Local tasks only... */
+    if (c->nodeID == e->nodeID) {
+
+      c->end_force_in = scheduler_addtask(s, task_type_end_force_in,
+                                          task_subtype_none, 0, 1, c, NULL);
+
+      scheduler_addunlock(s, c->end_force_in, c->parent->end_force_in);
+    }
   }
+
+  /* Recurse. */
+  if (c->split)
+    for (int k = 0; k < 8; k++)
+      if (c->progeny[k] != NULL)
+        engine_make_hierarchical_tasks_common(e, c->progeny[k]);
 }
 
 /**
@@ -544,6 +563,8 @@ void engine_make_hierarchical_tasks_gravity(struct engine *e, struct cell *c) {
                                          task_subtype_none, 0, 0, c, NULL);
 
         /* Implicit tasks for the up and down passes */
+        c->grav.drift_out = scheduler_addtask(s, task_type_drift_gpart_out,
+                                              task_subtype_none, 0, 1, c, NULL);
         c->grav.init_out = scheduler_addtask(s, task_type_init_grav_out,
                                              task_subtype_none, 0, 1, c, NULL);
         c->grav.down_in = scheduler_addtask(s, task_type_grav_down_in,
@@ -562,6 +583,7 @@ void engine_make_hierarchical_tasks_gravity(struct engine *e, struct cell *c) {
 
         /* Link in the implicit tasks */
         scheduler_addunlock(s, c->grav.init, c->grav.init_out);
+        scheduler_addunlock(s, c->grav.drift, c->grav.drift_out);
         scheduler_addunlock(s, c->grav.down_in, c->grav.down);
       }
     }
@@ -1129,7 +1151,7 @@ void engine_link_gravity_tasks(struct engine *e) {
 
       /* drift ---+-> gravity --> grav_down */
       /* init  --/    */
-      scheduler_addunlock(sched, ci->grav.super->grav.drift, t);
+      scheduler_addunlock(sched, ci->grav.super->grav.drift_out, t);
       scheduler_addunlock(sched, ci_parent->grav.init_out, t);
       scheduler_addunlock(sched, t, ci_parent->grav.down_in);
     }
@@ -1142,8 +1164,8 @@ void engine_link_gravity_tasks(struct engine *e) {
 #endif
 
       /* drift -----> gravity --> end_force */
-      scheduler_addunlock(sched, ci->grav.super->grav.drift, t);
-      scheduler_addunlock(sched, t, ci->super->end_force);
+      scheduler_addunlock(sched, ci->grav.super->grav.drift_out, t);
+      scheduler_addunlock(sched, t, ci->end_force_in);
     }
 
     /* Otherwise, pair interaction? */
@@ -1153,7 +1175,7 @@ void engine_link_gravity_tasks(struct engine *e) {
 
         /* drift ---+-> gravity --> grav_down */
         /* init  --/    */
-        scheduler_addunlock(sched, ci->grav.super->grav.drift, t);
+        scheduler_addunlock(sched, ci->grav.super->grav.drift_out, t);
         scheduler_addunlock(sched, ci_parent->grav.init_out, t);
         scheduler_addunlock(sched, t, ci_parent->grav.down_in);
       }
@@ -1162,7 +1184,7 @@ void engine_link_gravity_tasks(struct engine *e) {
         /* drift ---+-> gravity --> grav_down */
         /* init  --/    */
         if (ci->grav.super != cj->grav.super) /* Avoid double unlock */
-          scheduler_addunlock(sched, cj->grav.super->grav.drift, t);
+          scheduler_addunlock(sched, cj->grav.super->grav.drift_out, t);
 
         if (ci_parent != cj_parent) { /* Avoid double unlock */
           scheduler_addunlock(sched, cj_parent->grav.init_out, t);
@@ -1179,7 +1201,7 @@ void engine_link_gravity_tasks(struct engine *e) {
 #endif
       /* drift ---+-> gravity --> grav_down */
       /* init  --/    */
-      scheduler_addunlock(sched, ci->grav.super->grav.drift, t);
+      scheduler_addunlock(sched, ci->grav.super->grav.drift_out, t);
       scheduler_addunlock(sched, ci_parent->grav.init_out, t);
       scheduler_addunlock(sched, t, ci_parent->grav.down_in);
     }
@@ -1193,8 +1215,8 @@ void engine_link_gravity_tasks(struct engine *e) {
 #endif
 
       /* drift -----> gravity --> end_force */
-      scheduler_addunlock(sched, ci->grav.super->grav.drift, t);
-      scheduler_addunlock(sched, t, ci->super->end_force);
+      scheduler_addunlock(sched, ci->grav.super->grav.drift_out, t);
+      scheduler_addunlock(sched, t, ci->end_force_in);
     }
 
     /* Otherwise, sub-pair interaction? */
@@ -1204,7 +1226,7 @@ void engine_link_gravity_tasks(struct engine *e) {
 
         /* drift ---+-> gravity --> grav_down */
         /* init  --/    */
-        scheduler_addunlock(sched, ci->grav.super->grav.drift, t);
+        scheduler_addunlock(sched, ci->grav.super->grav.drift_out, t);
         scheduler_addunlock(sched, ci_parent->grav.init_out, t);
         scheduler_addunlock(sched, t, ci_parent->grav.down_in);
       }
@@ -1213,7 +1235,7 @@ void engine_link_gravity_tasks(struct engine *e) {
         /* drift ---+-> gravity --> grav_down */
         /* init  --/    */
         if (ci->grav.super != cj->grav.super) /* Avoid double unlock */
-          scheduler_addunlock(sched, cj->grav.super->grav.drift, t);
+          scheduler_addunlock(sched, cj->grav.super->grav.drift_out, t);
 
         if (ci_parent != cj_parent) { /* Avoid double unlock */
           scheduler_addunlock(sched, cj_parent->grav.init_out, t);
@@ -1349,7 +1371,7 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
       /* Now, build all the dependencies for the hydro */
       engine_make_hydro_loops_dependencies(sched, t, t2, t3, t->ci,
                                            with_cooling);
-      scheduler_addunlock(sched, t3, t->ci->super->end_force);
+      scheduler_addunlock(sched, t3, t->ci->super->end_force_in);
 #else
 
       /* Start by constructing the task for the second hydro loop */
@@ -1361,7 +1383,7 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
 
       /* Now, build all the dependencies for the hydro */
       engine_make_hydro_loops_dependencies(sched, t, t2, t->ci, with_cooling);
-      scheduler_addunlock(sched, t2, t->ci->super->end_force);
+      scheduler_addunlock(sched, t2, t->ci->end_force_in);
 #endif
     }
 
@@ -1396,14 +1418,14 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
       if (t->ci->nodeID == nodeID) {
         engine_make_hydro_loops_dependencies(sched, t, t2, t3, t->ci,
                                              with_cooling);
-        scheduler_addunlock(sched, t3, t->ci->super->end_force);
+        scheduler_addunlock(sched, t3, t->ci->super->end_force_in);
       }
       if (t->cj->nodeID == nodeID) {
         if (t->ci->hydro.super != t->cj->hydro.super)
           engine_make_hydro_loops_dependencies(sched, t, t2, t3, t->cj,
                                                with_cooling);
         if (t->ci->super != t->cj->super)
-          scheduler_addunlock(sched, t3, t->cj->super->end_force);
+          scheduler_addunlock(sched, t3, t->cj->super->end_force_in);
       }
 
 #else
@@ -1420,14 +1442,14 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
       /* that are local and are not descendant of the same super_hydro-cells */
       if (t->ci->nodeID == nodeID) {
         engine_make_hydro_loops_dependencies(sched, t, t2, t->ci, with_cooling);
-        scheduler_addunlock(sched, t2, t->ci->super->end_force);
+        scheduler_addunlock(sched, t2, t->ci->super->end_force_in);
       }
       if (t->cj->nodeID == nodeID) {
         if (t->ci->hydro.super != t->cj->hydro.super)
           engine_make_hydro_loops_dependencies(sched, t, t2, t->cj,
                                                with_cooling);
         if (t->ci->super != t->cj->super)
-          scheduler_addunlock(sched, t2, t->cj->super->end_force);
+          scheduler_addunlock(sched, t2, t->cj->super->end_force_in);
       }
 
 #endif
@@ -1461,7 +1483,7 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
       if (t->ci->nodeID == nodeID) {
         engine_make_hydro_loops_dependencies(sched, t, t2, t3, t->ci,
                                              with_cooling);
-        scheduler_addunlock(sched, t3, t->ci->super->end_force);
+        scheduler_addunlock(sched, t3, t->ci->super->end_force_in);
       }
 
 #else
@@ -1477,7 +1499,7 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
       /* that are local and are not descendant of the same super_hydro-cells */
       if (t->ci->nodeID == nodeID) {
         engine_make_hydro_loops_dependencies(sched, t, t2, t->ci, with_cooling);
-        scheduler_addunlock(sched, t2, t->ci->super->end_force);
+        scheduler_addunlock(sched, t2, t->ci->super->end_force_in);
       }
 #endif
     }
@@ -1517,14 +1539,14 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
       if (t->ci->nodeID == nodeID) {
         engine_make_hydro_loops_dependencies(sched, t, t2, t3, t->ci,
                                              with_cooling);
-        scheduler_addunlock(sched, t3, t->ci->super->end_force);
+        scheduler_addunlock(sched, t3, t->ci->super->end_force_in);
       }
       if (t->cj->nodeID == nodeID) {
         if (t->ci->hydro.super != t->cj->hydro.super)
           engine_make_hydro_loops_dependencies(sched, t, t2, t3, t->cj,
                                                with_cooling);
         if (t->ci->super != t->cj->super)
-          scheduler_addunlock(sched, t3, t->cj->super->end_force);
+          scheduler_addunlock(sched, t3, t->cj->super->end_force_in);
       }
 
 #else
@@ -1541,14 +1563,14 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
       /* that are local and are not descendant of the same super_hydro-cells */
       if (t->ci->nodeID == nodeID) {
         engine_make_hydro_loops_dependencies(sched, t, t2, t->ci, with_cooling);
-        scheduler_addunlock(sched, t2, t->ci->super->end_force);
+        scheduler_addunlock(sched, t2, t->ci->super->end_force_in);
       }
       if (t->cj->nodeID == nodeID) {
         if (t->ci->hydro.super != t->cj->hydro.super)
           engine_make_hydro_loops_dependencies(sched, t, t2, t->cj,
                                                with_cooling);
         if (t->ci->super != t->cj->super)
-          scheduler_addunlock(sched, t2, t->cj->super->end_force);
+          scheduler_addunlock(sched, t2, t->cj->super->end_force_in);
       }
 #endif
     }
@@ -1584,7 +1606,7 @@ void engine_link_stars_tasks_mapper(void *map_data, int num_elements,
       engine_make_stars_loops_dependencies(sched, t, t->ci);
       if (t->ci == t->ci->super)
         scheduler_addunlock(sched, t->ci->super->stars.ghost_out,
-                            t->ci->super->end_force);
+                            t->ci->super->end_force_in);
     }
 
     /* Otherwise, pair interaction? */
diff --git a/src/engine_marktasks.c b/src/engine_marktasks.c
index c891fb2ca..d3208de3e 100644
--- a/src/engine_marktasks.c
+++ b/src/engine_marktasks.c
@@ -443,7 +443,8 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
     }
 
     /* End force ? */
-    else if (t_type == task_type_end_force) {
+    else if (t_type == task_type_end_force ||
+             t_type == task_type_end_force_in) {
 
       if (cell_is_active_hydro(t->ci, e) || cell_is_active_gravity(t->ci, e))
         scheduler_activate(s, t);
diff --git a/src/task.c b/src/task.c
index 996c5113b..6580bf4a6 100644
--- a/src/task.c
+++ b/src/task.c
@@ -47,18 +47,40 @@
 #include "lock.h"
 
 /* Task type names. */
-const char *taskID_names[task_type_count] = {
-    "none",        "sort",           "self",
-    "pair",        "sub_self",       "sub_pair",
-    "init_grav",   "init_grav_out",  "ghost_in",
-    "ghost",       "ghost_out",      "extra_ghost",
-    "drift_part",  "drift_gpart",    "end_force",
-    "kick1",       "kick2",          "timestep",
-    "send",        "recv",           "grav_long_range",
-    "grav_mm",     "grav_down_in",   "grav_down",
-    "grav_mesh",   "cooling",        "star_formation",
-    "sourceterms", "logger",         "stars_ghost_in",
-    "stars_ghost", "stars_ghost_out"};
+const char *taskID_names[task_type_count] = {"none",
+                                             "sort",
+                                             "self",
+                                             "pair",
+                                             "sub_self",
+                                             "sub_pair",
+                                             "init_grav",
+                                             "init_grav_out",
+                                             "ghost_in",
+                                             "ghost",
+                                             "ghost_out",
+                                             "extra_ghost",
+                                             "drift_part",
+                                             "drift_gpart",
+                                             "drift_gpart_out",
+                                             "end_force",
+                                             "end_force_in",
+                                             "kick1",
+                                             "kick2",
+                                             "timestep",
+                                             "send",
+                                             "recv",
+                                             "grav_long_range",
+                                             "grav_mm",
+                                             "grav_down_in",
+                                             "grav_down",
+                                             "grav_mesh",
+                                             "cooling",
+                                             "star_formation",
+                                             "sourceterms",
+                                             "logger",
+                                             "stars_ghost_in",
+                                             "stars_ghost",
+                                             "stars_ghost_out"};
 
 /* Sub-task type names. */
 const char *subtaskID_names[task_subtype_count] = {
diff --git a/src/task.h b/src/task.h
index 9b3225fcf..90016472c 100644
--- a/src/task.h
+++ b/src/task.h
@@ -53,7 +53,9 @@ enum task_types {
   task_type_extra_ghost,
   task_type_drift_part,
   task_type_drift_gpart,
+  task_type_drift_gpart_out, /* Implicit */
   task_type_end_force,
+  task_type_end_force_in, /* Implicit */
   task_type_kick1,
   task_type_kick2,
   task_type_timestep,
-- 
GitLab


From cc8f33ae6f010d22910bac4c4565872082c5aa0b Mon Sep 17 00:00:00 2001
From: Matthieu Schaller <schaller@strw.leidenuniv.nl>
Date: Mon, 5 Nov 2018 18:48:17 +0100
Subject: [PATCH 03/12] Correctly set the implicit up-task for the end_force at
 all levels.

---
 src/engine_maketasks.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/engine_maketasks.c b/src/engine_maketasks.c
index f971aa003..e930ca949 100644
--- a/src/engine_maketasks.c
+++ b/src/engine_maketasks.c
@@ -501,8 +501,7 @@ void engine_make_hierarchical_tasks_common(struct engine *e, struct cell *c) {
   }
 
   /* We are below the super-cell but not below the maximal splitting depth */
-  else if ((c->super != NULL) &&
-           ((c->maxdepth - c->depth) >= space_subdepth_diff_grav)) {
+  else if (c->super != NULL) {
 
     /* Local tasks only... */
     if (c->nodeID == e->nodeID) {
-- 
GitLab


From 6dbca4000e3428681a74fa24d32d4f49c6545edc Mon Sep 17 00:00:00 2001
From: Matthieu Schaller <matthieu.schaller@durham.ac.uk>
Date: Tue, 6 Nov 2018 12:44:07 +0000
Subject: [PATCH 04/12] Communicate the maximal depth of the tree when
 exchanging cells at rebuild time.

---
 src/cell.c | 3 +++
 src/cell.h | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/src/cell.c b/src/cell.c
index b869803f1..79ccf0211 100644
--- a/src/cell.c
+++ b/src/cell.c
@@ -190,6 +190,7 @@ int cell_pack(struct cell *restrict c, struct pcell *restrict pc,
   pc->hydro.count = c->hydro.count;
   pc->grav.count = c->grav.count;
   pc->stars.count = c->stars.count;
+  pc->maxdepth = c->maxdepth;
 
   /* Copy the Multipole related information */
   if (with_gravity) {
@@ -292,6 +293,8 @@ int cell_unpack(struct pcell *restrict pc, struct cell *restrict c,
   c->hydro.count = pc->hydro.count;
   c->grav.count = pc->grav.count;
   c->stars.count = pc->stars.count;
+  c->maxdepth = pc->maxdepth;
+
 #ifdef SWIFT_DEBUG_CHECKS
   c->cellID = pc->cellID;
 #endif
diff --git a/src/cell.h b/src/cell.h
index 575b764b9..a5d0f7c61 100644
--- a/src/cell.h
+++ b/src/cell.h
@@ -149,6 +149,9 @@ struct pcell {
 
   } stars;
 
+  /*! Maximal depth in that part of the tree */
+  int maxdepth;
+
   /*! Relative indices of the cell's progeny. */
   int progeny[8];
 
-- 
GitLab


From 1466a867e468cfa35f3481321e654b71914e7e35 Mon Sep 17 00:00:00 2001
From: Matthieu Schaller <matthieu.schaller@durham.ac.uk>
Date: Tue, 6 Nov 2018 12:44:36 +0000
Subject: [PATCH 05/12] When regriding reset all the communication task
 pointers to NULL.

---
 src/space.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/space.c b/src/space.c
index 32b7718c2..19cf1e408 100644
--- a/src/space.c
+++ b/src/space.c
@@ -505,6 +505,14 @@ void space_regrid(struct space *s, int verbose) {
           c->grav.ti_old_multipole = ti_current;
 #ifdef WITH_MPI
           c->mpi.tag = -1;
+	  c->mpi.hydro.recv_xv = NULL;
+	  c->mpi.hydro.recv_rho = NULL;
+	  c->mpi.hydro.recv_gradient = NULL;
+	  c->mpi.hydro.send_xv = NULL;
+	  c->mpi.hydro.send_rho = NULL;
+	  c->mpi.hydro.send_gradient = NULL;
+	  c->mpi.grav.recv = NULL;
+	  c->mpi.grav.send = NULL;
 #endif  // WITH_MPI
           if (s->gravity) c->grav.multipole = &s->multipoles_top[cid];
 #ifdef SWIFT_DEBUG_CHECKS
-- 
GitLab


From aae577d056f554889ea9c9961b3a10d4837cfae6 Mon Sep 17 00:00:00 2001
From: Matthieu Schaller <matthieu.schaller@durham.ac.uk>
Date: Tue, 6 Nov 2018 15:04:08 +0000
Subject: [PATCH 06/12] Improved error message for MPI tasks that have an
 incorrect task.

---
 src/task.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/task.c b/src/task.c
index 6580bf4a6..13ce88452 100644
--- a/src/task.c
+++ b/src/task.c
@@ -412,8 +412,8 @@ int task_lock(struct task *t) {
         char buff[MPI_MAX_ERROR_STRING];
         int len;
         MPI_Error_string(err, buff, &len);
-        error("Failed to test request on send/recv task (tag=%lld, %s).",
-              t->flags, buff);
+        error("Failed to test request on send/recv task (type=%s/%s tag=%lld, %s).",
+              taskID_names[t->type], subtaskID_names[t->subtype], t->flags, buff);
       }
       return res;
 #else
-- 
GitLab


From 9d016745ac64880015ed58068066ecb4a35403c1 Mon Sep 17 00:00:00 2001
From: Matthieu Schaller <matthieu.schaller@durham.ac.uk>
Date: Tue, 6 Nov 2018 15:06:47 +0000
Subject: [PATCH 07/12] This version works for EAGLE-50.

---
 src/engine_maketasks.c | 62 ++++++++++++++++++++++--------------------
 src/engine_marktasks.c |  2 ++
 2 files changed, 34 insertions(+), 30 deletions(-)

diff --git a/src/engine_maketasks.c b/src/engine_maketasks.c
index e930ca949..5217bb694 100644
--- a/src/engine_maketasks.c
+++ b/src/engine_maketasks.c
@@ -169,7 +169,7 @@ void engine_addtasks_send_hydro(struct engine *e, struct cell *ci,
 
 #else
       /* The send_rho task should unlock the super_hydro-cell's kick task. */
-      scheduler_addunlock(s, t_rho, ci->super->end_force_in);
+      scheduler_addunlock(s, t_rho, ci->super->end_force);
 
       /* The send_rho task depends on the cell's ghost task. */
       scheduler_addunlock(s, ci->hydro.super->hydro.ghost_out, t_rho);
@@ -490,8 +490,8 @@ void engine_make_hierarchical_tasks_common(struct engine *e, struct cell *c) {
       } else {
         scheduler_addunlock(s, c->kick2, c->timestep);
       }
-
-      scheduler_addunlock(s, c->end_force_in, c->end_force);
+      
+      //scheduler_addunlock(s, c->end_force_in, c->end_force);
       scheduler_addunlock(s, c->timestep, c->kick1);
 
 #if defined(WITH_LOGGER)
@@ -589,14 +589,17 @@ void engine_make_hierarchical_tasks_gravity(struct engine *e, struct cell *c) {
   }
 
   /* We are below the super-cell but not below the maximal splitting depth */
-  else if ((c->grav.super != NULL) &&
-           ((c->maxdepth - c->depth) >= space_subdepth_diff_grav)) {
+  else if ((c->grav.super != NULL)) { // &&
+           //((c->maxdepth - c->depth) >= space_subdepth_diff_grav)) {
 
     /* Local tasks only... */
     if (c->nodeID == e->nodeID) {
 
       if (is_self_gravity) {
 
+        c->grav.drift_out = scheduler_addtask(s, task_type_drift_gpart_out,
+					      task_subtype_none, 0, 1, c, NULL);
+
         c->grav.init_out = scheduler_addtask(s, task_type_init_grav_out,
                                              task_subtype_none, 0, 1, c, NULL);
 
@@ -604,6 +607,7 @@ void engine_make_hierarchical_tasks_gravity(struct engine *e, struct cell *c) {
                                             task_subtype_none, 0, 1, c, NULL);
 
         scheduler_addunlock(s, c->parent->grav.init_out, c->grav.init_out);
+        scheduler_addunlock(s, c->parent->grav.drift_out, c->grav.drift_out);
         scheduler_addunlock(s, c->grav.down_in, c->parent->grav.down_in);
       }
     }
@@ -1150,7 +1154,7 @@ void engine_link_gravity_tasks(struct engine *e) {
 
       /* drift ---+-> gravity --> grav_down */
       /* init  --/    */
-      scheduler_addunlock(sched, ci->grav.super->grav.drift_out, t);
+      scheduler_addunlock(sched, ci->grav.drift_out, t);
       scheduler_addunlock(sched, ci_parent->grav.init_out, t);
       scheduler_addunlock(sched, t, ci_parent->grav.down_in);
     }
@@ -1163,8 +1167,8 @@ void engine_link_gravity_tasks(struct engine *e) {
 #endif
 
       /* drift -----> gravity --> end_force */
-      scheduler_addunlock(sched, ci->grav.super->grav.drift_out, t);
-      scheduler_addunlock(sched, t, ci->end_force_in);
+      scheduler_addunlock(sched, ci->grav.super->grav.drift, t);
+      scheduler_addunlock(sched, t, ci->super->end_force);
     }
 
     /* Otherwise, pair interaction? */
@@ -1174,7 +1178,7 @@ void engine_link_gravity_tasks(struct engine *e) {
 
         /* drift ---+-> gravity --> grav_down */
         /* init  --/    */
-        scheduler_addunlock(sched, ci->grav.super->grav.drift_out, t);
+        scheduler_addunlock(sched, ci->grav.drift_out, t);
         scheduler_addunlock(sched, ci_parent->grav.init_out, t);
         scheduler_addunlock(sched, t, ci_parent->grav.down_in);
       }
@@ -1182,8 +1186,7 @@ void engine_link_gravity_tasks(struct engine *e) {
 
         /* drift ---+-> gravity --> grav_down */
         /* init  --/    */
-        if (ci->grav.super != cj->grav.super) /* Avoid double unlock */
-          scheduler_addunlock(sched, cj->grav.super->grav.drift_out, t);
+	scheduler_addunlock(sched, cj->grav.drift_out, t);
 
         if (ci_parent != cj_parent) { /* Avoid double unlock */
           scheduler_addunlock(sched, cj_parent->grav.init_out, t);
@@ -1200,7 +1203,7 @@ void engine_link_gravity_tasks(struct engine *e) {
 #endif
       /* drift ---+-> gravity --> grav_down */
       /* init  --/    */
-      scheduler_addunlock(sched, ci->grav.super->grav.drift_out, t);
+      scheduler_addunlock(sched, ci->grav.drift_out, t);
       scheduler_addunlock(sched, ci_parent->grav.init_out, t);
       scheduler_addunlock(sched, t, ci_parent->grav.down_in);
     }
@@ -1214,8 +1217,8 @@ void engine_link_gravity_tasks(struct engine *e) {
 #endif
 
       /* drift -----> gravity --> end_force */
-      scheduler_addunlock(sched, ci->grav.super->grav.drift_out, t);
-      scheduler_addunlock(sched, t, ci->end_force_in);
+      scheduler_addunlock(sched, ci->grav.super->grav.drift, t);
+      scheduler_addunlock(sched, t, ci->super->end_force);
     }
 
     /* Otherwise, sub-pair interaction? */
@@ -1225,7 +1228,7 @@ void engine_link_gravity_tasks(struct engine *e) {
 
         /* drift ---+-> gravity --> grav_down */
         /* init  --/    */
-        scheduler_addunlock(sched, ci->grav.super->grav.drift_out, t);
+        scheduler_addunlock(sched, ci->grav.drift_out, t);
         scheduler_addunlock(sched, ci_parent->grav.init_out, t);
         scheduler_addunlock(sched, t, ci_parent->grav.down_in);
       }
@@ -1233,8 +1236,7 @@ void engine_link_gravity_tasks(struct engine *e) {
 
         /* drift ---+-> gravity --> grav_down */
         /* init  --/    */
-        if (ci->grav.super != cj->grav.super) /* Avoid double unlock */
-          scheduler_addunlock(sched, cj->grav.super->grav.drift_out, t);
+	scheduler_addunlock(sched, cj->grav.drift_out, t);
 
         if (ci_parent != cj_parent) { /* Avoid double unlock */
           scheduler_addunlock(sched, cj_parent->grav.init_out, t);
@@ -1370,7 +1372,7 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
       /* Now, build all the dependencies for the hydro */
       engine_make_hydro_loops_dependencies(sched, t, t2, t3, t->ci,
                                            with_cooling);
-      scheduler_addunlock(sched, t3, t->ci->super->end_force_in);
+      scheduler_addunlock(sched, t3, t->ci->super->end_force);
 #else
 
       /* Start by constructing the task for the second hydro loop */
@@ -1382,7 +1384,7 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
 
       /* Now, build all the dependencies for the hydro */
       engine_make_hydro_loops_dependencies(sched, t, t2, t->ci, with_cooling);
-      scheduler_addunlock(sched, t2, t->ci->end_force_in);
+      scheduler_addunlock(sched, t2, t->ci->super->end_force);
 #endif
     }
 
@@ -1417,14 +1419,14 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
       if (t->ci->nodeID == nodeID) {
         engine_make_hydro_loops_dependencies(sched, t, t2, t3, t->ci,
                                              with_cooling);
-        scheduler_addunlock(sched, t3, t->ci->super->end_force_in);
+        scheduler_addunlock(sched, t3, t->ci->super->end_force);
       }
       if (t->cj->nodeID == nodeID) {
         if (t->ci->hydro.super != t->cj->hydro.super)
           engine_make_hydro_loops_dependencies(sched, t, t2, t3, t->cj,
                                                with_cooling);
         if (t->ci->super != t->cj->super)
-          scheduler_addunlock(sched, t3, t->cj->super->end_force_in);
+          scheduler_addunlock(sched, t3, t->cj->super->end_force);
       }
 
 #else
@@ -1441,14 +1443,14 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
       /* that are local and are not descendant of the same super_hydro-cells */
       if (t->ci->nodeID == nodeID) {
         engine_make_hydro_loops_dependencies(sched, t, t2, t->ci, with_cooling);
-        scheduler_addunlock(sched, t2, t->ci->super->end_force_in);
+        scheduler_addunlock(sched, t2, t->ci->super->end_force);
       }
       if (t->cj->nodeID == nodeID) {
         if (t->ci->hydro.super != t->cj->hydro.super)
           engine_make_hydro_loops_dependencies(sched, t, t2, t->cj,
                                                with_cooling);
         if (t->ci->super != t->cj->super)
-          scheduler_addunlock(sched, t2, t->cj->super->end_force_in);
+          scheduler_addunlock(sched, t2, t->cj->super->end_force);
       }
 
 #endif
@@ -1482,7 +1484,7 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
       if (t->ci->nodeID == nodeID) {
         engine_make_hydro_loops_dependencies(sched, t, t2, t3, t->ci,
                                              with_cooling);
-        scheduler_addunlock(sched, t3, t->ci->super->end_force_in);
+        scheduler_addunlock(sched, t3, t->ci->super->end_force);
       }
 
 #else
@@ -1498,7 +1500,7 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
       /* that are local and are not descendant of the same super_hydro-cells */
       if (t->ci->nodeID == nodeID) {
         engine_make_hydro_loops_dependencies(sched, t, t2, t->ci, with_cooling);
-        scheduler_addunlock(sched, t2, t->ci->super->end_force_in);
+        scheduler_addunlock(sched, t2, t->ci->super->end_force);
       }
 #endif
     }
@@ -1538,14 +1540,14 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
       if (t->ci->nodeID == nodeID) {
         engine_make_hydro_loops_dependencies(sched, t, t2, t3, t->ci,
                                              with_cooling);
-        scheduler_addunlock(sched, t3, t->ci->super->end_force_in);
+        scheduler_addunlock(sched, t3, t->ci->super->end_force);
       }
       if (t->cj->nodeID == nodeID) {
         if (t->ci->hydro.super != t->cj->hydro.super)
           engine_make_hydro_loops_dependencies(sched, t, t2, t3, t->cj,
                                                with_cooling);
         if (t->ci->super != t->cj->super)
-          scheduler_addunlock(sched, t3, t->cj->super->end_force_in);
+          scheduler_addunlock(sched, t3, t->cj->super->end_force);
       }
 
 #else
@@ -1562,14 +1564,14 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
       /* that are local and are not descendant of the same super_hydro-cells */
       if (t->ci->nodeID == nodeID) {
         engine_make_hydro_loops_dependencies(sched, t, t2, t->ci, with_cooling);
-        scheduler_addunlock(sched, t2, t->ci->super->end_force_in);
+        scheduler_addunlock(sched, t2, t->ci->super->end_force);
       }
       if (t->cj->nodeID == nodeID) {
         if (t->ci->hydro.super != t->cj->hydro.super)
           engine_make_hydro_loops_dependencies(sched, t, t2, t->cj,
                                                with_cooling);
         if (t->ci->super != t->cj->super)
-          scheduler_addunlock(sched, t2, t->cj->super->end_force_in);
+          scheduler_addunlock(sched, t2, t->cj->super->end_force);
       }
 #endif
     }
@@ -1605,7 +1607,7 @@ void engine_link_stars_tasks_mapper(void *map_data, int num_elements,
       engine_make_stars_loops_dependencies(sched, t, t->ci);
       if (t->ci == t->ci->super)
         scheduler_addunlock(sched, t->ci->super->stars.ghost_out,
-                            t->ci->super->end_force_in);
+                            t->ci->super->end_force);
     }
 
     /* Otherwise, pair interaction? */
diff --git a/src/engine_marktasks.c b/src/engine_marktasks.c
index d3208de3e..2065f6364 100644
--- a/src/engine_marktasks.c
+++ b/src/engine_marktasks.c
@@ -261,6 +261,7 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
         }
       }
 
+      /* Gravity */
       if ((t_subtype == task_subtype_grav) &&
           ((ci_active_gravity && ci_nodeID == nodeID) ||
            (cj_active_gravity && cj_nodeID == nodeID))) {
@@ -475,6 +476,7 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
              t_type == task_type_grav_long_range ||
              t_type == task_type_init_grav ||
              t_type == task_type_init_grav_out ||
+             t_type == task_type_drift_gpart_out ||
              t_type == task_type_grav_down_in) {
       if (cell_is_active_gravity(t->ci, e)) scheduler_activate(s, t);
     }
-- 
GitLab


From 3d1124c708042b919125de9c4e2c8fb73829bfd9 Mon Sep 17 00:00:00 2001
From: Matthieu Schaller <matthieu.schaller@durham.ac.uk>
Date: Tue, 6 Nov 2018 15:46:11 +0000
Subject: [PATCH 08/12] Attach the gravity drift_out one level higher in the
 tree.

---
 src/engine_maketasks.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/engine_maketasks.c b/src/engine_maketasks.c
index 5217bb694..21b4cec0e 100644
--- a/src/engine_maketasks.c
+++ b/src/engine_maketasks.c
@@ -589,8 +589,8 @@ void engine_make_hierarchical_tasks_gravity(struct engine *e, struct cell *c) {
   }
 
   /* We are below the super-cell but not below the maximal splitting depth */
-  else if ((c->grav.super != NULL)) { // &&
-           //((c->maxdepth - c->depth) >= space_subdepth_diff_grav)) {
+  else if ((c->grav.super != NULL)  &&
+           ((c->maxdepth - c->depth) >= space_subdepth_diff_grav)) {
 
     /* Local tasks only... */
     if (c->nodeID == e->nodeID) {
@@ -1154,7 +1154,7 @@ void engine_link_gravity_tasks(struct engine *e) {
 
       /* drift ---+-> gravity --> grav_down */
       /* init  --/    */
-      scheduler_addunlock(sched, ci->grav.drift_out, t);
+      scheduler_addunlock(sched, ci_parent->grav.drift_out, t);
       scheduler_addunlock(sched, ci_parent->grav.init_out, t);
       scheduler_addunlock(sched, t, ci_parent->grav.down_in);
     }
@@ -1178,7 +1178,7 @@ void engine_link_gravity_tasks(struct engine *e) {
 
         /* drift ---+-> gravity --> grav_down */
         /* init  --/    */
-        scheduler_addunlock(sched, ci->grav.drift_out, t);
+        scheduler_addunlock(sched, ci_parent->grav.drift_out, t);
         scheduler_addunlock(sched, ci_parent->grav.init_out, t);
         scheduler_addunlock(sched, t, ci_parent->grav.down_in);
       }
@@ -1186,9 +1186,9 @@ void engine_link_gravity_tasks(struct engine *e) {
 
         /* drift ---+-> gravity --> grav_down */
         /* init  --/    */
-	scheduler_addunlock(sched, cj->grav.drift_out, t);
 
         if (ci_parent != cj_parent) { /* Avoid double unlock */
+	  scheduler_addunlock(sched, cj_parent->grav.drift_out, t);
           scheduler_addunlock(sched, cj_parent->grav.init_out, t);
           scheduler_addunlock(sched, t, cj_parent->grav.down_in);
         }
@@ -1203,7 +1203,7 @@ void engine_link_gravity_tasks(struct engine *e) {
 #endif
       /* drift ---+-> gravity --> grav_down */
       /* init  --/    */
-      scheduler_addunlock(sched, ci->grav.drift_out, t);
+      scheduler_addunlock(sched, ci_parent->grav.drift_out, t);
       scheduler_addunlock(sched, ci_parent->grav.init_out, t);
       scheduler_addunlock(sched, t, ci_parent->grav.down_in);
     }
@@ -1228,7 +1228,7 @@ void engine_link_gravity_tasks(struct engine *e) {
 
         /* drift ---+-> gravity --> grav_down */
         /* init  --/    */
-        scheduler_addunlock(sched, ci->grav.drift_out, t);
+        scheduler_addunlock(sched, ci_parent->grav.drift_out, t);
         scheduler_addunlock(sched, ci_parent->grav.init_out, t);
         scheduler_addunlock(sched, t, ci_parent->grav.down_in);
       }
@@ -1236,9 +1236,9 @@ void engine_link_gravity_tasks(struct engine *e) {
 
         /* drift ---+-> gravity --> grav_down */
         /* init  --/    */
-	scheduler_addunlock(sched, cj->grav.drift_out, t);
 
         if (ci_parent != cj_parent) { /* Avoid double unlock */
+	  scheduler_addunlock(sched, cj_parent->grav.drift_out, t);
           scheduler_addunlock(sched, cj_parent->grav.init_out, t);
           scheduler_addunlock(sched, t, cj_parent->grav.down_in);
         }
-- 
GitLab


From 7ee5f9066ff490e9d4e12ff8cccc9cd6deddc264 Mon Sep 17 00:00:00 2001
From: Matthieu Schaller <matthieu.schaller@durham.ac.uk>
Date: Tue, 6 Nov 2018 16:05:32 +0000
Subject: [PATCH 09/12] No need for the 'end_force_in' task.

---
 examples/analyse_tasks.py |  2 +-
 examples/plot_tasks.py    |  2 +-
 src/cell.c                |  2 --
 src/cell.h                |  3 ---
 src/engine_maketasks.c    | 29 ++++++-----------------------
 src/engine_marktasks.c    |  3 +--
 src/task.c                |  1 -
 src/task.h                |  1 -
 8 files changed, 9 insertions(+), 34 deletions(-)

diff --git a/examples/analyse_tasks.py b/examples/analyse_tasks.py
index 343814d58..5dc381411 100755
--- a/examples/analyse_tasks.py
+++ b/examples/analyse_tasks.py
@@ -53,7 +53,7 @@ infile = args.input
 #  Tasks and subtypes. Indexed as in tasks.h.
 TASKTYPES = ["none", "sort", "self", "pair", "sub_self", "sub_pair",
              "init_grav", "init_grav_out", "ghost_in", "ghost", "ghost_out", "extra_ghost", "drift_part", "drift_gpart", "drift_gpart_out",
-             "end_force", "end_force_in", "kick1", "kick2", "timestep", "send", "recv", "grav_long_range", "grav_mm", "grav_down_in", 
+             "end_force", "kick1", "kick2", "timestep", "send", "recv", "grav_long_range", "grav_mm", "grav_down_in", 
              "grav_down", "grav_mesh", "cooling", "star_formation", "sourceterms",
              "stars_ghost_in", "stars_ghost",   "stars_ghost_out",
              "count"]
diff --git a/examples/plot_tasks.py b/examples/plot_tasks.py
index 588061cd4..46fe56813 100755
--- a/examples/plot_tasks.py
+++ b/examples/plot_tasks.py
@@ -111,7 +111,7 @@ pl.rcParams.update(PLOT_PARAMS)
 #  Tasks and subtypes. Indexed as in tasks.h.
 TASKTYPES = ["none", "sort", "self", "pair", "sub_self", "sub_pair",
              "init_grav", "init_grav_out", "ghost_in", "ghost", "ghost_out", "extra_ghost", "drift_part", "drift_gpart", "drift_gpart_out",
-             "end_force", "end_force_in", "kick1", "kick2", "timestep", "send", "recv", "grav_long_range", "grav_mm", "grav_down_in", 
+             "end_force", "kick1", "kick2", "timestep", "send", "recv", "grav_long_range", "grav_mm", "grav_down_in", 
              "grav_down", "grav_mesh", "cooling", "star_formation", "sourceterms",
              "stars_ghost_in", "stars_ghost",   "stars_ghost_out",
              "count"]
diff --git a/src/cell.c b/src/cell.c
index 79ccf0211..645dfbd3e 100644
--- a/src/cell.c
+++ b/src/cell.c
@@ -2770,7 +2770,6 @@ int cell_unskip_hydro_tasks(struct cell *c, struct scheduler *s) {
     if (c->kick2 != NULL) scheduler_activate(s, c->kick2);
     if (c->timestep != NULL) scheduler_activate(s, c->timestep);
     if (c->end_force != NULL) scheduler_activate(s, c->end_force);
-    if (c->end_force_in != NULL) scheduler_activate(s, c->end_force_in);
     if (c->hydro.cooling != NULL) scheduler_activate(s, c->hydro.cooling);
     if (c->hydro.star_formation != NULL)
       scheduler_activate(s, c->hydro.star_formation);
@@ -2920,7 +2919,6 @@ int cell_unskip_gravity_tasks(struct cell *c, struct scheduler *s) {
     if (c->kick2 != NULL) scheduler_activate(s, c->kick2);
     if (c->timestep != NULL) scheduler_activate(s, c->timestep);
     if (c->end_force != NULL) scheduler_activate(s, c->end_force);
-    if (c->end_force_in != NULL) scheduler_activate(s, c->end_force_in);
     if (c->grav.down != NULL) scheduler_activate(s, c->grav.down);
     if (c->grav.down_in != NULL) scheduler_activate(s, c->grav.down_in);
     if (c->grav.mesh != NULL) scheduler_activate(s, c->grav.mesh);
diff --git a/src/cell.h b/src/cell.h
index a5d0f7c61..bc3a21dbb 100644
--- a/src/cell.h
+++ b/src/cell.h
@@ -544,9 +544,6 @@ struct cell {
   /*! The task to end the force calculation */
   struct task *end_force;
 
-  /*! Implicit task (going up- and down the tree) for the end force */
-  struct task *end_force_in;
-
   /*! The first kick task */
   struct task *kick1;
 
diff --git a/src/engine_maketasks.c b/src/engine_maketasks.c
index 21b4cec0e..35f161f92 100644
--- a/src/engine_maketasks.c
+++ b/src/engine_maketasks.c
@@ -462,10 +462,6 @@ void engine_make_hierarchical_tasks_common(struct engine *e, struct cell *c) {
       c->end_force = scheduler_addtask(s, task_type_end_force,
                                        task_subtype_none, 0, 0, c, NULL);
 
-      /* Implicit task for the up pass */
-      c->end_force_in = scheduler_addtask(s, task_type_end_force_in,
-                                          task_subtype_none, 0, 1, c, NULL);
-
       /* Subgrid tasks */
       if (is_with_cooling) {
 
@@ -491,33 +487,20 @@ void engine_make_hierarchical_tasks_common(struct engine *e, struct cell *c) {
         scheduler_addunlock(s, c->kick2, c->timestep);
       }
       
-      //scheduler_addunlock(s, c->end_force_in, c->end_force);
       scheduler_addunlock(s, c->timestep, c->kick1);
 
 #if defined(WITH_LOGGER)
       scheduler_addunlock(s, c->kick1, c->logger);
 #endif
     }
-  }
-
-  /* We are below the super-cell but not below the maximal splitting depth */
-  else if (c->super != NULL) {
-
-    /* Local tasks only... */
-    if (c->nodeID == e->nodeID) {
-
-      c->end_force_in = scheduler_addtask(s, task_type_end_force_in,
-                                          task_subtype_none, 0, 1, c, NULL);
+  } else { /* We are above the super-cell so need to go deeper */
 
-      scheduler_addunlock(s, c->end_force_in, c->parent->end_force_in);
-    }
+    /* Recurse. */
+    if (c->split)
+      for (int k = 0; k < 8; k++)
+	if (c->progeny[k] != NULL)
+	  engine_make_hierarchical_tasks_common(e, c->progeny[k]);
   }
-
-  /* Recurse. */
-  if (c->split)
-    for (int k = 0; k < 8; k++)
-      if (c->progeny[k] != NULL)
-        engine_make_hierarchical_tasks_common(e, c->progeny[k]);
 }
 
 /**
diff --git a/src/engine_marktasks.c b/src/engine_marktasks.c
index 2065f6364..3d607bcf8 100644
--- a/src/engine_marktasks.c
+++ b/src/engine_marktasks.c
@@ -444,8 +444,7 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
     }
 
     /* End force ? */
-    else if (t_type == task_type_end_force ||
-             t_type == task_type_end_force_in) {
+    else if (t_type == task_type_end_force) {
 
       if (cell_is_active_hydro(t->ci, e) || cell_is_active_gravity(t->ci, e))
         scheduler_activate(s, t);
diff --git a/src/task.c b/src/task.c
index 13ce88452..1665672fe 100644
--- a/src/task.c
+++ b/src/task.c
@@ -63,7 +63,6 @@ const char *taskID_names[task_type_count] = {"none",
                                              "drift_gpart",
                                              "drift_gpart_out",
                                              "end_force",
-                                             "end_force_in",
                                              "kick1",
                                              "kick2",
                                              "timestep",
diff --git a/src/task.h b/src/task.h
index 90016472c..64b611534 100644
--- a/src/task.h
+++ b/src/task.h
@@ -55,7 +55,6 @@ enum task_types {
   task_type_drift_gpart,
   task_type_drift_gpart_out, /* Implicit */
   task_type_end_force,
-  task_type_end_force_in, /* Implicit */
   task_type_kick1,
   task_type_kick2,
   task_type_timestep,
-- 
GitLab


From 5c4fbfb2b93bdeab0d0851d108f9bb8ca18c8081 Mon Sep 17 00:00:00 2001
From: Matthieu Schaller <schaller@strw.leidenuniv.nl>
Date: Tue, 6 Nov 2018 17:09:42 +0100
Subject: [PATCH 10/12] Applied code formatting tool.

---
 src/engine_maketasks.c | 16 +++++++---------
 src/space.c            | 16 ++++++++--------
 src/task.c             |  6 ++++--
 3 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/src/engine_maketasks.c b/src/engine_maketasks.c
index 35f161f92..54a735139 100644
--- a/src/engine_maketasks.c
+++ b/src/engine_maketasks.c
@@ -486,7 +486,7 @@ void engine_make_hierarchical_tasks_common(struct engine *e, struct cell *c) {
       } else {
         scheduler_addunlock(s, c->kick2, c->timestep);
       }
-      
+
       scheduler_addunlock(s, c->timestep, c->kick1);
 
 #if defined(WITH_LOGGER)
@@ -498,8 +498,8 @@ void engine_make_hierarchical_tasks_common(struct engine *e, struct cell *c) {
     /* Recurse. */
     if (c->split)
       for (int k = 0; k < 8; k++)
-	if (c->progeny[k] != NULL)
-	  engine_make_hierarchical_tasks_common(e, c->progeny[k]);
+        if (c->progeny[k] != NULL)
+          engine_make_hierarchical_tasks_common(e, c->progeny[k]);
   }
 }
 
@@ -572,7 +572,7 @@ void engine_make_hierarchical_tasks_gravity(struct engine *e, struct cell *c) {
   }
 
   /* We are below the super-cell but not below the maximal splitting depth */
-  else if ((c->grav.super != NULL)  &&
+  else if ((c->grav.super != NULL) &&
            ((c->maxdepth - c->depth) >= space_subdepth_diff_grav)) {
 
     /* Local tasks only... */
@@ -581,7 +581,7 @@ void engine_make_hierarchical_tasks_gravity(struct engine *e, struct cell *c) {
       if (is_self_gravity) {
 
         c->grav.drift_out = scheduler_addtask(s, task_type_drift_gpart_out,
-					      task_subtype_none, 0, 1, c, NULL);
+                                              task_subtype_none, 0, 1, c, NULL);
 
         c->grav.init_out = scheduler_addtask(s, task_type_init_grav_out,
                                              task_subtype_none, 0, 1, c, NULL);
@@ -1169,9 +1169,8 @@ void engine_link_gravity_tasks(struct engine *e) {
 
         /* drift ---+-> gravity --> grav_down */
         /* init  --/    */
-
         if (ci_parent != cj_parent) { /* Avoid double unlock */
-	  scheduler_addunlock(sched, cj_parent->grav.drift_out, t);
+          scheduler_addunlock(sched, cj_parent->grav.drift_out, t);
           scheduler_addunlock(sched, cj_parent->grav.init_out, t);
           scheduler_addunlock(sched, t, cj_parent->grav.down_in);
         }
@@ -1219,9 +1218,8 @@ void engine_link_gravity_tasks(struct engine *e) {
 
         /* drift ---+-> gravity --> grav_down */
         /* init  --/    */
-
         if (ci_parent != cj_parent) { /* Avoid double unlock */
-	  scheduler_addunlock(sched, cj_parent->grav.drift_out, t);
+          scheduler_addunlock(sched, cj_parent->grav.drift_out, t);
           scheduler_addunlock(sched, cj_parent->grav.init_out, t);
           scheduler_addunlock(sched, t, cj_parent->grav.down_in);
         }
diff --git a/src/space.c b/src/space.c
index 19cf1e408..983e1d2a9 100644
--- a/src/space.c
+++ b/src/space.c
@@ -505,14 +505,14 @@ void space_regrid(struct space *s, int verbose) {
           c->grav.ti_old_multipole = ti_current;
 #ifdef WITH_MPI
           c->mpi.tag = -1;
-	  c->mpi.hydro.recv_xv = NULL;
-	  c->mpi.hydro.recv_rho = NULL;
-	  c->mpi.hydro.recv_gradient = NULL;
-	  c->mpi.hydro.send_xv = NULL;
-	  c->mpi.hydro.send_rho = NULL;
-	  c->mpi.hydro.send_gradient = NULL;
-	  c->mpi.grav.recv = NULL;
-	  c->mpi.grav.send = NULL;
+          c->mpi.hydro.recv_xv = NULL;
+          c->mpi.hydro.recv_rho = NULL;
+          c->mpi.hydro.recv_gradient = NULL;
+          c->mpi.hydro.send_xv = NULL;
+          c->mpi.hydro.send_rho = NULL;
+          c->mpi.hydro.send_gradient = NULL;
+          c->mpi.grav.recv = NULL;
+          c->mpi.grav.send = NULL;
 #endif  // WITH_MPI
           if (s->gravity) c->grav.multipole = &s->multipoles_top[cid];
 #ifdef SWIFT_DEBUG_CHECKS
diff --git a/src/task.c b/src/task.c
index 1665672fe..9d4be3aaa 100644
--- a/src/task.c
+++ b/src/task.c
@@ -411,8 +411,10 @@ int task_lock(struct task *t) {
         char buff[MPI_MAX_ERROR_STRING];
         int len;
         MPI_Error_string(err, buff, &len);
-        error("Failed to test request on send/recv task (type=%s/%s tag=%lld, %s).",
-              taskID_names[t->type], subtaskID_names[t->subtype], t->flags, buff);
+        error(
+            "Failed to test request on send/recv task (type=%s/%s tag=%lld, "
+            "%s).",
+            taskID_names[t->type], subtaskID_names[t->subtype], t->flags, buff);
       }
       return res;
 #else
-- 
GitLab


From d8428c4e22de5f749cbb137aa9e58b8a23192484 Mon Sep 17 00:00:00 2001
From: Matthieu Schaller <schaller@strw.leidenuniv.nl>
Date: Wed, 7 Nov 2018 15:52:07 +0100
Subject: [PATCH 11/12] Added more timers to the main sections of the engine to
 understand where the un-accounted time is.

---
 src/engine.c | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/src/engine.c b/src/engine.c
index 539558315..2fbffb1a8 100644
--- a/src/engine.c
+++ b/src/engine.c
@@ -1044,6 +1044,8 @@ void engine_repartition_trigger(struct engine *e) {
 
 #ifdef WITH_MPI
 
+  const ticks tic = getticks();
+
   /* Do nothing if there have not been enough steps since the last
    * repartition, don't want to repeat this too often or immediately after
    * a repartition step. Also nothing to do when requested. */
@@ -1112,6 +1114,10 @@ void engine_repartition_trigger(struct engine *e) {
   /* We always reset CPU time for next check, unless it will not be used. */
   if (e->reparttype->type != REPART_NONE)
     e->cputime_last_step = clocks_get_cputime_used();
+
+  if (e->verbose)
+    message("took %.3f %s", clocks_from_ticks(getticks() - tic),
+            clocks_getunit());
 #endif
 }
 
@@ -1591,6 +1597,8 @@ void engine_exchange_top_multipoles(struct engine *e) {
 
 #ifdef WITH_MPI
 
+  ticks tic = getticks();
+
 #ifdef SWIFT_DEBUG_CHECKS
   for (int i = 0; i < e->s->nr_cells; ++i) {
     const struct gravity_tensors *m = &e->s->multipoles_top[i];
@@ -1659,6 +1667,9 @@ void engine_exchange_top_multipoles(struct engine *e) {
         counter, e->total_nr_gparts);
 #endif
 
+  if (e->verbose)
+    message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+            clocks_getunit());
 #else
   error("SWIFT was not compiled with MPI support.");
 #endif
@@ -1993,6 +2004,8 @@ void engine_rebuild(struct engine *e, int repartitioned,
   /* Re-build the space. */
   space_rebuild(e->s, repartitioned, e->verbose);
 
+  const ticks tic2 = getticks();
+
   /* Update the global counters of particles */
   long long num_particles[3] = {e->s->nr_parts, e->s->nr_gparts,
                                 e->s->nr_sparts};
@@ -2009,6 +2022,10 @@ void engine_rebuild(struct engine *e, int repartitioned,
   e->nr_inhibited_gparts = 0;
   e->nr_inhibited_sparts = 0;
 
+  if (e->verbose)
+    message("updating particle counts took %.3f %s.",
+            clocks_from_ticks(getticks() - tic2), clocks_getunit());
+
   /* Re-compute the mesh forces */
   if ((e->policy & engine_policy_self_gravity) && e->s->periodic)
     pm_mesh_compute_potential(e->mesh, e->s, &e->threadpool, e->verbose);
@@ -3862,6 +3879,8 @@ void engine_makeproxies(struct engine *e) {
 void engine_split(struct engine *e, struct partition *initial_partition) {
 
 #ifdef WITH_MPI
+  const ticks tic = getticks();
+
   struct space *s = e->s;
 
   /* Do the initial partition of the cells. */
@@ -3942,6 +3961,10 @@ void engine_split(struct engine *e, struct partition *initial_partition) {
                     s->nr_sparts, e->verbose);
 #endif
 
+  if (e->verbose)
+    message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+            clocks_getunit());
+
 #else
   error("SWIFT was not compiled with MPI support.");
 #endif
@@ -5114,6 +5137,8 @@ void engine_init_output_lists(struct engine *e, struct swift_params *params) {
  */
 void engine_recompute_displacement_constraint(struct engine *e) {
 
+  const ticks tic = getticks();
+
   /* Get the cosmological information */
   const struct cosmology *cosmo = e->cosmology;
   const float Om = cosmo->Omega_m;
@@ -5224,6 +5249,10 @@ void engine_recompute_displacement_constraint(struct engine *e) {
 
   if (e->verbose)
     message("max_dt_RMS_displacement = %e", e->dt_max_RMS_displacement);
+
+  if (e->verbose)
+    message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+            clocks_getunit());
 }
 
 /**
-- 
GitLab


From 83103e4003cb6713032eb58659933a9c3862c1a8 Mon Sep 17 00:00:00 2001
From: Matthieu Schaller <schaller@strw.leidenuniv.nl>
Date: Thu, 8 Nov 2018 18:18:12 +0100
Subject: [PATCH 12/12] Post-merge fixes.

---
 tools/task_plots/analyse_tasks.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/task_plots/analyse_tasks.py b/tools/task_plots/analyse_tasks.py
index 603417a3c..0b6439ff4 100755
--- a/tools/task_plots/analyse_tasks.py
+++ b/tools/task_plots/analyse_tasks.py
@@ -77,6 +77,7 @@ TASKTYPES = [
     "extra_ghost",
     "drift_part",
     "drift_gpart",
+    "drift_gpart_out",
     "end_force",
     "kick1",
     "kick2",
-- 
GitLab