Draft: Refactoring of the time-step communication tasks

Significant re-factoring of the way the time-step sizes are being exchanged.

Summary:

• A new top-level task collects the time-step sizes from the super level to the top-level. This was formerly done by making engine_collect_end_of_step() recurse.
• The timestep, timestep_limiter, and timestep_sync tasks all unlock that top-level task.
• engine_collect_end_of_step() now only loops (via threadpool) over the local top-level cells. No recursion any more.
• For each pair of top-level cells in the proxies we construct a pair of send/recv comm tasks.
• That comm task packs up the dt of the whole hierarchy sends it and unpacks the time-step sizes.
• The top-level time-step collection task unlocks the send.
• The individual per-species tend communication tasks that used to live at the super level are removed.
• The second call to engine_launch() done every step to deal with the timestep limiter effect is removed (as it is now properly dealt with by the top-level task dependency)

This should help speed up the smallest steps by reducing the level of the plateau we usually see in the "main sequence" plots.

src/cell.h

@@ -228,36 +228,46 @@ struct pcell_step {
 
   struct {
 
-    /*! Minimal integer end-of-timestep in this cell (hydro) */
-    integertime_t ti_end_min;
+    struct {
 
-    /*! Maximal distance any #part has travelled since last rebuild */
-    float dx_max_part;
-  } hydro;
+      struct {
 
-  struct {
+        /*! Minimal integer end-of-timestep in this cell (hydro) */
+        integertime_t ti_end_min;
 
-    /*! Minimal integer end-of-timestep in this cell (gravity) */
-    integertime_t ti_end_min;
-  } grav;
+        /*! Maximal distance any #part has travelled since last rebuild */
+        float dx_max_part;
+      } hydro;
 
-  struct {
+      struct {
 
-    /*! Minimal integer end-of-timestep in this cell (stars) */
-    integertime_t ti_end_min;
+        /*! Minimal integer end-of-timestep in this cell (gravity) */
+        integertime_t ti_end_min;
+      } grav;
 
-    /*! Maximal distance any #part has travelled since last rebuild */
-    float dx_max_part;
-  } stars;
+      struct {
 
-  struct {
+        /*! Minimal integer end-of-timestep in this cell (stars) */
+        integertime_t ti_end_min;
 
-    /*! Minimal integer end-of-timestep in this cell (black_holes) */
-    integertime_t ti_end_min;
+        /*! Maximal distance any #part has travelled since last rebuild */
+        float dx_max_part;
+      } stars;
 
-    /*! Maximal distance any #part has travelled since last rebuild */
-    float dx_max_part;
-  } black_holes;
+      struct {
+
+        /*! Minimal integer end-of-timestep in this cell (black_holes) */
+        integertime_t ti_end_min;
+
+        /*! Maximal distance any #part has travelled since last rebuild */
+        float dx_max_part;
+      } black_holes;
+
+    } ti_end;
+
+    /* Did we actually update anything? */
+    char do_anything;
+  };
 };
 
 /**
@@ -448,6 +458,9 @@ struct cell {
   /*! The maximal depth of this cell and its progenies */
   char maxdepth;
 
+  /*! Flag that the time-step info in this cell changed in this step */
+  char dt_changed;
+
 #if defined(SWIFT_DEBUG_CHECKS) || defined(SWIFT_CELL_GRAPH)
   /* Cell ID (for debugging) */
   long long cellID;
@@ -500,8 +513,8 @@ void cell_unpack_bpart_swallow(struct cell *c,
                                const struct black_holes_bpart_data *data);
 int cell_pack_tags(const struct cell *c, int *tags);
 int cell_unpack_tags(const int *tags, struct cell *c);
-int cell_pack_end_step(const struct cell *c, struct pcell_step *pcell);
-int cell_unpack_end_step(struct cell *c, const struct pcell_step *pcell);
+void cell_pack_end_step(const struct cell *c, struct pcell_step *pcell);
+void cell_unpack_end_step(struct cell *c, const struct pcell_step *pcell);
 void cell_pack_timebin(const struct cell *const c, timebin_t *const t);
 void cell_unpack_timebin(struct cell *const c, timebin_t *const t);
 int cell_pack_multipoles(struct cell *c, struct gravity_tensors *m);