Split the star formation task into two parts: The star formation bit and the...

Split the star formation task into two parts: The star formation bit and the resorting bit. Move the sf_count communication to just depend on the first one in order to speed things up.

src/cell.c

@@ -3309,6 +3309,7 @@ int cell_unskip_hydro_tasks(struct cell *c, struct scheduler *s) {
 
     if (c->top->hydro.star_formation != NULL) {
       scheduler_activate(s, c->top->hydro.star_formation);
+      scheduler_activate(s, c->top->hydro.stars_resort);
       cell_activate_drift_spart(c, s);
     }
   }
@@ -3462,13 +3463,6 @@ int cell_unskip_gravity_tasks(struct cell *c, struct scheduler *s) {
     if (c->grav.long_range != NULL) scheduler_activate(s, c->grav.long_range);
     if (c->grav.end_force != NULL) scheduler_activate(s, c->grav.end_force);
     if (c->logger != NULL) scheduler_activate(s, c->logger);
-
-    /* Subgrid tasks */
-    if ((e->policy & engine_policy_cooling) && c->hydro.cooling != NULL)
-      scheduler_activate(s, c->hydro.cooling);
-    if ((e->policy & engine_policy_star_formation) &&
-        c->hydro.star_formation != NULL)
-      scheduler_activate(s, c->hydro.star_formation);
   }
 
   return rebuild;

src/cell.h

@@ -288,7 +288,8 @@ enum cell_flags {
   cell_flag_do_stars_drift = (1UL << 9),
   cell_flag_do_stars_sub_drift = (1UL << 10),
   cell_flag_do_bh_drift = (1UL << 11),
-  cell_flag_do_bh_sub_drift = (1UL << 12)
+  cell_flag_do_bh_sub_drift = (1UL << 12),
+  cell_flag_do_stars_resort = (1UL << 13)
 };
 
 /**
@@ -377,6 +378,9 @@ struct cell {
     /*! Task for star formation */
     struct task *star_formation;
 
+    /*! Task for sorting the stars again after a SF event */
+    struct task *stars_resort;
+
     /*! Max smoothing length in this cell. */
     double h_max;
 

src/engine_maketasks.c

@@ -702,6 +702,9 @@ void engine_make_hierarchical_tasks_common(struct engine *e, struct cell *c) {
     if (with_star_formation && c->hydro.count > 0) {
       c->hydro.star_formation = scheduler_addtask(
           s, task_type_star_formation, task_subtype_none, 0, 0, c, NULL);
+      c->hydro.stars_resort = scheduler_addtask(
+          s, task_type_stars_resort, task_subtype_none, 0, 0, c, NULL);
+      scheduler_addunlock(s, c->hydro.star_formation, c->hydro.stars_resort);
     }
   }
 
@@ -996,8 +999,7 @@ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c) {
         scheduler_addunlock(s, c->stars.stars_out, c->super->timestep);
 
         if (with_star_formation && c->hydro.count > 0) {
-          scheduler_addunlock(s, c->top->hydro.star_formation,
-                              c->stars.stars_in);
+          scheduler_addunlock(s, c->top->hydro.stars_resort, c->stars.stars_in);
         }
       }
 

src/engine_marktasks.c

@@ -905,7 +905,8 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
     }
 
     /* Subgrid tasks: star formation */
-    else if (t_type == task_type_star_formation) {
+    else if (t_type == task_type_star_formation ||
+             t_type == task_type_stars_resort) {
       if (cell_is_active_hydro(t->ci, e)) {
         scheduler_activate(s, t);
         cell_activate_super_spart_drifts(t->ci, s);

src/runner.c

@@ -1180,14 +1180,27 @@ void runner_do_star_formation(struct runner *r, struct cell *c, int timer) {
    * re-compute them. */
   if (with_feedback && (c == c->top) &&
       (current_stars_count != c->stars.count)) {
-
+    cell_set_flag(c, cell_flag_do_stars_resort);
     cell_clear_stars_sort_flags(c, /*clear_unused_flags=*/0);
-    runner_do_all_stars_sort(r, c);
   }
 
   if (timer) TIMER_TOC(timer_do_star_formation);
 }
 
+void runner_do_stars_resort(struct runner *r, struct cell *c, int timer) {
+
+#ifdef SWIFT_DEBUG_CHECKS
+  if (c->nodeID != r->e->nodeID) error("Task must be run locally!");
+  if (c->depth != 0) error("Task must be run at the top-level");
+#endif
+
+  /* Did we demand a recalculation of the stars'sorts? */
+  if (cell_get_flag(c, cell_flag_do_stars_resort)) {
+    runner_do_all_stars_sort(r, c);
+    cell_clear_flag(c, cell_flag_do_stars_resort);
+  }
+}
+
 /**
  * @brief Sort the entries in ascending order using QuickSort.
  *
@@ -4335,6 +4348,9 @@ void *runner_main(void *data) {
         case task_type_star_formation:
           runner_do_star_formation(r, t->ci, 1);
           break;
+        case task_type_stars_resort:
+          runner_do_stars_resort(r, t->ci, 1);
+          break;
         case task_type_fof_self:
           runner_do_fof_self(r, t->ci, 1);
           break;

src/space.c

@@ -232,6 +232,8 @@ void space_rebuild_recycle_mapper(void *map_data, int num_elements,
     c->hydro.ghost_in = NULL;
     c->hydro.ghost_out = NULL;
     c->hydro.ghost = NULL;
+    c->hydro.star_formation = NULL;
+    c->hydro.stars_resort = NULL;
     c->stars.ghost = NULL;
     c->stars.density = NULL;
     c->stars.feedback = NULL;

src/task.c

@@ -89,6 +89,7 @@ const char *taskID_names[task_type_count] = {"none",
                                              "stars_ghost",
                                              "stars_ghost_out",
                                              "stars_sort",
+                                             "stars_resort",
                                              "bh_in",
                                              "bh_out",
                                              "bh_ghost",
@@ -168,6 +169,7 @@ __attribute__((always_inline)) INLINE static enum task_actions task_acts_on(
     case task_type_drift_spart:
     case task_type_stars_ghost:
     case task_type_stars_sort:
+    case task_type_stars_resort:
       return task_action_spart;
       break;
 
@@ -399,6 +401,7 @@ void task_unlock(struct task *t) {
       break;
 
     case task_type_stars_sort:
+    case task_type_stars_resort:
       cell_sunlocktree(ci);
       break;
 
@@ -519,6 +522,7 @@ int task_lock(struct task *t) {
       break;
 
     case task_type_stars_sort:
+    case task_type_stars_resort:
       if (ci->stars.hold) return 0;
       if (cell_slocktree(ci) != 0) return 0;
       break;

src/task.h

@@ -84,6 +84,7 @@ enum task_types {
   task_type_stars_ghost,
   task_type_stars_ghost_out, /* Implicit */
   task_type_stars_sort,
+  task_type_stars_resort,
   task_type_bh_in,  /* Implicit */
   task_type_bh_out, /* Implicit */
   task_type_bh_ghost,

tools/task_plots/analyse_tasks.py

@@ -104,6 +104,7 @@ TASKTYPES = [
     "stars_ghost",
     "stars_ghost_out",
     "stars_sort",
+    "stars_resort",
     "bh_in",
     "bh_out",
     "bh_ghost",

tools/task_plots/plot_tasks.py

@@ -189,6 +189,7 @@ TASKTYPES = [
     "stars_ghost",
     "stars_ghost_out",
     "stars_sort",
+    "stars_resort",
     "bh_in",
     "bh_out",
     "bh_ghost",