diff --git a/src/cell.c b/src/cell.c
index 32a47e9451076ef18f70b737c68d074d2593026e..3c6d6e4507b9a175e1741f0bd7b60c6726279c0d 100644
--- a/src/cell.c
+++ b/src/cell.c
@@ -71,6 +71,8 @@
#include "tools.h"
#include "tracers.h"
+extern int engine_star_resort_task_depth;
+
/* Global variables. */
int cell_next_tag = 0;
@@ -2398,6 +2400,77 @@ void cell_clear_limiter_flags(struct cell *c, void *data) {
cell_flag_do_hydro_limiter | cell_flag_do_hydro_sub_limiter);
}
+/**
+ * @brief Recursively clear the stars_resort flag in a cell hierarchy.
+ *
+ * @param c The #cell to act on.
+ */
+void cell_set_star_resort_flag(struct cell *c) {
+
+ cell_set_flag(c, cell_flag_do_stars_resort);
+
+ /* Abort if we reched the level where the resorting task lives */
+ if (c->depth == engine_star_resort_task_depth || c->hydro.super == c) return;
+
+ if (c->split) {
+ for (int k = 0; k < 8; ++k)
+ if (c->progeny[k] != NULL) cell_set_star_resort_flag(c->progeny[k]);
+ }
+}
+
+/**
+ * @brief Recurses in a cell hierarchy down to the level where the
+ * star resort tasks are and activates them.
+ *
+ * The function will fail if called *below* the super-level
+ *
+ * @param c The #cell to recurse into.
+ * @param s The #scheduler.
+ */
+void cell_activate_star_resort_tasks(struct cell *c, struct scheduler *s) {
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->hydro.super != NULL && c->hydro.super != c)
+ error("Function called below the super level!");
+#endif
+
+ /* The resort tasks are at either the chosen depth or the super level,
+ * whichever comes first. */
+ if (c->depth == engine_star_resort_task_depth || c->hydro.super == c) {
+ scheduler_activate(s, c->hydro.stars_resort);
+ } else {
+ for (int k = 0; k < 8; ++k) {
+ if (c->progeny[k] != NULL) {
+ cell_activate_star_resort_tasks(c->progeny[k], s);
+ }
+ }
+ }
+}
+
+/**
+ * @brief Activate the star formation task as well as the resorting of stars
+ *
+ * Must be called at the top-level in the tree (where the SF task is...)
+ *
+ * @param c The (top-level) #cell.
+ * @param s The #scheduler.
+ */
+void cell_activate_star_formation_tasks(struct cell *c, struct scheduler *s) {
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->depth != 0) error("Function should be called at the top-level only");
+#endif
+
+ /* Have we already unskipped that task? */
+ if (c->hydro.star_formation->skip == 0) return;
+
+ /* Activate the star formation task */
+ scheduler_activate(s, c->hydro.star_formation);
+
+ /* Activate the star resort tasks at whatever level they are */
+ cell_activate_star_resort_tasks(c, s);
+}
+
/**
* @brief Recurse down in a cell hierarchy until the hydro.super level is
* reached and activate the spart drift at that level.
@@ -3431,9 +3504,7 @@ int cell_unskip_hydro_tasks(struct cell *c, struct scheduler *s) {
if (c->logger != NULL) scheduler_activate(s, c->logger);
if (c->top->hydro.star_formation != NULL) {
- scheduler_activate(s, c->top->hydro.star_formation);
- scheduler_activate(s, c->top->hydro.stars_resort);
- cell_activate_drift_spart(c, s);
+ cell_activate_star_formation_tasks(c->top, s);
}
}
@@ -5163,10 +5234,17 @@ void cell_recursively_shift_sparts(struct cell *c,
/* When directly above the leaf with the new particle: increase the particle
* count */
/* When after the leaf with the new particle: shift by one position */
- if (main_branch)
+ if (main_branch) {
c->stars.count++;
- else
+
+ /* Indicate that the cell is not sorted and cancel the pointer sorting
+ * arrays. */
+ c->stars.sorted = 0;
+ cell_free_stars_sorts(c);
+
+ } else {
c->stars.parts++;
+ }
}
/**
diff --git a/src/cell.h b/src/cell.h
index 9c9e2e64f85305e37fe912e0436e5dbde333f6d8..f59576b4fc303580fc37db31e2fecb707ec774e6 100644
--- a/src/cell.h
+++ b/src/cell.h
@@ -874,6 +874,8 @@ void cell_drift_multipole(struct cell *c, const struct engine *e);
void cell_drift_all_multipoles(struct cell *c, const struct engine *e);
void cell_check_timesteps(struct cell *c);
void cell_store_pre_drift_values(struct cell *c);
+void cell_set_star_resort_flag(struct cell *c);
+void cell_activate_star_formation_tasks(struct cell *c, struct scheduler *s);
void cell_activate_subcell_hydro_tasks(struct cell *ci, struct cell *cj,
struct scheduler *s);
void cell_activate_subcell_grav_tasks(struct cell *ci, struct cell *cj,
diff --git a/src/engine.c b/src/engine.c
index 1799d5b09332e1ef9ef13f3b2912c27c03425f47..c3ace9f9f7d237708f57f5ddfd575b11d3c62ed1 100644
--- a/src/engine.c
+++ b/src/engine.c
@@ -3327,7 +3327,7 @@ void engine_skip_force_and_kick(struct engine *e) {
t->type == task_type_drift_gpart_out || t->type == task_type_cooling ||
t->type == task_type_stars_in || t->type == task_type_stars_out ||
t->type == task_type_star_formation ||
- t->type == task_type_extra_ghost ||
+ t->type == task_type_stars_resort || t->type == task_type_extra_ghost ||
t->type == task_type_bh_swallow_ghost1 ||
t->type == task_type_bh_swallow_ghost2 ||
t->type == task_type_bh_swallow_ghost3 ||
diff --git a/src/engine.h b/src/engine.h
index 967a430871648ac4cb91f390f8b85675a314d3a8..dc5c2c9ddd9a843d3e677e9b56c47f7451515b2f 100644
--- a/src/engine.h
+++ b/src/engine.h
@@ -113,6 +113,7 @@ enum engine_step_properties {
#define engine_default_timesteps_file_name "timesteps"
#define engine_max_parts_per_ghost_default 1000
#define engine_max_sparts_per_ghost_default 1000
+#define engine_star_resort_task_depth_default 2
#define engine_tasks_per_cell_margin 1.2
/**
diff --git a/src/engine_maketasks.c b/src/engine_maketasks.c
index 036c5aef0adb44a47777a4ef3c661368819096c2..6a8341068cb87d2994a8d658a9a7255462ef294e 100644
--- a/src/engine_maketasks.c
+++ b/src/engine_maketasks.c
@@ -54,6 +54,7 @@
extern int engine_max_parts_per_ghost;
extern int engine_max_sparts_per_ghost;
+extern int engine_star_resort_task_depth;
/**
* @brief Add send tasks for the gravity pairs to a hierarchy of cells.
@@ -775,9 +776,6 @@ void engine_make_hierarchical_tasks_common(struct engine *e, struct cell *c) {
if (with_star_formation && c->hydro.count > 0) {
c->hydro.star_formation = scheduler_addtask(
s, task_type_star_formation, task_subtype_none, 0, 0, c, NULL);
- c->hydro.stars_resort = scheduler_addtask(
- s, task_type_stars_resort, task_subtype_none, 0, 0, c, NULL);
- scheduler_addunlock(s, c->hydro.star_formation, c->hydro.stars_resort);
}
}
@@ -979,8 +977,11 @@ void engine_add_ghosts(struct engine *e, struct cell *c, struct task *ghost_in,
*
* @param e The #engine.
* @param c The #cell.
+ * @param star_resort_cell Pointer to the cell where the star_resort task has
+ * been created. NULL above that level or if not running with star formation.
*/
-void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c) {
+void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c,
+ struct cell *star_resort_cell) {
struct scheduler *s = &e->sched;
const int with_stars = (e->policy & engine_policy_stars);
@@ -989,6 +990,25 @@ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c) {
const int with_star_formation = (e->policy & engine_policy_star_formation);
const int with_black_holes = (e->policy & engine_policy_black_holes);
+ /* Are we are the level where we create the stars' resort tasks?
+ * If the tree is shallow, we need to do this at the super-level if the
+ * super-level is above the level we want */
+ if ((c->nodeID == e->nodeID) && (star_resort_cell == NULL) &&
+ (c->depth == engine_star_resort_task_depth || c->hydro.super == c)) {
+
+ if (with_star_formation && c->hydro.count > 0) {
+
+ /* Record this is the level where we re-sort */
+ star_resort_cell = c;
+
+ c->hydro.stars_resort = scheduler_addtask(
+ s, task_type_stars_resort, task_subtype_none, 0, 0, c, NULL);
+
+ scheduler_addunlock(s, c->top->hydro.star_formation,
+ c->hydro.stars_resort);
+ }
+ }
+
/* Are we in a super-cell ? */
if (c->hydro.super == c) {
@@ -1078,7 +1098,8 @@ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c) {
scheduler_addunlock(s, c->stars.stars_out, c->super->timestep);
if (with_star_formation && c->hydro.count > 0) {
- scheduler_addunlock(s, c->top->hydro.stars_resort, c->stars.stars_in);
+ scheduler_addunlock(s, star_resort_cell->hydro.stars_resort,
+ c->stars.stars_in);
}
}
@@ -1114,7 +1135,8 @@ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c) {
if (c->split)
for (int k = 0; k < 8; k++)
if (c->progeny[k] != NULL)
- engine_make_hierarchical_tasks_hydro(e, c->progeny[k]);
+ engine_make_hierarchical_tasks_hydro(e, c->progeny[k],
+ star_resort_cell);
}
}
@@ -1131,7 +1153,8 @@ void engine_make_hierarchical_tasks_mapper(void *map_data, int num_elements,
/* Make the common tasks (time integration) */
engine_make_hierarchical_tasks_common(e, c);
/* Add the hydro stuff */
- if (with_hydro) engine_make_hierarchical_tasks_hydro(e, c);
+ if (with_hydro)
+ engine_make_hierarchical_tasks_hydro(e, c, /*star_resort_cell=*/NULL);
/* And the gravity stuff */
if (with_self_gravity || with_ext_gravity)
engine_make_hierarchical_tasks_gravity(e, c);
diff --git a/src/engine_marktasks.c b/src/engine_marktasks.c
index dcf1be29a677ce69d9fa471f7f734583ee6842eb..7f4e2d4c4b4ae4865eb2c2320d021be534cc57c3 100644
--- a/src/engine_marktasks.c
+++ b/src/engine_marktasks.c
@@ -939,10 +939,9 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
}
/* Subgrid tasks: star formation */
- else if (t_type == task_type_star_formation ||
- t_type == task_type_stars_resort) {
+ else if (t_type == task_type_star_formation) {
if (cell_is_active_hydro(t->ci, e)) {
- scheduler_activate(s, t);
+ cell_activate_star_formation_tasks(t->ci, s);
cell_activate_super_spart_drifts(t->ci, s);
}
}
diff --git a/src/runner.c b/src/runner.c
index eb63cca223886d594c8bef9c2072762213a6f495..82626e090439841f1a2e499e7b1ca987771f3c14 100644
--- a/src/runner.c
+++ b/src/runner.c
@@ -1216,8 +1216,7 @@ void runner_do_star_formation(struct runner *r, struct cell *c, int timer) {
* re-compute them. */
if (with_feedback && (c == c->top) &&
(current_stars_count != c->stars.count)) {
- cell_set_flag(c, cell_flag_do_stars_resort);
- cell_clear_stars_sort_flags(c, /*clear_unused_flags=*/0);
+ cell_set_star_resort_flag(c);
}
if (timer) TIMER_TOC(timer_do_star_formation);
@@ -1237,7 +1236,6 @@ void runner_do_stars_resort(struct runner *r, struct cell *c, const int timer) {
#ifdef SWIFT_DEBUG_CHECKS
if (c->nodeID != r->e->nodeID) error("Task must be run locally!");
- if (c->depth != 0) error("Task must be run at the top-level");
#endif
TIMER_TIC;
diff --git a/src/space.c b/src/space.c
index 322211ca6c313420a95a4d2e39937594b7ef3b96..6594fab935d55e969faa7482e71f4a2b73954399 100644
--- a/src/space.c
+++ b/src/space.c
@@ -92,6 +92,9 @@ int space_extra_gparts = space_extra_gparts_default;
int engine_max_parts_per_ghost = engine_max_parts_per_ghost_default;
int engine_max_sparts_per_ghost = engine_max_sparts_per_ghost_default;
+/*! Maximal depth at which the stars resort task can be pushed */
+int engine_star_resort_task_depth = engine_star_resort_task_depth_default;
+
/*! Expected maximal number of strays received at a rebuild */
int space_expected_max_nr_strays = space_expected_max_nr_strays_default;
#if defined(SWIFT_DEBUG_CHECKS) || defined(SWIFT_CELL_GRAPH)
@@ -5413,6 +5416,18 @@ void space_struct_dump(struct space *s, FILE *stream) {
"space_extra_sparts", "space_extra_sparts");
restart_write_blocks(&space_extra_bparts, sizeof(int), 1, stream,
"space_extra_bparts", "space_extra_bparts");
+ restart_write_blocks(&space_expected_max_nr_strays, sizeof(int), 1, stream,
+ "space_expected_max_nr_strays",
+ "space_expected_max_nr_strays");
+ restart_write_blocks(&engine_max_parts_per_ghost, sizeof(int), 1, stream,
+ "engine_max_parts_per_ghost",
+ "engine_max_parts_per_ghost");
+ restart_write_blocks(&engine_max_sparts_per_ghost, sizeof(int), 1, stream,
+ "engine_max_sparts_per_ghost",
+ "engine_max_sparts_per_ghost");
+ restart_write_blocks(&engine_star_resort_task_depth, sizeof(int), 1, stream,
+ "engine_star_resort_task_depth",
+ "engine_star_resort_task_depth");
/* More things to write. */
if (s->nr_parts > 0) {
@@ -5471,6 +5486,14 @@ void space_struct_restore(struct space *s, FILE *stream) {
"space_extra_sparts");
restart_read_blocks(&space_extra_bparts, sizeof(int), 1, stream, NULL,
"space_extra_bparts");
+ restart_read_blocks(&space_expected_max_nr_strays, sizeof(int), 1, stream,
+ NULL, "space_expected_max_nr_strays");
+ restart_read_blocks(&engine_max_parts_per_ghost, sizeof(int), 1, stream, NULL,
+ "engine_max_parts_per_ghost");
+ restart_read_blocks(&engine_max_sparts_per_ghost, sizeof(int), 1, stream,
+ NULL, "engine_max_sparts_per_ghost");
+ restart_read_blocks(&engine_star_resort_task_depth, sizeof(int), 1, stream,
+ NULL, "engine_star_resort_task_depth");
/* Things that should be reconstructed in a rebuild. */
s->cells_top = NULL;