From b91db48efe85b0e58998164a144342eda6dad4aa Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Fri, 5 Jul 2019 17:51:36 +0200 Subject: [PATCH 01/14] Skip the stars resort task on the initialisation step. --- src/engine.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/engine.c b/src/engine.c index 5e2ed48cf..242e07ce4 100644 --- a/src/engine.c +++ b/src/engine.c @@ -3327,6 +3327,7 @@ void engine_skip_force_and_kick(struct engine *e) { t->type == task_type_drift_gpart_out || t->type == task_type_cooling || t->type == task_type_stars_in || t->type == task_type_stars_out || t->type == task_type_star_formation || + t->type == task_type_stars_resort || t->type == task_type_extra_ghost || t->type == task_type_bh_swallow_ghost1 || t->type == task_type_bh_swallow_ghost2 || -- GitLab From 8769fc25da65dcdf44039849dbe4b147b94cab15 Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Fri, 5 Jul 2019 17:52:03 +0200 Subject: [PATCH 02/14] Move the stars resort task to the super level from the top-level. --- src/cell.c | 2 +- src/engine_maketasks.c | 11 +++++++---- src/runner.c | 5 ++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/cell.c b/src/cell.c index 0649d31b3..492a84606 100644 --- a/src/cell.c +++ b/src/cell.c @@ -3431,7 +3431,7 @@ int cell_unskip_hydro_tasks(struct cell *c, struct scheduler *s) { if (c->top->hydro.star_formation != NULL) { scheduler_activate(s, c->top->hydro.star_formation); - scheduler_activate(s, c->top->hydro.stars_resort); + scheduler_activate(s, c->super->hydro.stars_resort); cell_activate_drift_spart(c, s); } } diff --git a/src/engine_maketasks.c b/src/engine_maketasks.c index 036c5aef0..49cccb21a 100644 --- a/src/engine_maketasks.c +++ b/src/engine_maketasks.c @@ -775,9 +775,6 @@ void engine_make_hierarchical_tasks_common(struct engine *e, struct cell *c) { if (with_star_formation && c->hydro.count > 0) { c->hydro.star_formation = scheduler_addtask( s, task_type_star_formation, task_subtype_none, 0, 0, c, NULL); - c->hydro.stars_resort = scheduler_addtask( - s, task_type_stars_resort, task_subtype_none, 0, 0, c, NULL); - scheduler_addunlock(s, c->hydro.star_formation, c->hydro.stars_resort); } } @@ -810,6 +807,11 @@ void engine_make_hierarchical_tasks_common(struct engine *e, struct cell *c) { if (with_star_formation && c->hydro.count > 0) { scheduler_addunlock(s, c->kick2, c->top->hydro.star_formation); scheduler_addunlock(s, c->top->hydro.star_formation, c->timestep); + + c->hydro.stars_resort = scheduler_addtask( + s, task_type_stars_resort, task_subtype_none, 0, 0, c, NULL); + scheduler_addunlock(s, c->top->hydro.star_formation, + c->hydro.stars_resort); } /* Time-step limiting */ @@ -1078,7 +1080,8 @@ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c) { scheduler_addunlock(s, c->stars.stars_out, c->super->timestep); if (with_star_formation && c->hydro.count > 0) { - scheduler_addunlock(s, c->top->hydro.stars_resort, c->stars.stars_in); + scheduler_addunlock(s, c->super->hydro.stars_resort, + c->stars.stars_in); } } diff --git a/src/runner.c b/src/runner.c index 05dc33e97..b7d9678dc 100644 --- a/src/runner.c +++ b/src/runner.c @@ -1180,8 +1180,7 @@ void runner_do_star_formation(struct runner *r, struct cell *c, int timer) { /* Did we get a star? (Or did we run out of spare ones?) */ if (sp != NULL) { - /* message("We formed a star id=%lld cellID=%d", sp->id, - * c->cellID); */ + message("We formed a star id=%lld cellID=%d", sp->id, c->cellID); /* Copy the properties of the gas particle to the star particle */ star_formation_copy_properties(p, xp, sp, e, sf_props, cosmo, @@ -1236,7 +1235,7 @@ void runner_do_stars_resort(struct runner *r, struct cell *c, const int timer) { #ifdef SWIFT_DEBUG_CHECKS if (c->nodeID != r->e->nodeID) error("Task must be run locally!"); - if (c->depth != 0) error("Task must be run at the top-level"); + if (c->hydro.super != c) error("Task must be run at the super-level"); #endif TIMER_TIC; -- GitLab From 06ff1631d24fe45a31aabe429e73973c34437536 Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Mon, 8 Jul 2019 17:40:04 +0200 Subject: [PATCH 03/14] Put the star resort task at the hydro-super level not the overall super level. --- src/engine_maketasks.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/engine_maketasks.c b/src/engine_maketasks.c index 49cccb21a..8e095acde 100644 --- a/src/engine_maketasks.c +++ b/src/engine_maketasks.c @@ -807,11 +807,6 @@ void engine_make_hierarchical_tasks_common(struct engine *e, struct cell *c) { if (with_star_formation && c->hydro.count > 0) { scheduler_addunlock(s, c->kick2, c->top->hydro.star_formation); scheduler_addunlock(s, c->top->hydro.star_formation, c->timestep); - - c->hydro.stars_resort = scheduler_addtask( - s, task_type_stars_resort, task_subtype_none, 0, 0, c, NULL); - scheduler_addunlock(s, c->top->hydro.star_formation, - c->hydro.stars_resort); } /* Time-step limiting */ @@ -1080,8 +1075,13 @@ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c) { scheduler_addunlock(s, c->stars.stars_out, c->super->timestep); if (with_star_formation && c->hydro.count > 0) { - scheduler_addunlock(s, c->super->hydro.stars_resort, - c->stars.stars_in); + + c->hydro.stars_resort = scheduler_addtask( + s, task_type_stars_resort, task_subtype_none, 0, 0, c, NULL); + scheduler_addunlock(s, c->top->hydro.star_formation, + c->hydro.stars_resort); + + scheduler_addunlock(s, c->hydro.stars_resort, c->stars.stars_in); } } -- GitLab From 56679915e76eb92bec933f27fddc79db94f24537 Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Tue, 9 Jul 2019 14:05:00 +0100 Subject: [PATCH 04/14] Correctly activate the resort tasks when unskipping. --- src/cell.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/cell.c b/src/cell.c index 492a84606..def787852 100644 --- a/src/cell.c +++ b/src/cell.c @@ -3431,7 +3431,11 @@ int cell_unskip_hydro_tasks(struct cell *c, struct scheduler *s) { if (c->top->hydro.star_formation != NULL) { scheduler_activate(s, c->top->hydro.star_formation); - scheduler_activate(s, c->super->hydro.stars_resort); + } + + if (c->hydro.super != NULL && + c->hydro.super->hydro.stars_resort != NULL) { + scheduler_activate(s, c->hydro.super->hydro.stars_resort); cell_activate_drift_spart(c, s); } } -- GitLab From 07f5e41d1865469f3ab00ba7ee633455a12e5a2a Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Fri, 9 Aug 2019 15:00:41 +0200 Subject: [PATCH 05/14] Code formatting --- src/cell.c | 3 +-- src/engine.c | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/cell.c b/src/cell.c index def787852..006ae37d0 100644 --- a/src/cell.c +++ b/src/cell.c @@ -3433,8 +3433,7 @@ int cell_unskip_hydro_tasks(struct cell *c, struct scheduler *s) { scheduler_activate(s, c->top->hydro.star_formation); } - if (c->hydro.super != NULL && - c->hydro.super->hydro.stars_resort != NULL) { + if (c->hydro.super != NULL && c->hydro.super->hydro.stars_resort != NULL) { scheduler_activate(s, c->hydro.super->hydro.stars_resort); cell_activate_drift_spart(c, s); } diff --git a/src/engine.c b/src/engine.c index 242e07ce4..c2a7bf88a 100644 --- a/src/engine.c +++ b/src/engine.c @@ -3327,8 +3327,7 @@ void engine_skip_force_and_kick(struct engine *e) { t->type == task_type_drift_gpart_out || t->type == task_type_cooling || t->type == task_type_stars_in || t->type == task_type_stars_out || t->type == task_type_star_formation || - t->type == task_type_stars_resort || - t->type == task_type_extra_ghost || + t->type == task_type_stars_resort || t->type == task_type_extra_ghost || t->type == task_type_bh_swallow_ghost1 || t->type == task_type_bh_swallow_ghost2 || t->type == task_type_bh_swallow_ghost3 || -- GitLab From e1c93af984755a5ada8b8bdf51a0147066004237 Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Fri, 9 Aug 2019 16:17:16 +0200 Subject: [PATCH 06/14] Split the star resort task into 8 pieces at a depth of 1 in the tree. --- src/engine_maketasks.c | 45 ++++++++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/src/engine_maketasks.c b/src/engine_maketasks.c index 8e095acde..d376bb43a 100644 --- a/src/engine_maketasks.c +++ b/src/engine_maketasks.c @@ -977,7 +977,8 @@ void engine_add_ghosts(struct engine *e, struct cell *c, struct task *ghost_in, * @param e The #engine. * @param c The #cell. */ -void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c) { +void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c, + struct cell *star_resort_cell) { struct scheduler *s = &e->sched; const int with_stars = (e->policy & engine_policy_stars); @@ -986,6 +987,33 @@ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c) { const int with_star_formation = (e->policy & engine_policy_star_formation); const int with_black_holes = (e->policy & engine_policy_black_holes); + /* Are we at the top-level but this is also the super-level? */ + if (c->top == c && c->hydro.super == c && c->nodeID == e->nodeID) { + + /* Record this is the level where we re-sort */ + star_resort_cell = c; + + if (with_star_formation && c->hydro.count > 0) { + c->hydro.stars_resort = scheduler_addtask( + s, task_type_stars_resort, task_subtype_none, 0, 0, c, NULL); + scheduler_addunlock(s, c->hydro.star_formation, c->hydro.stars_resort); + } + } + + if (c->depth == 1 && c->nodeID == e->nodeID) { + + /* Record this is the level where we re-sort */ + star_resort_cell = c; + + if (with_star_formation && c->hydro.count > 0) { + c->hydro.stars_resort = scheduler_addtask( + s, task_type_stars_resort, task_subtype_none, 0, 0, c, NULL); + + scheduler_addunlock(s, c->top->hydro.star_formation, + c->hydro.stars_resort); + } + } + /* Are we in a super-cell ? */ if (c->hydro.super == c) { @@ -1075,13 +1103,8 @@ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c) { scheduler_addunlock(s, c->stars.stars_out, c->super->timestep); if (with_star_formation && c->hydro.count > 0) { - - c->hydro.stars_resort = scheduler_addtask( - s, task_type_stars_resort, task_subtype_none, 0, 0, c, NULL); - scheduler_addunlock(s, c->top->hydro.star_formation, - c->hydro.stars_resort); - - scheduler_addunlock(s, c->hydro.stars_resort, c->stars.stars_in); + scheduler_addunlock(s, star_resort_cell->hydro.stars_resort, + c->stars.stars_in); } } @@ -1117,7 +1140,8 @@ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c) { if (c->split) for (int k = 0; k < 8; k++) if (c->progeny[k] != NULL) - engine_make_hierarchical_tasks_hydro(e, c->progeny[k]); + engine_make_hierarchical_tasks_hydro(e, c->progeny[k], + star_resort_cell); } } @@ -1134,7 +1158,8 @@ void engine_make_hierarchical_tasks_mapper(void *map_data, int num_elements, /* Make the common tasks (time integration) */ engine_make_hierarchical_tasks_common(e, c); /* Add the hydro stuff */ - if (with_hydro) engine_make_hierarchical_tasks_hydro(e, c); + if (with_hydro) + engine_make_hierarchical_tasks_hydro(e, c, /*star_resort_cell=*/NULL); /* And the gravity stuff */ if (with_self_gravity || with_ext_gravity) engine_make_hierarchical_tasks_gravity(e, c); -- GitLab From a47112ee2ecf2be21a7657ba9d65911e1396baa1 Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Mon, 12 Aug 2019 13:40:00 +0200 Subject: [PATCH 07/14] The star formation task should active the resort flag of its children. --- src/runner.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/runner.c b/src/runner.c index b7d9678dc..48cdce801 100644 --- a/src/runner.c +++ b/src/runner.c @@ -1215,6 +1215,11 @@ void runner_do_star_formation(struct runner *r, struct cell *c, int timer) { if (with_feedback && (c == c->top) && (current_stars_count != c->stars.count)) { cell_set_flag(c, cell_flag_do_stars_resort); + if (c->split) { + for (int k = 0; k < 8; ++k) + if (c->progeny[k] != NULL) + cell_set_flag(c->progeny[k], cell_flag_do_stars_resort); + } cell_clear_stars_sort_flags(c, /*clear_unused_flags=*/0); } @@ -1235,7 +1240,6 @@ void runner_do_stars_resort(struct runner *r, struct cell *c, const int timer) { #ifdef SWIFT_DEBUG_CHECKS if (c->nodeID != r->e->nodeID) error("Task must be run locally!"); - if (c->hydro.super != c) error("Task must be run at the super-level"); #endif TIMER_TIC; -- GitLab From 4916ecb3e2a23a08e2901b3a53079e41c522695c Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Mon, 12 Aug 2019 14:08:17 +0200 Subject: [PATCH 08/14] Add an implicit star-resort task at the top-level to serve as glue. --- src/cell.c | 35 +++++++++++++++++++++++++++++------ src/cell.h | 1 + src/engine_maketasks.c | 19 +++++++++++++------ src/engine_marktasks.c | 5 ++--- 4 files changed, 45 insertions(+), 15 deletions(-) diff --git a/src/cell.c b/src/cell.c index 006ae37d0..e0470fc68 100644 --- a/src/cell.c +++ b/src/cell.c @@ -2397,6 +2397,34 @@ void cell_clear_limiter_flags(struct cell *c, void *data) { cell_flag_do_hydro_limiter | cell_flag_do_hydro_sub_limiter); } +void cell_activate_star_formation_tasks(struct cell *c, struct scheduler *s) { + + if (c->depth != 0) error("Function should be called at the top-level only"); + + /* Have we already unskipped that task? */ + if (c->hydro.star_formation->skip == 0) return; + + /* Activate the star formation task */ + scheduler_activate(s, c->hydro.star_formation); + scheduler_activate(s, c->hydro.stars_resort); + + /* Shallow tree case -> the resort task is at this level */ + if (c->hydro.super == c) { + scheduler_activate(s, c->hydro.stars_resort); + } + + /* Deep tree case -> the resort is in the progenies */ + else { + + for (int k = 0; k < 8; ++k) { + if (c->progeny[k] != NULL) { + message("hello"); + scheduler_activate(s, c->progeny[k]->hydro.stars_resort); + } + } + } +} + /** * @brief Recurse down in a cell hierarchy until the hydro.super level is * reached and activate the spart drift at that level. @@ -3430,12 +3458,7 @@ int cell_unskip_hydro_tasks(struct cell *c, struct scheduler *s) { if (c->logger != NULL) scheduler_activate(s, c->logger); if (c->top->hydro.star_formation != NULL) { - scheduler_activate(s, c->top->hydro.star_formation); - } - - if (c->hydro.super != NULL && c->hydro.super->hydro.stars_resort != NULL) { - scheduler_activate(s, c->hydro.super->hydro.stars_resort); - cell_activate_drift_spart(c, s); + cell_activate_star_formation_tasks(c->top, s); } } diff --git a/src/cell.h b/src/cell.h index 9c9e2e64f..666063f1c 100644 --- a/src/cell.h +++ b/src/cell.h @@ -874,6 +874,7 @@ void cell_drift_multipole(struct cell *c, const struct engine *e); void cell_drift_all_multipoles(struct cell *c, const struct engine *e); void cell_check_timesteps(struct cell *c); void cell_store_pre_drift_values(struct cell *c); +void cell_activate_star_formation_tasks(struct cell *c, struct scheduler *s); void cell_activate_subcell_hydro_tasks(struct cell *ci, struct cell *cj, struct scheduler *s); void cell_activate_subcell_grav_tasks(struct cell *ci, struct cell *cj, diff --git a/src/engine_maketasks.c b/src/engine_maketasks.c index d376bb43a..af3e3b75c 100644 --- a/src/engine_maketasks.c +++ b/src/engine_maketasks.c @@ -988,15 +988,22 @@ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c, const int with_black_holes = (e->policy & engine_policy_black_holes); /* Are we at the top-level but this is also the super-level? */ - if (c->top == c && c->hydro.super == c && c->nodeID == e->nodeID) { + if (c->top == c && c->nodeID == e->nodeID) { /* Record this is the level where we re-sort */ star_resort_cell = c; if (with_star_formation && c->hydro.count > 0) { - c->hydro.stars_resort = scheduler_addtask( - s, task_type_stars_resort, task_subtype_none, 0, 0, c, NULL); - scheduler_addunlock(s, c->hydro.star_formation, c->hydro.stars_resort); + + if (c->hydro.super == c) { + + c->hydro.stars_resort = scheduler_addtask( + s, task_type_stars_resort, task_subtype_none, 0, 0, c, NULL); + scheduler_addunlock(s, c->hydro.star_formation, c->hydro.stars_resort); + } else { + c->hydro.stars_resort = scheduler_addtask( + s, task_type_stars_resort, task_subtype_none, 0, 1, c, NULL); + } } } @@ -1011,6 +1018,7 @@ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c, scheduler_addunlock(s, c->top->hydro.star_formation, c->hydro.stars_resort); + scheduler_addunlock(s, c->hydro.stars_resort, c->top->hydro.stars_resort); } } @@ -1103,8 +1111,7 @@ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c, scheduler_addunlock(s, c->stars.stars_out, c->super->timestep); if (with_star_formation && c->hydro.count > 0) { - scheduler_addunlock(s, star_resort_cell->hydro.stars_resort, - c->stars.stars_in); + scheduler_addunlock(s, c->top->hydro.stars_resort, c->stars.stars_in); } } diff --git a/src/engine_marktasks.c b/src/engine_marktasks.c index dcf1be29a..7f4e2d4c4 100644 --- a/src/engine_marktasks.c +++ b/src/engine_marktasks.c @@ -939,10 +939,9 @@ void engine_marktasks_mapper(void *map_data, int num_elements, } /* Subgrid tasks: star formation */ - else if (t_type == task_type_star_formation || - t_type == task_type_stars_resort) { + else if (t_type == task_type_star_formation) { if (cell_is_active_hydro(t->ci, e)) { - scheduler_activate(s, t); + cell_activate_star_formation_tasks(t->ci, s); cell_activate_super_spart_drifts(t->ci, s); } } -- GitLab From 18eab64f7c2da53436ff97d738f42ed62f806753 Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Mon, 12 Aug 2019 14:56:06 +0200 Subject: [PATCH 09/14] No need for the top-level implicit resort task. --- src/cell.c | 2 -- src/engine_maketasks.c | 7 ++----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/cell.c b/src/cell.c index e0470fc68..2d4ae2551 100644 --- a/src/cell.c +++ b/src/cell.c @@ -2406,7 +2406,6 @@ void cell_activate_star_formation_tasks(struct cell *c, struct scheduler *s) { /* Activate the star formation task */ scheduler_activate(s, c->hydro.star_formation); - scheduler_activate(s, c->hydro.stars_resort); /* Shallow tree case -> the resort task is at this level */ if (c->hydro.super == c) { @@ -2418,7 +2417,6 @@ void cell_activate_star_formation_tasks(struct cell *c, struct scheduler *s) { for (int k = 0; k < 8; ++k) { if (c->progeny[k] != NULL) { - message("hello"); scheduler_activate(s, c->progeny[k]->hydro.stars_resort); } } diff --git a/src/engine_maketasks.c b/src/engine_maketasks.c index af3e3b75c..a3d6ad042 100644 --- a/src/engine_maketasks.c +++ b/src/engine_maketasks.c @@ -1000,9 +1000,6 @@ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c, c->hydro.stars_resort = scheduler_addtask( s, task_type_stars_resort, task_subtype_none, 0, 0, c, NULL); scheduler_addunlock(s, c->hydro.star_formation, c->hydro.stars_resort); - } else { - c->hydro.stars_resort = scheduler_addtask( - s, task_type_stars_resort, task_subtype_none, 0, 1, c, NULL); } } } @@ -1018,7 +1015,6 @@ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c, scheduler_addunlock(s, c->top->hydro.star_formation, c->hydro.stars_resort); - scheduler_addunlock(s, c->hydro.stars_resort, c->top->hydro.stars_resort); } } @@ -1111,7 +1107,8 @@ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c, scheduler_addunlock(s, c->stars.stars_out, c->super->timestep); if (with_star_formation && c->hydro.count > 0) { - scheduler_addunlock(s, c->top->hydro.stars_resort, c->stars.stars_in); + scheduler_addunlock(s, star_resort_cell->hydro.stars_resort, + c->stars.stars_in); } } -- GitLab From 6253661d7e9e0ecb0843fe54bc7ed930f714efd7 Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Wed, 14 Aug 2019 12:05:06 +0200 Subject: [PATCH 10/14] Make the depth of the resort task a user-defined parameter. --- src/cell.c | 31 +++++++++++++++++-------------- src/engine.h | 1 + src/engine_maketasks.c | 27 ++++++++------------------- src/space.c | 1 + 4 files changed, 27 insertions(+), 33 deletions(-) diff --git a/src/cell.c b/src/cell.c index 2d4ae2551..de1f6f87e 100644 --- a/src/cell.c +++ b/src/cell.c @@ -70,6 +70,8 @@ #include "tools.h" #include "tracers.h" +extern int engine_star_resort_task_depth; + /* Global variables. */ int cell_next_tag = 0; @@ -2397,6 +2399,19 @@ void cell_clear_limiter_flags(struct cell *c, void *data) { cell_flag_do_hydro_limiter | cell_flag_do_hydro_sub_limiter); } +void cell_activate_star_resort_tasks(struct cell *c, struct scheduler *s) { + + if (c->depth == engine_star_resort_task_depth || c->hydro.super == c) { + scheduler_activate(s, c->hydro.stars_resort); + } else { + for (int k = 0; k < 8; ++k) { + if (c->progeny[k] != NULL) { + cell_activate_star_resort_tasks(c->progeny[k], s); + } + } + } +} + void cell_activate_star_formation_tasks(struct cell *c, struct scheduler *s) { if (c->depth != 0) error("Function should be called at the top-level only"); @@ -2407,20 +2422,8 @@ void cell_activate_star_formation_tasks(struct cell *c, struct scheduler *s) { /* Activate the star formation task */ scheduler_activate(s, c->hydro.star_formation); - /* Shallow tree case -> the resort task is at this level */ - if (c->hydro.super == c) { - scheduler_activate(s, c->hydro.stars_resort); - } - - /* Deep tree case -> the resort is in the progenies */ - else { - - for (int k = 0; k < 8; ++k) { - if (c->progeny[k] != NULL) { - scheduler_activate(s, c->progeny[k]->hydro.stars_resort); - } - } - } + /* Activate the star resort tasks at whatever level they are */ + cell_activate_star_resort_tasks(c, s); } /** diff --git a/src/engine.h b/src/engine.h index 967a43087..dc5c2c9dd 100644 --- a/src/engine.h +++ b/src/engine.h @@ -113,6 +113,7 @@ enum engine_step_properties { #define engine_default_timesteps_file_name "timesteps" #define engine_max_parts_per_ghost_default 1000 #define engine_max_sparts_per_ghost_default 1000 +#define engine_star_resort_task_depth_default 2 #define engine_tasks_per_cell_margin 1.2 /** diff --git a/src/engine_maketasks.c b/src/engine_maketasks.c index a3d6ad042..3759ac26f 100644 --- a/src/engine_maketasks.c +++ b/src/engine_maketasks.c @@ -54,6 +54,7 @@ extern int engine_max_parts_per_ghost; extern int engine_max_sparts_per_ghost; +extern int engine_star_resort_task_depth; /** * @brief Add send tasks for the gravity pairs to a hierarchy of cells. @@ -987,29 +988,17 @@ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c, const int with_star_formation = (e->policy & engine_policy_star_formation); const int with_black_holes = (e->policy & engine_policy_black_holes); - /* Are we at the top-level but this is also the super-level? */ - if (c->top == c && c->nodeID == e->nodeID) { - - /* Record this is the level where we re-sort */ - star_resort_cell = c; + /* Are we are the level where we create the stars' resort tasks? + * If the tree is shallow, we need to do this at the super-level if the + * super-level is above the level we want */ + if ((c->nodeID == e->nodeID) && (star_resort_cell == NULL) && + (c->depth == engine_star_resort_task_depth || c->hydro.super == c)) { if (with_star_formation && c->hydro.count > 0) { - if (c->hydro.super == c) { + /* Record this is the level where we re-sort */ + star_resort_cell = c; - c->hydro.stars_resort = scheduler_addtask( - s, task_type_stars_resort, task_subtype_none, 0, 0, c, NULL); - scheduler_addunlock(s, c->hydro.star_formation, c->hydro.stars_resort); - } - } - } - - if (c->depth == 1 && c->nodeID == e->nodeID) { - - /* Record this is the level where we re-sort */ - star_resort_cell = c; - - if (with_star_formation && c->hydro.count > 0) { c->hydro.stars_resort = scheduler_addtask( s, task_type_stars_resort, task_subtype_none, 0, 0, c, NULL); diff --git a/src/space.c b/src/space.c index 250af5efa..40dc87fdc 100644 --- a/src/space.c +++ b/src/space.c @@ -91,6 +91,7 @@ int space_extra_gparts = space_extra_gparts_default; /*! Maximum number of particles per ghost */ int engine_max_parts_per_ghost = engine_max_parts_per_ghost_default; int engine_max_sparts_per_ghost = engine_max_sparts_per_ghost_default; +int engine_star_resort_task_depth = engine_star_resort_task_depth_default; /*! Expected maximal number of strays received at a rebuild */ int space_expected_max_nr_strays = space_expected_max_nr_strays_default; -- GitLab From 86b5b804041cde2951f5e14f41a00f891127ef14 Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Wed, 14 Aug 2019 13:44:45 +0200 Subject: [PATCH 11/14] Reset the star resort flag at the right level in the tree. --- src/cell.c | 13 +++++++++++++ src/cell.h | 1 + src/runner.c | 7 +------ 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/cell.c b/src/cell.c index de1f6f87e..b03877550 100644 --- a/src/cell.c +++ b/src/cell.c @@ -2399,6 +2399,19 @@ void cell_clear_limiter_flags(struct cell *c, void *data) { cell_flag_do_hydro_limiter | cell_flag_do_hydro_sub_limiter); } +void cell_set_star_resort_flag(struct cell *c) { + + cell_set_flag(c, cell_flag_do_stars_resort); + + /* Abort if we reched the level where the resorting task lives */ + if (c->depth == engine_star_resort_task_depth || c->hydro.super == c) return; + + if (c->split) { + for (int k = 0; k < 8; ++k) + if (c->progeny[k] != NULL) cell_set_star_resort_flag(c->progeny[k]); + } +} + void cell_activate_star_resort_tasks(struct cell *c, struct scheduler *s) { if (c->depth == engine_star_resort_task_depth || c->hydro.super == c) { diff --git a/src/cell.h b/src/cell.h index 666063f1c..f59576b4f 100644 --- a/src/cell.h +++ b/src/cell.h @@ -874,6 +874,7 @@ void cell_drift_multipole(struct cell *c, const struct engine *e); void cell_drift_all_multipoles(struct cell *c, const struct engine *e); void cell_check_timesteps(struct cell *c); void cell_store_pre_drift_values(struct cell *c); +void cell_set_star_resort_flag(struct cell *c); void cell_activate_star_formation_tasks(struct cell *c, struct scheduler *s); void cell_activate_subcell_hydro_tasks(struct cell *ci, struct cell *cj, struct scheduler *s); diff --git a/src/runner.c b/src/runner.c index 48cdce801..1999d57cb 100644 --- a/src/runner.c +++ b/src/runner.c @@ -1214,12 +1214,7 @@ void runner_do_star_formation(struct runner *r, struct cell *c, int timer) { * re-compute them. */ if (with_feedback && (c == c->top) && (current_stars_count != c->stars.count)) { - cell_set_flag(c, cell_flag_do_stars_resort); - if (c->split) { - for (int k = 0; k < 8; ++k) - if (c->progeny[k] != NULL) - cell_set_flag(c->progeny[k], cell_flag_do_stars_resort); - } + cell_set_star_resort_flag(c); cell_clear_stars_sort_flags(c, /*clear_unused_flags=*/0); } -- GitLab From 23d220817ebd30073617e1cdc62a967d515a339e Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Wed, 14 Aug 2019 23:25:04 +0200 Subject: [PATCH 12/14] Only cancel the stars sorted flag in the cells that have received a new particle. --- src/cell.c | 11 +++++++++-- src/runner.c | 1 - 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/cell.c b/src/cell.c index b03877550..2637a9703 100644 --- a/src/cell.c +++ b/src/cell.c @@ -5202,10 +5202,17 @@ void cell_recursively_shift_sparts(struct cell *c, /* When directly above the leaf with the new particle: increase the particle * count */ /* When after the leaf with the new particle: shift by one position */ - if (main_branch) + if (main_branch) { c->stars.count++; - else + + /* Indicate that the cell is not sorted and cancel the pointer sorting + * arrays. */ + c->stars.sorted = 0; + cell_free_stars_sorts(c); + + } else { c->stars.parts++; + } } /** diff --git a/src/runner.c b/src/runner.c index 1999d57cb..cfa33ca84 100644 --- a/src/runner.c +++ b/src/runner.c @@ -1215,7 +1215,6 @@ void runner_do_star_formation(struct runner *r, struct cell *c, int timer) { if (with_feedback && (c == c->top) && (current_stars_count != c->stars.count)) { cell_set_star_resort_flag(c); - cell_clear_stars_sort_flags(c, /*clear_unused_flags=*/0); } if (timer) TIMER_TOC(timer_do_star_formation); -- GitLab From 9bf02d19bac971b1bce3bb371f6c3448d4a449f7 Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Thu, 15 Aug 2019 11:39:18 +0200 Subject: [PATCH 13/14] Documentation and debugging checks fixes. --- src/cell.c | 31 +++++++++++++++++++++++++++++++ src/engine_maketasks.c | 2 ++ src/runner.c | 3 ++- 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/cell.c b/src/cell.c index 2637a9703..a2ce43c9b 100644 --- a/src/cell.c +++ b/src/cell.c @@ -2399,6 +2399,11 @@ void cell_clear_limiter_flags(struct cell *c, void *data) { cell_flag_do_hydro_limiter | cell_flag_do_hydro_sub_limiter); } +/** + * @brief Recursively clear the stars_resort flag in a cell hierarchy. + * + * @param c The #cell to act on. + */ void cell_set_star_resort_flag(struct cell *c) { cell_set_flag(c, cell_flag_do_stars_resort); @@ -2412,8 +2417,24 @@ void cell_set_star_resort_flag(struct cell *c) { } } +/** + * @brief Recurses in a cell hierarchy down to the level where the + * star resort tasks are and activates them. + * + * The function will fail if called *below* the super-level + * + * @param c The #cell to recurse into. + * @param s The #scheduler. + */ void cell_activate_star_resort_tasks(struct cell *c, struct scheduler *s) { +#ifdef SWIFT_DEBUG_CHECKS + if (c->hydro.super != NULL && c->hydro.super != c) + error("Function called below the super level!"); +#endif + + /* The resort tasks are at either the chosen depth or the super level, + * whichever comes first. */ if (c->depth == engine_star_resort_task_depth || c->hydro.super == c) { scheduler_activate(s, c->hydro.stars_resort); } else { @@ -2425,9 +2446,19 @@ void cell_activate_star_resort_tasks(struct cell *c, struct scheduler *s) { } } +/** + * @brief Activate the star formation task as well as the resorting of stars + * + * Must be called at the top-level in the tree (where the SF task is...) + * + * @param c The (top-level) #cell. + * @param s The #scheduler. + */ void cell_activate_star_formation_tasks(struct cell *c, struct scheduler *s) { +#ifdef SWIFT_DEBUG_CHECKS if (c->depth != 0) error("Function should be called at the top-level only"); +#endif /* Have we already unskipped that task? */ if (c->hydro.star_formation->skip == 0) return; diff --git a/src/engine_maketasks.c b/src/engine_maketasks.c index 3759ac26f..6a8341068 100644 --- a/src/engine_maketasks.c +++ b/src/engine_maketasks.c @@ -977,6 +977,8 @@ void engine_add_ghosts(struct engine *e, struct cell *c, struct task *ghost_in, * * @param e The #engine. * @param c The #cell. + * @param star_resort_cell Pointer to the cell where the star_resort task has + * been created. NULL above that level or if not running with star formation. */ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c, struct cell *star_resort_cell) { diff --git a/src/runner.c b/src/runner.c index cfa33ca84..18e398542 100644 --- a/src/runner.c +++ b/src/runner.c @@ -1180,7 +1180,8 @@ void runner_do_star_formation(struct runner *r, struct cell *c, int timer) { /* Did we get a star? (Or did we run out of spare ones?) */ if (sp != NULL) { - message("We formed a star id=%lld cellID=%d", sp->id, c->cellID); + /* message("We formed a star id=%lld cellID=%d", sp->id, + * c->cellID); */ /* Copy the properties of the gas particle to the star particle */ star_formation_copy_properties(p, xp, sp, e, sf_props, cosmo, -- GitLab From 9b83d540485abfa2eab2b2b6127ae180fe1355e7 Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Thu, 15 Aug 2019 12:00:03 +0200 Subject: [PATCH 14/14] Add the new global variable and some missing ones to the list of things written to and read from check-point files. --- src/space.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/space.c b/src/space.c index 40dc87fdc..d22eba11d 100644 --- a/src/space.c +++ b/src/space.c @@ -91,6 +91,8 @@ int space_extra_gparts = space_extra_gparts_default; /*! Maximum number of particles per ghost */ int engine_max_parts_per_ghost = engine_max_parts_per_ghost_default; int engine_max_sparts_per_ghost = engine_max_sparts_per_ghost_default; + +/*! Maximal depth at which the stars resort task can be pushed */ int engine_star_resort_task_depth = engine_star_resort_task_depth_default; /*! Expected maximal number of strays received at a rebuild */ @@ -5415,6 +5417,18 @@ void space_struct_dump(struct space *s, FILE *stream) { "space_extra_sparts", "space_extra_sparts"); restart_write_blocks(&space_extra_bparts, sizeof(int), 1, stream, "space_extra_bparts", "space_extra_bparts"); + restart_write_blocks(&space_expected_max_nr_strays, sizeof(int), 1, stream, + "space_expected_max_nr_strays", + "space_expected_max_nr_strays"); + restart_write_blocks(&engine_max_parts_per_ghost, sizeof(int), 1, stream, + "engine_max_parts_per_ghost", + "engine_max_parts_per_ghost"); + restart_write_blocks(&engine_max_sparts_per_ghost, sizeof(int), 1, stream, + "engine_max_sparts_per_ghost", + "engine_max_sparts_per_ghost"); + restart_write_blocks(&engine_star_resort_task_depth, sizeof(int), 1, stream, + "engine_star_resort_task_depth", + "engine_star_resort_task_depth"); /* More things to write. */ if (s->nr_parts > 0) { @@ -5473,6 +5487,14 @@ void space_struct_restore(struct space *s, FILE *stream) { "space_extra_sparts"); restart_read_blocks(&space_extra_bparts, sizeof(int), 1, stream, NULL, "space_extra_bparts"); + restart_read_blocks(&space_expected_max_nr_strays, sizeof(int), 1, stream, + NULL, "space_expected_max_nr_strays"); + restart_read_blocks(&engine_max_parts_per_ghost, sizeof(int), 1, stream, NULL, + "engine_max_parts_per_ghost"); + restart_read_blocks(&engine_max_sparts_per_ghost, sizeof(int), 1, stream, + NULL, "engine_max_sparts_per_ghost"); + restart_read_blocks(&engine_star_resort_task_depth, sizeof(int), 1, stream, + NULL, "engine_star_resort_task_depth"); /* Things that should be reconstructed in a rebuild. */ s->cells_top = NULL; -- GitLab