diff --git a/examples/EAGLE_25/run.sh b/examples/EAGLE_25/run.sh index af1218f70729663d8efe337c312f6ef2fe8d6620..5961cf01a3a011ee26f0b411e619dd7207d5db47 100755 --- a/examples/EAGLE_25/run.sh +++ b/examples/EAGLE_25/run.sh @@ -7,5 +7,5 @@ then ./getIC.sh fi -../swift --cosmology --hydro --self-gravity --stars--threads=16 eagle_25.yml 2>&1 | tee output.log +../swift --cosmology --hydro --self-gravity --stars --threads=16 eagle_25.yml 2>&1 | tee output.log diff --git a/examples/parameter_example.yml b/examples/parameter_example.yml index 9fdfc993c3f9ad125f4f588db159c6ef5c997c84..2fcc1545958346ed914e1feb52bdb9a829efaa07 100644 --- a/examples/parameter_example.yml +++ b/examples/parameter_example.yml @@ -68,6 +68,7 @@ Scheduler: cell_extra_sparts: 400 # (Optional) Number of spare sparts per top-level allocated at rebuild time for on-the-fly creation. max_top_level_cells: 12 # (Optional) Maximal number of top-level cells in any dimension. The number of top-level cells will be the cube of this (this is the default value). tasks_per_cell: 0 # (Optional) The average number of tasks per cell. If not large enough the simulation will fail (means guess...). + links_per_tasks: 10 # (Optional) The average number of links per tasks (before adding the communication tasks). If not large enough the simulation will fail (means guess...). Defaults to 10. mpi_message_limit: 4096 # (Optional) Maximum MPI task message size to send non-buffered, KB. # Parameters governing the time integration (Set dt_min and dt_max to the same value for a fixed time-step run.) diff --git a/src/engine.c b/src/engine.c index a974b29362c059cf22a69288d6db66a2f0a4b84c..1cc5527751707ce497788d7798fe9144a6c8ee1a 100644 --- a/src/engine.c +++ b/src/engine.c @@ -146,7 +146,9 @@ void engine_addlink(struct engine *e, struct link **l, struct task *t) { /* Get the next free link. */ const size_t ind = atomic_inc(&e->nr_links); if (ind >= e->size_links) { - error("Link table overflow."); + error( + "Link table overflow. Increase the value of " + "`Scheduler:links_per_tasks`."); } struct link *res = &e->links[ind]; @@ -1827,10 +1829,10 @@ void engine_exchange_proxy_multipoles(struct engine *e) { * * @param e The #engine. */ -void engine_print_task_counts(struct engine *e) { +void engine_print_task_counts(const struct engine *e) { const ticks tic = getticks(); - struct scheduler *const sched = &e->sched; + const struct scheduler *sched = &e->sched; const int nr_tasks = sched->nr_tasks; const struct task *const tasks = sched->tasks; @@ -1877,7 +1879,7 @@ void engine_print_task_counts(struct engine *e) { * * @return the estimated total number of tasks */ -int engine_estimate_nr_tasks(struct engine *e) { +int engine_estimate_nr_tasks(const struct engine *e) { int tasks_per_cell = e->tasks_per_cell; if (tasks_per_cell > 0) return e->s->tot_cells * tasks_per_cell; @@ -1886,8 +1888,7 @@ int engine_estimate_nr_tasks(struct engine *e) { * basically use a formula <n1>*ntopcells + <n2>*(totcells - ntopcells). * Where <n1> is the expected maximum tasks per top-level/super cell, and * <n2> the expected maximum tasks for all other cells. These should give - * a safe upper limit. - */ + * a safe upper limit. */ int n1 = 0; int n2 = 0; if (e->policy & engine_policy_hydro) { @@ -4680,6 +4681,10 @@ void engine_config(int restart, struct engine *e, struct swift_params *params, else maxtasks = engine_estimate_nr_tasks(e); + /* Estimated number of links per tasks */ + e->links_per_tasks = + parser_get_opt_param_int(params, "Scheduler:links_per_tasks", 10); + /* Init the scheduler. */ scheduler_init(&e->sched, e->s, maxtasks, nr_queues, (e->policy & scheduler_flag_steal), e->nodeID, &e->threadpool); diff --git a/src/engine.h b/src/engine.h index ed73ff6b0bb4e8ac0eb2a8a5ad63f5e3427e4eeb..757f2b00add656ed2ae7b40ebe431f0487cb9ad2 100644 --- a/src/engine.h +++ b/src/engine.h @@ -100,6 +100,7 @@ enum engine_step_properties { #define engine_parts_size_grow 1.05 #define engine_max_proxy_centre_frac 0.2 #define engine_redistribute_alloc_margin 1.2 +#define engine_rebuild_link_alloc_margin 1.2 #define engine_default_energy_file_name "energy" #define engine_default_timesteps_file_name "timesteps" #define engine_max_parts_per_ghost 1000 @@ -330,6 +331,10 @@ struct engine { * of the various task arrays. */ size_t tasks_per_cell; + /* Average number of links per tasks. This number is used before + the creation of communication tasks so needs to be large enough. */ + size_t links_per_tasks; + /* Are we talkative ? */ int verbose; @@ -440,7 +445,7 @@ int engine_is_done(struct engine *e); void engine_pin(void); void engine_unpin(void); void engine_clean(struct engine *e); -int engine_estimate_nr_tasks(struct engine *e); +int engine_estimate_nr_tasks(const struct engine *e); /* Function prototypes, engine_maketasks.c. */ void engine_maketasks(struct engine *e); diff --git a/src/engine_maketasks.c b/src/engine_maketasks.c index a7bdae77b821719c902863b1da4272e4be1099f2..adcb3e69707691c5218ebc9834f1e8dd0226c80a 100644 --- a/src/engine_maketasks.c +++ b/src/engine_maketasks.c @@ -2276,42 +2276,6 @@ void engine_maketasks(struct engine *e) { if (e->sched.nr_tasks == 0 && (s->nr_gparts > 0 || s->nr_parts > 0)) error("We have particles but no hydro or gravity tasks were created."); - /* Free the old list of cell-task links. */ - if (e->links != NULL) free(e->links); - e->size_links = 0; - -/* The maximum number of links is the - * number of cells (s->tot_cells) times the number of neighbours (26) times - * the number of interaction types, so 26 * 2 (density, force) pairs - * and 2 (density, force) self. - */ -#ifdef EXTRA_HYDRO_LOOP - const size_t hydro_tasks_per_cell = 27 * 3; -#else - const size_t hydro_tasks_per_cell = 27 * 2; -#endif - const size_t self_grav_tasks_per_cell = 125; - const size_t ext_grav_tasks_per_cell = 1; - const size_t stars_tasks_per_cell = 27; - const size_t limiter_tasks_per_cell = 27; - - if (e->policy & engine_policy_hydro) - e->size_links += s->tot_cells * hydro_tasks_per_cell; - if (e->policy & engine_policy_external_gravity) - e->size_links += s->tot_cells * ext_grav_tasks_per_cell; - if (e->policy & engine_policy_self_gravity) - e->size_links += s->tot_cells * self_grav_tasks_per_cell; - if (e->policy & engine_policy_stars) - e->size_links += s->tot_cells * stars_tasks_per_cell; - if (e->policy & engine_policy_limiter) - e->size_links += s->tot_cells * limiter_tasks_per_cell; - - /* Allocate the new link list */ - if ((e->links = (struct link *)malloc(sizeof(struct link) * e->size_links)) == - NULL) - error("Failed to allocate cell-task links."); - e->nr_links = 0; - tic2 = getticks(); /* Split the tasks. */ @@ -2329,6 +2293,20 @@ void engine_maketasks(struct engine *e) { } #endif + /* Free the old list of cell-task links. */ + if (e->links != NULL) free(e->links); + e->size_links = e->sched.nr_tasks * e->links_per_tasks; + + /* Make sure that we have space for more links than last time. */ + if (e->size_links < e->nr_links * engine_rebuild_link_alloc_margin) + e->size_links = e->nr_links * engine_rebuild_link_alloc_margin; + + /* Allocate the new link list */ + if ((e->links = (struct link *)malloc(sizeof(struct link) * e->size_links)) == + NULL) + error("Failed to allocate cell-task links."); + e->nr_links = 0; + tic2 = getticks(); /* Count the number of tasks associated with each cell and @@ -2485,6 +2463,21 @@ void engine_maketasks(struct engine *e) { } #endif + /* Report the number of tasks we actually used */ + if (e->verbose) + message( + "Nr. of tasks: %d allocated tasks: %d ratio: %f memory use: %zd MB.", + e->sched.nr_tasks, e->sched.size, + (float)e->sched.nr_tasks / (float)e->sched.size, + e->sched.size * sizeof(struct task) / (1024 * 1024)); + + /* Report the number of links we actually used */ + if (e->verbose) + message( + "Nr. of links: %zd allocated links: %zd ratio: %f memory use: %zd MB.", + e->nr_links, e->size_links, (float)e->nr_links / (float)e->size_links, + e->size_links * sizeof(struct link) / (1024 * 1024)); + tic2 = getticks(); /* Set the unlocks per task. */ diff --git a/tools/analyse_runtime.py b/tools/analyse_runtime.py index f2f198dfb80d6373e63296b6350fe6768191dd39..9fc6ce8e08bd88ba6d2ba51dca495a9e1cb58352 100755 --- a/tools/analyse_runtime.py +++ b/tools/analyse_runtime.py @@ -61,6 +61,7 @@ labels = [ "engine_recompute_displacement_constraint:", "engine_exchange_top_multipoles:", "updating particle counts", + "engine_estimate_nr_tasks:", "Making gravity tasks", "Making hydro tasks", "Splitting tasks", @@ -121,6 +122,7 @@ is_rebuild = [ 1, 1, 1, + 1, 0, 0, 0,