Commit 7500cfc0 authored by Peter W. Draper's avatar Peter W. Draper
Browse files

Update for tasks estimate when using MPI

Hopefully still quite conservative
parent e4d7516b
......@@ -2869,7 +2869,8 @@ void engine_print_task_counts(struct engine *e) {
else
counts[(int)tasks[k].type] += 1;
}
message("Total = %d (per cell = %d)", nr_tasks, nr_tasks / e->s->tot_cells);
message("Total = %d (per cell = %d)", nr_tasks,
(int)ceil((double)nr_tasks / e->s->tot_cells));
#ifdef WITH_MPI
printf("[%04i] %s engine_print_task_counts: task counts are [ %s=%i",
e->nodeID, clocks_get_timesincestart(), taskID_names[0], counts[0]);
......@@ -2905,68 +2906,101 @@ void engine_print_task_counts(struct engine *e) {
int engine_estimate_nr_tasks(struct engine *e) {
int tasks_per_cell = e->tasks_per_cell;
if (tasks_per_cell <= 0) {
/* Our guess differs depending on the types of tasks we are using, but we
* basically use a formula <n1>*ntopcells + <n2>*(totcells - ntopcells).
* Where <n1> is the expected maximum tasks per top-level/super cell, and
* <n2> the expected maximum tasks for all other cells. These should give
* a safe upper limit.
*/
int n1 = 0;
int n2 = 0;
if (e->policy & engine_policy_hydro) {
n1 += 36;
n2 += 2;
if (tasks_per_cell > 0)
return e->s->tot_cells * tasks_per_cell;
/* Our guess differs depending on the types of tasks we are using, but we
* basically use a formula <n1>*ntopcells + <n2>*(totcells - ntopcells).
* Where <n1> is the expected maximum tasks per top-level/super cell, and
* <n2> the expected maximum tasks for all other cells. These should give
* a safe upper limit.
*/
int n1 = 0;
int n2 = 0;
if (e->policy & engine_policy_hydro) {
n1 += 36;
n2 += 2;
#ifdef WITH_MPI
n1 += 6;
n1 += 6;
#endif
#ifdef EXTRA_HYDRO_LOOP
n1 += 15;
n1 += 15;
#ifdef WITH_MPI
n1 += 2;
n1 += 2;
#endif
#endif
}
if (e->policy & engine_policy_self_gravity) {
n1 += 24;
n2 += 1;
}
if (e->policy & engine_policy_self_gravity) {
n1 += 24;
n2 += 1;
#ifdef WITH_MPI
n2 += 2;
n2 += 2;
#endif
}
if (e->policy & engine_policy_external_gravity) {
n1 += 1;
}
if (e->policy & engine_policy_cosmology) {
n1 += 2;
}
if (e->policy & engine_policy_cooling) {
n1 += 2;
}
if (e->policy & engine_policy_sourceterms) {
n1 += 2;
}
if (e->policy & engine_policy_stars) {
n1 += 2;
}
message("n1 = %d, n2 = %d (%d/%d)", n1, n2, e->s->tot_cells, e->s->nr_cells);
double ntasks = n1 * e->s->nr_cells + n2 * (e->s->tot_cells - e->s->nr_cells);
}
if (e->policy & engine_policy_external_gravity) {
n1 += 2;
}
if (e->policy & engine_policy_cosmology) {
n1 += 2;
}
if (e->policy & engine_policy_cooling) {
n1 += 2;
}
if (e->policy & engine_policy_sourceterms) {
n1 += 2;
}
if (e->policy & engine_policy_stars) {
n1 += 2;
}
/* In principle we need fewer tasks per rank when using MPI, but we could
* have imbalances, so we don't allow for it. */
//if (e->nr_nodes > 1)
// ntasks /= (e->nr_nodes - 1);
#ifdef WITH_MPI
tasks_per_cell = ceil(ntasks / e->s->tot_cells);
if (tasks_per_cell < 1.0) tasks_per_cell = 1.0;
if (e->verbose)
message("tasks per cell estimated as: %d, maximum tasks: %d",
tasks_per_cell, e->s->tot_cells * tasks_per_cell);
/* We need fewer tasks per rank when using MPI, but we could have
* imbalances, so we need to work using the locally active cells, not just
* some equipartition amongst the nodes. Don't want to recurse the whole
* cell tree, so just make a guess of the maximum possible total cells. */
int ntop = 0;
int ncells = 0;
for (int k = 0; k < e->s->nr_cells; k++) {
struct cell *c = &e->s->cells_top[k];
/* Any cells with particles will have tasks (local & foreign). */
int nparts = c->count + c->gcount + c->scount;
if (nparts > 0) {
ntop++;
ncells++;
/* Count cell depth until we get below the parts per cell threshold. */
int depth = 3;
while ( nparts > space_splitsize) {
depth++;
nparts /= 8;
ncells += (1 << depth);
}
}
}
/* If no local cells, we are probably still initialising, so just keep
* room for the top-level. */
if (ncells == 0) {
ntop = e->s->nr_cells;
ncells = ntop;
}
return e->s->tot_cells * tasks_per_cell;
#else
int ntop = e->s->nr_cells;
int ncells = e->s->tot_cells;
#endif
double ntasks = n1 * ntop + n2 * (ncells - ntop);
tasks_per_cell = ceil(ntasks / ncells);
if (tasks_per_cell < 1.0) tasks_per_cell = 1.0;
if (e->verbose)
message("tasks per cell estimated as: %d, maximum tasks: %d",
tasks_per_cell, ncells * tasks_per_cell);
return ncells * tasks_per_cell;
}
/**
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment