Commit 75e8417e authored by Peter W. Draper's avatar Peter W. Draper
Browse files

Merge branch 'drift-fixes' into 'master'

Drift all particles before repartitioning

Drifting them afterwards can result in some cells having inconsistent particle lists, so this must be done first. This fixes the crashes I was seeing in MPI runs of the SodShock with 4x12 cores.


See merge request !241
parents 2ec778b8 05cf40a0
...@@ -40,3 +40,4 @@ PointMass: ...@@ -40,3 +40,4 @@ PointMass:
position_y: 50. position_y: 50.
position_z: 50. position_z: 50.
mass: 1e10 # mass of external point mass in internal units mass: 1e10 # mass of external point mass in internal units
timestep_mult: 1e-2
...@@ -1935,14 +1935,14 @@ void engine_maketasks(struct engine *e) { ...@@ -1935,14 +1935,14 @@ void engine_maketasks(struct engine *e) {
scheduler_ranktasks(sched); scheduler_ranktasks(sched);
/* Weight the tasks. */ /* Weight the tasks. */
scheduler_reweight(sched); scheduler_reweight(sched, e->verbose);
/* Set the tasks age. */ /* Set the tasks age. */
e->tasks_age = 0; e->tasks_age = 0;
if (e->verbose) if (e->verbose)
message("took %.3f %s.", clocks_from_ticks(getticks() - tic), message("took %.3f %s (including reweight).",
clocks_getunit()); clocks_from_ticks(getticks() - tic), clocks_getunit());
} }
/** /**
...@@ -2267,8 +2267,11 @@ void engine_rebuild(struct engine *e) { ...@@ -2267,8 +2267,11 @@ void engine_rebuild(struct engine *e) {
* @brief Prepare the #engine by re-building the cells and tasks. * @brief Prepare the #engine by re-building the cells and tasks.
* *
* @param e The #engine to prepare. * @param e The #engine to prepare.
* @param nodrift Whether to drift particles before rebuilding or not. Will
* not be necessary if all particles have already been
* drifted (before repartitioning for instance).
*/ */
void engine_prepare(struct engine *e) { void engine_prepare(struct engine *e, int nodrift) {
TIMER_TIC; TIMER_TIC;
...@@ -2284,32 +2287,32 @@ void engine_prepare(struct engine *e) { ...@@ -2284,32 +2287,32 @@ void engine_prepare(struct engine *e) {
rebuild = buff; rebuild = buff;
#endif #endif
/* Did this not go through? */ /* And rebuild if necessary. */
if (rebuild) { if (rebuild) {
/* First drift all particles to the current time */ /* Drift all particles to the current time if needed. */
e->drift_all = 1; if (!nodrift) {
threadpool_map(&e->threadpool, runner_do_drift_mapper, e->s->cells_top, e->drift_all = 1;
e->s->nr_cells, sizeof(struct cell), 1, e); engine_drift(e);
/* Restore the default drifting policy */ /* Restore the default drifting policy */
e->drift_all = (e->policy & engine_policy_drift_all); e->drift_all = (e->policy & engine_policy_drift_all);
}
/* And now rebuild */
engine_rebuild(e); engine_rebuild(e);
} }
/* Re-rank the tasks every now and then. */ /* Re-rank the tasks every now and then. */
if (e->tasks_age % engine_tasksreweight == 1) { if (e->tasks_age % engine_tasksreweight == 1) {
scheduler_reweight(&e->sched); scheduler_reweight(&e->sched, e->verbose);
} }
e->tasks_age += 1; e->tasks_age += 1;
TIMER_TOC(timer_prepare); TIMER_TOC(timer_prepare);
if (e->verbose) if (e->verbose)
message("took %.3f %s.", clocks_from_ticks(getticks() - tic), message("took %.3f %s (including marktask, rebuild and reweight).",
clocks_getunit()); clocks_from_ticks(getticks() - tic), clocks_getunit());
} }
/** /**
...@@ -2401,6 +2404,7 @@ void engine_collect_kick(struct cell *c) { ...@@ -2401,6 +2404,7 @@ void engine_collect_kick(struct cell *c) {
*/ */
void engine_collect_timestep(struct engine *e) { void engine_collect_timestep(struct engine *e) {
const ticks tic = getticks();
int updates = 0, g_updates = 0; int updates = 0, g_updates = 0;
int ti_end_min = max_nr_timesteps; int ti_end_min = max_nr_timesteps;
const struct space *s = e->s; const struct space *s = e->s;
...@@ -2445,6 +2449,10 @@ void engine_collect_timestep(struct engine *e) { ...@@ -2445,6 +2449,10 @@ void engine_collect_timestep(struct engine *e) {
e->ti_end_min = ti_end_min; e->ti_end_min = ti_end_min;
e->updates = updates; e->updates = updates;
e->g_updates = g_updates; e->g_updates = g_updates;
if (e->verbose)
message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
clocks_getunit());
} }
/** /**
...@@ -2454,6 +2462,7 @@ void engine_collect_timestep(struct engine *e) { ...@@ -2454,6 +2462,7 @@ void engine_collect_timestep(struct engine *e) {
*/ */
void engine_print_stats(struct engine *e) { void engine_print_stats(struct engine *e) {
const ticks tic = getticks();
const struct space *s = e->s; const struct space *s = e->s;
double e_kin = 0.0, e_int = 0.0, e_pot = 0.0, entropy = 0.0, mass = 0.0; double e_kin = 0.0, e_int = 0.0, e_pot = 0.0, entropy = 0.0, mass = 0.0;
...@@ -2520,6 +2529,10 @@ void engine_print_stats(struct engine *e) { ...@@ -2520,6 +2529,10 @@ void engine_print_stats(struct engine *e) {
mom[2], ang_mom[0], ang_mom[1], ang_mom[2]); mom[2], ang_mom[0], ang_mom[1], ang_mom[2]);
fflush(e->file_stats); fflush(e->file_stats);
} }
if (e->verbose)
message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
clocks_getunit());
} }
/** /**
...@@ -2533,6 +2546,8 @@ void engine_print_stats(struct engine *e) { ...@@ -2533,6 +2546,8 @@ void engine_print_stats(struct engine *e) {
void engine_launch(struct engine *e, int nr_runners, unsigned int mask, void engine_launch(struct engine *e, int nr_runners, unsigned int mask,
unsigned int submask) { unsigned int submask) {
const ticks tic = getticks();
/* Prepare the scheduler. */ /* Prepare the scheduler. */
atomic_inc(&e->sched.waiting); atomic_inc(&e->sched.waiting);
...@@ -2557,6 +2572,10 @@ void engine_launch(struct engine *e, int nr_runners, unsigned int mask, ...@@ -2557,6 +2572,10 @@ void engine_launch(struct engine *e, int nr_runners, unsigned int mask,
while (e->barrier_launch || e->barrier_running) while (e->barrier_launch || e->barrier_running)
if (pthread_cond_wait(&e->barrier_cond, &e->barrier_mutex) != 0) if (pthread_cond_wait(&e->barrier_cond, &e->barrier_mutex) != 0)
error("Error while waiting for barrier."); error("Error while waiting for barrier.");
if (e->verbose)
message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
clocks_getunit());
} }
/** /**
...@@ -2576,7 +2595,7 @@ void engine_init_particles(struct engine *e, int flag_entropy_ICs) { ...@@ -2576,7 +2595,7 @@ void engine_init_particles(struct engine *e, int flag_entropy_ICs) {
if (e->nodeID == 0) message("Running initialisation fake time-step."); if (e->nodeID == 0) message("Running initialisation fake time-step.");
engine_prepare(e); engine_prepare(e, 1);
engine_marktasks(e); engine_marktasks(e);
...@@ -2677,8 +2696,7 @@ void engine_step(struct engine *e) { ...@@ -2677,8 +2696,7 @@ void engine_step(struct engine *e) {
/* Drift everybody to the snapshot position */ /* Drift everybody to the snapshot position */
e->drift_all = 1; e->drift_all = 1;
threadpool_map(&e->threadpool, runner_do_drift_mapper, e->s->cells_top, engine_drift(e);
e->s->nr_cells, sizeof(struct cell), 1, e);
/* Restore the default drifting policy */ /* Restore the default drifting policy */
e->drift_all = (e->policy & engine_policy_drift_all); e->drift_all = (e->policy & engine_policy_drift_all);
...@@ -2716,15 +2734,20 @@ void engine_step(struct engine *e) { ...@@ -2716,15 +2734,20 @@ void engine_step(struct engine *e) {
e->timeLastStatistics += e->deltaTimeStatistics; e->timeLastStatistics += e->deltaTimeStatistics;
} }
/* Drift only the necessary particles */ /* Drift only the necessary particles, that all means all particles
threadpool_map(&e->threadpool, runner_do_drift_mapper, e->s->cells_top, * if we are about to repartition. */
e->s->nr_cells, sizeof(struct cell), 1, e); int repart = (e->forcerepart != REPART_NONE);
e->drift_all = repart || e->drift_all;
engine_drift(e);
/* Re-distribute the particles amongst the nodes? */ /* Re-distribute the particles amongst the nodes? */
if (e->forcerepart != REPART_NONE) engine_repartition(e); if (repart) engine_repartition(e);
/* Prepare the space. */ /* Prepare the space. */
engine_prepare(e); engine_prepare(e, e->drift_all);
/* Restore the default drifting policy */
e->drift_all = (e->policy & engine_policy_drift_all);
/* Build the masks corresponding to the policy */ /* Build the masks corresponding to the policy */
unsigned int mask = 0, submask = 0; unsigned int mask = 0, submask = 0;
...@@ -2811,6 +2834,21 @@ int engine_is_done(struct engine *e) { ...@@ -2811,6 +2834,21 @@ int engine_is_done(struct engine *e) {
return !(e->ti_current < max_nr_timesteps); return !(e->ti_current < max_nr_timesteps);
} }
/**
* @brief Drift particles using the current engine drift policy.
*
* @param e The #engine.
*/
void engine_drift(struct engine *e) {
const ticks tic = getticks();
threadpool_map(&e->threadpool, runner_do_drift_mapper, e->s->cells_top,
e->s->nr_cells, sizeof(struct cell), 1, e);
if (e->verbose)
message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
clocks_getunit());
}
/** /**
* @brief Create and fill the proxies. * @brief Create and fill the proxies.
* *
......
...@@ -214,6 +214,7 @@ struct engine { ...@@ -214,6 +214,7 @@ struct engine {
/* Function prototypes. */ /* Function prototypes. */
void engine_barrier(struct engine *e, int tid); void engine_barrier(struct engine *e, int tid);
void engine_compute_next_snapshot_time(struct engine *e); void engine_compute_next_snapshot_time(struct engine *e);
void engine_drift(struct engine *e);
void engine_dump_snapshot(struct engine *e); void engine_dump_snapshot(struct engine *e);
void engine_init(struct engine *e, struct space *s, void engine_init(struct engine *e, struct space *s,
const struct swift_params *params, int nr_nodes, int nodeID, const struct swift_params *params, int nr_nodes, int nodeID,
...@@ -225,7 +226,7 @@ void engine_init(struct engine *e, struct space *s, ...@@ -225,7 +226,7 @@ void engine_init(struct engine *e, struct space *s,
const struct cooling_data *cooling); const struct cooling_data *cooling);
void engine_launch(struct engine *e, int nr_runners, unsigned int mask, void engine_launch(struct engine *e, int nr_runners, unsigned int mask,
unsigned int submask); unsigned int submask);
void engine_prepare(struct engine *e); void engine_prepare(struct engine *e, int nodrift);
void engine_print(struct engine *e); void engine_print(struct engine *e);
void engine_init_particles(struct engine *e, int flag_entropy_ICs); void engine_init_particles(struct engine *e, int flag_entropy_ICs);
void engine_step(struct engine *e); void engine_step(struct engine *e);
......
...@@ -890,10 +890,11 @@ void scheduler_reset(struct scheduler *s, int size) { ...@@ -890,10 +890,11 @@ void scheduler_reset(struct scheduler *s, int size) {
* @brief Compute the task weights * @brief Compute the task weights
* *
* @param s The #scheduler. * @param s The #scheduler.
* @param verbose Are we talkative ?
*/ */
void scheduler_reweight(struct scheduler *s, int verbose) {
void scheduler_reweight(struct scheduler *s) { const ticks tic = getticks();
const int nr_tasks = s->nr_tasks; const int nr_tasks = s->nr_tasks;
int *tid = s->tasks_ind; int *tid = s->tasks_ind;
struct task *tasks = s->tasks; struct task *tasks = s->tasks;
...@@ -902,11 +903,8 @@ void scheduler_reweight(struct scheduler *s) { ...@@ -902,11 +903,8 @@ void scheduler_reweight(struct scheduler *s) {
0.4025, 0.1897, 0.4025, 0.1897, 0.4025, 0.4025, 0.1897, 0.4025, 0.1897, 0.4025,
0.5788, 0.4025, 0.5788}; 0.5788, 0.4025, 0.5788};
const float wscale = 0.001; const float wscale = 0.001;
// ticks tic;
/* Run through the tasks backwards and set their waits and /* Run through the tasks backwards and set their weights. */
weights. */
// tic = getticks();
for (int k = nr_tasks - 1; k >= 0; k--) { for (int k = nr_tasks - 1; k >= 0; k--) {
struct task *t = &tasks[tid[k]]; struct task *t = &tasks[tid[k]];
t->weight = 0; t->weight = 0;
...@@ -963,8 +961,10 @@ void scheduler_reweight(struct scheduler *s) { ...@@ -963,8 +961,10 @@ void scheduler_reweight(struct scheduler *s) {
break; break;
} }
} }
// message( "weighting tasks took %.3f %s." ,
// clocks_from_ticks( getticks() - tic ), clocks_getunit()); if (verbose)
message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
clocks_getunit());
/* int min = tasks[0].weight, max = tasks[0].weight; /* int min = tasks[0].weight, max = tasks[0].weight;
for ( int k = 1 ; k < nr_tasks ; k++ ) for ( int k = 1 ; k < nr_tasks ; k++ )
......
...@@ -116,7 +116,7 @@ void scheduler_start(struct scheduler *s, unsigned int mask, ...@@ -116,7 +116,7 @@ void scheduler_start(struct scheduler *s, unsigned int mask,
unsigned int submask); unsigned int submask);
void scheduler_reset(struct scheduler *s, int nr_tasks); void scheduler_reset(struct scheduler *s, int nr_tasks);
void scheduler_ranktasks(struct scheduler *s); void scheduler_ranktasks(struct scheduler *s);
void scheduler_reweight(struct scheduler *s); void scheduler_reweight(struct scheduler *s, int verbose);
struct task *scheduler_addtask(struct scheduler *s, enum task_types type, struct task *scheduler_addtask(struct scheduler *s, enum task_types type,
enum task_subtypes subtype, int flags, int wait, enum task_subtypes subtype, int flags, int wait,
struct cell *ci, struct cell *cj, int tight); struct cell *ci, struct cell *cj, int tight);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment