Skip to content
GitLab
Commit 8e6278cb authored by Matthieu Schaller's avatar Matthieu Schaller
Browse files

Fixed a moronic bug. Unskipping MPI sends and recvs for gravity should not be... 

Fixed a moronic bug. Unskipping MPI sends and recvs for gravity should not be done *only* for local pairs. Obviously....
parent 42ffbf1d
Hide whitespace changes
Inline Side-by-side
examples/UniformDMBox/uniformBox.yml
View file @ 8e6278cb
......@@ -21,7 +21,7 @@ Scheduler:
Snapshots:
basename: uniformDMBox # Common part of the name of output files
time_first: 0. # Time of the first output (in internal units)
delta_time: 0.01 # Time difference between consecutive outputs (in internal units)
delta_time: 10. # Time difference between consecutive outputs (in internal units)
# Parameters for the self-gravity scheme
Gravity:
......@@ -31,7 +31,7 @@ Gravity:
# Parameters governing the conserved quantities statistics
Statistics:
delta_time: 1e-2 # Time between statistics output
delta_time: 5. # Time between statistics output
# Parameters related to the initial conditions
InitialConditions:
......
src/cell.c
View file @ 8e6278cb
......@@ -1928,6 +1928,9 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) {
const int nodeID = e->nodeID;
int rebuild = 0;
if(c->loc[0] == 0. && c->loc[1] == 0. && c->loc[2] == 0.)
message("Found me! active=%d gcount=%d split=%d", cell_is_active(c, e), c->gcount, c->split);
/* Un-skip the density tasks involved with this cell. */
for (struct link *l = c->density; l != NULL; l = l->next) {
struct task *t = l->t;
......@@ -1936,6 +1939,9 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) {
const int ci_active = cell_is_active(ci, e);
const int cj_active = (cj != NULL) ? cell_is_active(cj, e) : 0;
if(c->loc[0] == 0. && c->loc[1] == 0. && c->loc[2] == 0.)
message("oO");
/* Only activate tasks that involve a local active cell. */
if ((ci_active && ci->nodeID == nodeID) ||
(cj_active && cj->nodeID == nodeID)) {
......@@ -2071,6 +2077,9 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) {
(cj_active && cj->nodeID == engine_rank)) {
scheduler_activate(s, t);
/* if(c->loc[0] == 0. && c->loc[1] == 0. && c->loc[2] == 0.) */
/* message("grav task found!"); */
/* Set the drifting flags */
if (t->type == task_type_self &&
t->subtype == task_subtype_external_grav) {
......@@ -2079,61 +2088,64 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) {
cell_activate_subcell_grav_tasks(t->ci, NULL, s);
} else if (t->type == task_type_pair) {
cell_activate_subcell_grav_tasks(t->ci, t->cj, s);
}
}
#ifdef WITH_MPI
/* Activate the send/recv tasks. */
if (ci->nodeID != engine_rank) {
/* /\* If the local cell is active, receive data from the foreign cell. *\/ */
/* if (cj_active) { */
scheduler_activate(s, ci->recv_grav);
/* } */
/* /\* If the foreign cell is active, we want its ti_end values. *\/ */
/* if (ci_active) */scheduler_activate(s, ci->recv_ti);
/* /\* Is the foreign cell active and will need stuff from us? *\/ */
/* if (ci_active) { */
scheduler_activate_send(s, cj->send_grav, ci->nodeID);
/* /\* Drift the cell which will be sent at the level at which it is */
/* sent, i.e. drift the cell specified in the send task (l->t) */
/* itself. *\/ */
cell_activate_drift_gpart(cj, s);
/* } */
/* /\* If the local cell is active, send its ti_end values. *\/ */
/* if (cj_active) */scheduler_activate_send(s, cj->send_ti, ci->nodeID);
} else if (cj->nodeID != engine_rank) {
/* /\* If the local cell is active, receive data from the foreign cell. *\/ */
/* if (ci_active) { */
scheduler_activate(s, cj->recv_grav);
/* } */
/* /\* If the foreign cell is active, we want its ti_end values. *\/ */
/* if (cj_active) */
scheduler_activate(s, cj->recv_ti);
/* /\* Is the foreign cell active and will need stuff from us? *\/ */
/* if (cj_active) { */
scheduler_activate_send(s, ci->send_grav, cj->nodeID);
/* /\* Drift the cell which will be sent at the level at which it is */
/* sent, i.e. drift the cell specified in the send task (l->t) */
/* itself. *\/ */
cell_activate_drift_gpart(ci, s);
/* } */
if(t->type == task_type_pair) {
/* /\* If the local cell is active, send its ti_end values. *\/ */
/* if (ci_active) */
scheduler_activate_send(s, ci->send_ti, cj->nodeID);
}
#endif
#ifdef WITH_MPI
/* Activate the send/recv tasks. */
if (ci->nodeID != engine_rank) {
/* If the local cell is active, receive data from the foreign cell. */
if (cj_active) {
scheduler_activate(s, ci->recv_grav);
}
/* If the foreign cell is active, we want its ti_end values. */
if (ci_active)scheduler_activate(s, ci->recv_ti);
/* Is the foreign cell active and will need stuff from us? */
if (ci_active) {
scheduler_activate_send(s, cj->send_grav, ci->nodeID);
/* Drift the cell which will be sent at the level at which it is
sent, i.e. drift the cell specified in the send task (l->t)
itself. */
cell_activate_drift_gpart(cj, s);
}
/* If the local cell is active, send its ti_end values. */
if (cj_active)scheduler_activate_send(s, cj->send_ti, ci->nodeID);
} else if (cj->nodeID != engine_rank) {
/* If the local cell is active, receive data from the foreign cell. */
if (ci_active) {
scheduler_activate(s, cj->recv_grav);
}
/* If the foreign cell is active, we want its ti_end values. */
if (cj_active)
scheduler_activate(s, cj->recv_ti);
/* Is the foreign cell active and will need stuff from us? */
if (cj_active) {
scheduler_activate_send(s, ci->send_grav, cj->nodeID);
/* Drift the cell which will be sent at the level at which it is
sent, i.e. drift the cell specified in the send task (l->t)
itself. */
cell_activate_drift_gpart(ci, s);
}
/* If the local cell is active, send its ti_end values. */
if (ci_active)
scheduler_activate_send(s, ci->send_ti, cj->nodeID);
}
#endif
}
}
......@@ -2153,8 +2165,8 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) {
if (c->kick1 != NULL) scheduler_activate(s, c->kick1);
if (c->kick2 != NULL) scheduler_activate(s, c->kick2);
if (c->timestep != NULL) scheduler_activate(s, c->timestep);
if (c->grav_ghost[0] != NULL) scheduler_activate(s, c->grav_ghost[0]);
if (c->grav_ghost[1] != NULL) scheduler_activate(s, c->grav_ghost[1]);
//if (c->grav_ghost[0] != NULL) scheduler_activate(s, c->grav_ghost[0]);
//if (c->grav_ghost[1] != NULL) scheduler_activate(s, c->grav_ghost[1]);
if (c->grav_down != NULL) scheduler_activate(s, c->grav_down);
if (c->grav_long_range != NULL) scheduler_activate(s, c->grav_long_range);
if (c->cooling != NULL) scheduler_activate(s, c->cooling);
......
src/engine.c
View file @ 8e6278cb
......@@ -3262,8 +3262,8 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
}
/* Periodic gravity stuff (Note this is not linked to a cell) ? */
else if (t->type == task_type_grav_top_level ||
t->type == task_type_grav_ghost) {
else if (t->type == task_type_grav_top_level ){ //||
// t->type == task_type_grav_ghost) {
scheduler_activate(s, t);
}
......@@ -3566,11 +3566,13 @@ void engine_prepare(struct engine *e) {
message("Rebuild done");
/* Unskip active tasks and check for rebuild */
engine_unskip(e);
//engine_marktasks(e);
space_print_cells(e->s);
/* Unskip active tasks and check for rebuild */
//engine_unskip(e);
engine_marktasks(e);
// engine_print_task_counts(e);
/* Re-rank the tasks every now and then. */
if (e->tasks_age % engine_tasksreweight == 1) {
......@@ -4216,6 +4218,7 @@ void engine_step(struct engine *e) {
engine_launch(e);
TIMER_TOC(timer_runners);
//error("done");
#ifdef SWIFT_GRAVITY_FORCE_CHECKS
/* Check the accuracy of the gravity calculation */
if (e->policy & engine_policy_self_gravity)
......@@ -4307,6 +4310,8 @@ void engine_unskip(struct engine *e) {
/* Activate all the regular tasks */
threadpool_map(&e->threadpool, runner_do_unskip_mapper, e->s->local_cells_top,
e->s->nr_local_cells, sizeof(int), 1, e);
//threadpool_map(&e->threadpool, runner_do_unskip_mapper, e->s->cells_top,
// e->s->nr_cells, sizeof(int), 1, e);
/* And the top level gravity FFT one */
if (e->s->periodic && (e->policy & engine_policy_self_gravity))
......
src/runner.c
View file @ 8e6278cb
......@@ -840,6 +840,9 @@ void runner_do_ghost(struct runner *r, struct cell *c, int timer) {
*/
static void runner_do_unskip(struct cell *c, struct engine *e) {
if(c->loc[0] == 0. && c->loc[1] == 0. && c->loc[2] == 0.)
message("Found me! active=%d gcount=%d split=%d", cell_is_active(c, e), c->gcount, c->split);
/* Ignore empty cells. */
if (c->count == 0 && c->gcount == 0) return;
......@@ -874,7 +877,7 @@ void runner_do_unskip_mapper(void *map_data, int num_elements,
struct engine *e = (struct engine *)extra_data;
struct space *s = e->s;
int *local_cells = (int *)map_data;
for (int ind = 0; ind < num_elements; ind++) {
struct cell *c = &s->cells_top[local_cells[ind]];
if (c != NULL) runner_do_unskip(c, e);
......
src/runner_doiact_fft.c
View file @ 8e6278cb
......@@ -163,6 +163,8 @@ void runner_do_grav_fft(struct runner* r, int timer) {
#ifdef HAVE_FFTW
message("doing FFT");
const struct engine* e = r->e;
const struct space* s = e->s;
const integertime_t ti_current = e->ti_current;
......
src/space.c
View file @ 8e6278cb
......@@ -3187,7 +3187,7 @@ void space_print_cells(const struct space *s) {
const int cid = cell_getid(s->cdim, i, j, k);
const struct cell *c = &s->cells_top[cid];
fprintf(file, "|(%d-%lld-%d-%d-%d)", c->nodeID, c->ti_end_min, c->gcount, (c->recv_grav!=NULL), (c->send_grav != NULL));
fprintf(file, "|(%d-%lld-%d-%d%d-%d)", c->nodeID, c->ti_end_min, c->gcount, (c->recv_grav!=NULL), (c->recv_grav != NULL ? c->recv_grav->skip==0 : 0), (c->send_grav != NULL));
#endif
}
fprintf(file, "|\n");
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment