From a6aaf48ca5327df495024038ef17cfb5dad80e3c Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Mon, 5 Dec 2016 18:39:12 +0000 Subject: [PATCH 01/25] Created the task and dependencies --- src/cell.c | 7 +++++++ src/cell.h | 3 +++ src/engine.c | 7 +++++++ src/runner.c | 22 +++++++++++++++------- src/scheduler.c | 37 ++++++++++++++++++++++++++++++++----- src/task.c | 3 ++- src/task.h | 1 + 7 files changed, 67 insertions(+), 13 deletions(-) diff --git a/src/cell.c b/src/cell.c index e2767cdaa..fcbbfe4ae 100644 --- a/src/cell.c +++ b/src/cell.c @@ -878,6 +878,12 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) { } } + /* Activate the drift on both sides */ + if (t->type == task_type_pair || t->type == task_type_sub_pair) { + scheduler_activate(s, ci->drift); + scheduler_activate(s, cj->drift); + } + /* Check whether there was too much particle motion */ if (t->type == task_type_pair || t->type == task_type_sub_pair) { if (t->tight && @@ -956,6 +962,7 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) { if (c->ghost != NULL) scheduler_activate(s, c->ghost); if (c->init != NULL) scheduler_activate(s, c->init); if (c->kick != NULL) scheduler_activate(s, c->kick); + if (c->drift != NULL) scheduler_activate(s, c->drift); if (c->cooling != NULL) scheduler_activate(s, c->cooling); if (c->sourceterms != NULL) scheduler_activate(s, c->sourceterms); diff --git a/src/cell.h b/src/cell.h index 2cd13cf2a..d978de71f 100644 --- a/src/cell.h +++ b/src/cell.h @@ -147,6 +147,9 @@ struct cell { /*! The extra ghost task for complex hydro schemes */ struct task *extra_ghost; + /*! The drift task */ + struct task *drift; + /*! The kick task */ struct task *kick; diff --git a/src/engine.c b/src/engine.c index e989aefd5..111f0a230 100644 --- a/src/engine.c +++ b/src/engine.c @@ -139,6 +139,9 @@ void engine_make_hierarchical_tasks(struct engine *e, struct cell *c) { c->init = scheduler_addtask(s, task_type_init, task_subtype_none, 0, 0, c, NULL, 0); + /* c->drift = scheduler_addtask(s, task_type_drift, task_subtype_none, 0, 0, */ + /* c, NULL, 0); */ + c->kick = scheduler_addtask(s, task_type_kick, task_subtype_none, 0, 0, c, NULL, 0); @@ -1990,6 +1993,10 @@ void engine_marktasks_mapper(void *map_data, int num_elements, const struct cell *ci = t->ci; const struct cell *cj = t->cj; + /* Activate the drift on both sides */ + if(ci->drift) scheduler_activate(s, ci->drift); + if(ci->drift) scheduler_activate(s, cj->drift); + /* Too much particle movement? */ if (t->tight && (max(ci->h_max, cj->h_max) + ci->dx_max + cj->dx_max > cj->dmin || diff --git a/src/runner.c b/src/runner.c index 2d6da4e4a..a62fb87a8 100644 --- a/src/runner.c +++ b/src/runner.c @@ -752,7 +752,7 @@ void runner_do_ghost(struct runner *r, struct cell *c, int timer) { * @param drift whether to actually drift the particles, will not be * necessary for non-local cells. */ -static void runner_do_drift(struct cell *c, struct engine *e, int drift) { +static void runner_do_unskip(struct cell *c, struct engine *e, int drift) { /* Unskip any active tasks. */ if (cell_is_active(c, e)) { @@ -770,7 +770,7 @@ static void runner_do_drift(struct cell *c, struct engine *e, int drift) { for (int k = 0; k < 8; k++) { if (c->progeny[k] != NULL) { struct cell *cp = c->progeny[k]; - runner_do_drift(cp, e, 0); + runner_do_unskip(cp, e, 0); } } } @@ -778,7 +778,6 @@ static void runner_do_drift(struct cell *c, struct engine *e, int drift) { } /* Now, we can drift */ - /* Get some information first */ const double timeBase = e->timeBase; const int ti_old = c->ti_old; @@ -856,7 +855,7 @@ static void runner_do_drift(struct cell *c, struct engine *e, int drift) { struct cell *cp = c->progeny[k]; /* Recurse. */ - runner_do_drift(cp, e, drift); + runner_do_unskip(cp, e, drift); dx_max = max(dx_max, cp->dx_max); h_max = max(h_max, cp->h_max); } @@ -887,13 +886,15 @@ void runner_do_drift_mapper(void *map_data, int num_elements, for (int ind = 0; ind < num_elements; ind++) { struct cell *c = &cells[ind]; #ifdef WITH_MPI - if (c != NULL) runner_do_drift(c, e, (c->nodeID == e->nodeID)); + if (c != NULL) runner_do_unskip(c, e, (c->nodeID == e->nodeID)); #else - if (c != NULL) runner_do_drift(c, e, 1); + if (c != NULL) runner_do_unskip(c, e, 1); #endif } } +void runner_do_drift(struct runner *r, struct cell *c, int timer) {} + /** * @brief Kick particles in momentum space and collect statistics (floating * time-step case) @@ -1126,7 +1127,7 @@ void *runner_main(void *data) { /* Check that we haven't scheduled an inactive task */ #ifdef SWIFT_DEBUG_CHECKS if (cj == NULL) { /* self */ - if (!cell_is_active(ci, e) && t->type != task_type_sort) + if (!cell_is_active(ci, e) && t->type != task_type_sort && t->type != task_type_drift) error( "Task (type='%s/%s') should have been skipped ti_current=%d " "c->ti_end_min=%d", @@ -1142,6 +1143,10 @@ void *runner_main(void *data) { taskID_names[t->type], subtaskID_names[t->subtype], e->ti_current, ci->ti_end_min, t->flags); + /* Special treatement for drifts */ + if (!cell_is_active(ci, e) && t->type == task_type_drift) + {;} + } else { /* pair */ if (!cell_is_active(ci, e) && !cell_is_active(cj, e)) error( @@ -1231,6 +1236,9 @@ void *runner_main(void *data) { runner_do_extra_ghost(r, ci, 1); break; #endif + case task_type_drift: + // runner_do_drift(r, ci, 1); + break; case task_type_kick: runner_do_kick(r, ci, 1); break; diff --git a/src/scheduler.c b/src/scheduler.c index 0d7c8c475..4574d4a1d 100644 --- a/src/scheduler.c +++ b/src/scheduler.c @@ -133,6 +133,7 @@ static void scheduler_splittask(struct task *t, struct scheduler *s) { /* Non-splittable task? */ if ((t->ci == NULL || (t->type == task_type_pair && t->cj == NULL)) || ((t->type == task_type_kick) && t->ci->nodeID != s->nodeID) || + ((t->type == task_type_drift) && t->ci->nodeID != s->nodeID) || ((t->type == task_type_init) && t->ci->nodeID != s->nodeID)) { t->type = task_type_none; t->skip = 1; @@ -603,7 +604,7 @@ static void scheduler_splittask(struct task *t, struct scheduler *s) { tl->flags = space_getsid(s->space, &t->ci, &t->cj, shift); } - /* Otherwise, if not spilt, stitch-up the sorting. */ + /* Otherwise, if not spilt, stitch-up the sorting and drift. */ } else { /* Create the sort for ci. */ @@ -613,9 +614,19 @@ static void scheduler_splittask(struct task *t, struct scheduler *s) { 1 << sid, 0, ci, NULL, 0); else ci->sorts->flags |= (1 << sid); + + scheduler_addunlock(s, ci->sorts, t); + + /* Create the drift for ci. */ + if (ci->drift == NULL) { + ci->drift = scheduler_addtask(s, task_type_drift, task_subtype_none, + 0, 0, ci, NULL, 0); + scheduler_addunlock(s, ci->drift, ci->sorts); + } lock_unlock_blind(&ci->lock); - scheduler_addunlock(s, ci->sorts, t); + + /* Create the sort for cj. */ lock_lock(&cj->lock); if (cj->sorts == NULL) @@ -623,8 +634,16 @@ static void scheduler_splittask(struct task *t, struct scheduler *s) { 1 << sid, 0, cj, NULL, 0); else cj->sorts->flags |= (1 << sid); - lock_unlock_blind(&cj->lock); + scheduler_addunlock(s, cj->sorts, t); + + /* Create the drift for cj. */ + if (cj->drift == NULL) { + cj->drift = scheduler_addtask(s, task_type_drift, task_subtype_none, + 0, 0, cj, NULL, 0); + scheduler_addunlock(s, cj->drift, cj->sorts); + } + lock_unlock_blind(&cj->lock); } } /* pair interaction? */ @@ -782,7 +801,9 @@ void scheduler_set_unlocks(struct scheduler *s) { for (int i = 0; i < t->nr_unlock_tasks; i++) { for (int j = i + 1; j < t->nr_unlock_tasks; j++) { if (t->unlock_tasks[i] == t->unlock_tasks[j]) - error("duplicate unlock!"); + error("duplicate unlock! t->type=%s/%s unlocking type=%s/%s", + taskID_names[t->type], subtaskID_names[t->subtype], + taskID_names[t->unlock_tasks[i]->type], subtaskID_names[t->unlock_tasks[i]->subtype]); } } } @@ -1065,7 +1086,7 @@ void scheduler_start(struct scheduler *s) { if (cj == NULL) { /* self */ if (ci->ti_end_min == ti_current && t->skip && - t->type != task_type_sort) + t->type != task_type_sort && t->type != task_type_drift) error( "Task (type='%s/%s') should not have been skipped ti_current=%d " "c->ti_end_min=%d", @@ -1081,6 +1102,11 @@ void scheduler_start(struct scheduler *s) { taskID_names[t->type], subtaskID_names[t->subtype], ti_current, ci->ti_end_min, t->flags); + /* Special treatement for drifts */ + if (ci->ti_end_min == ti_current && t->skip && + t->type == task_type_drift) + {;} + } else { /* pair */ if ((ci->ti_end_min == ti_current || cj->ti_end_min == ti_current) && @@ -1148,6 +1174,7 @@ void scheduler_enqueue(struct scheduler *s, struct task *t) { case task_type_sort: case task_type_ghost: case task_type_kick: + case task_type_drift: case task_type_init: qid = t->ci->super->owner; break; diff --git a/src/task.c b/src/task.c index ea97fdd1b..19a895915 100644 --- a/src/task.c +++ b/src/task.c @@ -49,7 +49,7 @@ /* Task type names. */ const char *taskID_names[task_type_count] = { "none", "sort", "self", "pair", "sub_self", - "sub_pair", "init", "ghost", "extra_ghost", "kick", + "sub_pair", "init", "ghost", "extra_ghost", "drift", "kick", "send", "recv", "grav_gather_m", "grav_fft", "grav_mm", "grav_up", "cooling", "sourceterms"}; @@ -150,6 +150,7 @@ __attribute__((always_inline)) INLINE static enum task_actions task_acts_on( case task_type_kick: case task_type_send: case task_type_recv: + case task_type_drift: if (t->ci->count > 0 && t->ci->gcount > 0) return task_action_all; else if (t->ci->count > 0) diff --git a/src/task.h b/src/task.h index c9425fdd1..b4767f036 100644 --- a/src/task.h +++ b/src/task.h @@ -45,6 +45,7 @@ enum task_types { task_type_init, task_type_ghost, task_type_extra_ghost, + task_type_drift, task_type_kick, task_type_send, task_type_recv, -- GitLab From 456d9fad2e45ceb548cac734a894386c52edb37e Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Mon, 5 Dec 2016 18:56:16 +0000 Subject: [PATCH 02/25] Activate drift --- src/cell.c | 9 ++++++--- src/engine.c | 12 +++++++++++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/cell.c b/src/cell.c index fcbbfe4ae..23e5e5781 100644 --- a/src/cell.c +++ b/src/cell.c @@ -880,10 +880,14 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) { /* Activate the drift on both sides */ if (t->type == task_type_pair || t->type == task_type_sub_pair) { - scheduler_activate(s, ci->drift); - scheduler_activate(s, cj->drift); + if (ci->drift != NULL) scheduler_activate(s, ci->drift); + if (cj->drift != NULL) scheduler_activate(s, cj->drift); } + if (t->type == task_type_self || t->type == task_type_sub_self) { + if (ci->drift != NULL) scheduler_activate(s, ci->drift); + } + /* Check whether there was too much particle motion */ if (t->type == task_type_pair || t->type == task_type_sub_pair) { if (t->tight && @@ -962,7 +966,6 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) { if (c->ghost != NULL) scheduler_activate(s, c->ghost); if (c->init != NULL) scheduler_activate(s, c->init); if (c->kick != NULL) scheduler_activate(s, c->kick); - if (c->drift != NULL) scheduler_activate(s, c->drift); if (c->cooling != NULL) scheduler_activate(s, c->cooling); if (c->sourceterms != NULL) scheduler_activate(s, c->sourceterms); diff --git a/src/engine.c b/src/engine.c index 111f0a230..879fbc28b 100644 --- a/src/engine.c +++ b/src/engine.c @@ -1986,6 +1986,16 @@ void engine_marktasks_mapper(void *map_data, int num_elements, if (t->ci->ti_end_min <= ti_end) scheduler_activate(s, t); } + /* Self? */ + else if (t->type == task_type_self || t->type == task_type_sub_self) { + + /* Local pointers. */ + const struct cell *ci = t->ci; + + /* Activate the drift */ + if(ci->drift) scheduler_activate(s, ci->drift); + } + /* Pair? */ else if (t->type == task_type_pair || t->type == task_type_sub_pair) { @@ -1995,7 +2005,7 @@ void engine_marktasks_mapper(void *map_data, int num_elements, /* Activate the drift on both sides */ if(ci->drift) scheduler_activate(s, ci->drift); - if(ci->drift) scheduler_activate(s, cj->drift); + if(cj->drift) scheduler_activate(s, cj->drift); /* Too much particle movement? */ if (t->tight && -- GitLab From 7c2c2cc0bb55f4c9b17cf49fc10d7d05a2e4cb7e Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Mon, 5 Dec 2016 21:45:11 +0000 Subject: [PATCH 03/25] Drift task now scheduled correctly --- src/cell.c | 10 ++-------- src/engine.c | 11 ++++------- src/runner.c | 14 ++++++++------ src/scheduler.c | 44 ++++++++++++++++++++++---------------------- src/space.c | 1 + src/task.c | 8 ++++---- 6 files changed, 41 insertions(+), 47 deletions(-) diff --git a/src/cell.c b/src/cell.c index 23e5e5781..9bd2a104c 100644 --- a/src/cell.c +++ b/src/cell.c @@ -879,15 +879,9 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) { } /* Activate the drift on both sides */ - if (t->type == task_type_pair || t->type == task_type_sub_pair) { - if (ci->drift != NULL) scheduler_activate(s, ci->drift); - if (cj->drift != NULL) scheduler_activate(s, cj->drift); - } + if (ci == c && cj != NULL) scheduler_activate(s, cj->drift); + if (cj == c && ci != NULL) scheduler_activate(s, ci->drift); - if (t->type == task_type_self || t->type == task_type_sub_self) { - if (ci->drift != NULL) scheduler_activate(s, ci->drift); - } - /* Check whether there was too much particle motion */ if (t->type == task_type_pair || t->type == task_type_sub_pair) { if (t->tight && diff --git a/src/engine.c b/src/engine.c index 879fbc28b..e892f674e 100644 --- a/src/engine.c +++ b/src/engine.c @@ -139,9 +139,6 @@ void engine_make_hierarchical_tasks(struct engine *e, struct cell *c) { c->init = scheduler_addtask(s, task_type_init, task_subtype_none, 0, 0, c, NULL, 0); - /* c->drift = scheduler_addtask(s, task_type_drift, task_subtype_none, 0, 0, */ - /* c, NULL, 0); */ - c->kick = scheduler_addtask(s, task_type_kick, task_subtype_none, 0, 0, c, NULL, 0); @@ -1993,9 +1990,9 @@ void engine_marktasks_mapper(void *map_data, int num_elements, const struct cell *ci = t->ci; /* Activate the drift */ - if(ci->drift) scheduler_activate(s, ci->drift); + if (ci->drift) scheduler_activate(s, ci->drift); } - + /* Pair? */ else if (t->type == task_type_pair || t->type == task_type_sub_pair) { @@ -2004,8 +2001,8 @@ void engine_marktasks_mapper(void *map_data, int num_elements, const struct cell *cj = t->cj; /* Activate the drift on both sides */ - if(ci->drift) scheduler_activate(s, ci->drift); - if(cj->drift) scheduler_activate(s, cj->drift); + if (ci->drift) scheduler_activate(s, ci->drift); + if (cj->drift) scheduler_activate(s, cj->drift); /* Too much particle movement? */ if (t->tight && diff --git a/src/runner.c b/src/runner.c index a62fb87a8..c5975c12a 100644 --- a/src/runner.c +++ b/src/runner.c @@ -1127,7 +1127,8 @@ void *runner_main(void *data) { /* Check that we haven't scheduled an inactive task */ #ifdef SWIFT_DEBUG_CHECKS if (cj == NULL) { /* self */ - if (!cell_is_active(ci, e) && t->type != task_type_sort && t->type != task_type_drift) + if (!cell_is_active(ci, e) && t->type != task_type_sort && + t->type != task_type_drift) error( "Task (type='%s/%s') should have been skipped ti_current=%d " "c->ti_end_min=%d", @@ -1143,10 +1144,11 @@ void *runner_main(void *data) { taskID_names[t->type], subtaskID_names[t->subtype], e->ti_current, ci->ti_end_min, t->flags); - /* Special treatement for drifts */ - if (!cell_is_active(ci, e) && t->type == task_type_drift) - {;} - + /* Special treatement for drifts */ + if (!cell_is_active(ci, e) && t->type == task_type_drift) { + ; + } + } else { /* pair */ if (!cell_is_active(ci, e) && !cell_is_active(cj, e)) error( @@ -1237,7 +1239,7 @@ void *runner_main(void *data) { break; #endif case task_type_drift: - // runner_do_drift(r, ci, 1); + runner_do_drift(r, ci, 1); break; case task_type_kick: runner_do_kick(r, ci, 1); diff --git a/src/scheduler.c b/src/scheduler.c index 4574d4a1d..55ec764e5 100644 --- a/src/scheduler.c +++ b/src/scheduler.c @@ -133,7 +133,7 @@ static void scheduler_splittask(struct task *t, struct scheduler *s) { /* Non-splittable task? */ if ((t->ci == NULL || (t->type == task_type_pair && t->cj == NULL)) || ((t->type == task_type_kick) && t->ci->nodeID != s->nodeID) || - ((t->type == task_type_drift) && t->ci->nodeID != s->nodeID) || + ((t->type == task_type_drift) && t->ci->nodeID != s->nodeID) || ((t->type == task_type_init) && t->ci->nodeID != s->nodeID)) { t->type = task_type_none; t->skip = 1; @@ -614,19 +614,17 @@ static void scheduler_splittask(struct task *t, struct scheduler *s) { 1 << sid, 0, ci, NULL, 0); else ci->sorts->flags |= (1 << sid); - - scheduler_addunlock(s, ci->sorts, t); - /* Create the drift for ci. */ - if (ci->drift == NULL) { - ci->drift = scheduler_addtask(s, task_type_drift, task_subtype_none, + scheduler_addunlock(s, ci->sorts, t); + + /* Create the drift for ci. */ + if (ci->drift == NULL) { + ci->drift = scheduler_addtask(s, task_type_drift, task_subtype_none, 0, 0, ci, NULL, 0); - scheduler_addunlock(s, ci->drift, ci->sorts); - } + scheduler_addunlock(s, ci->drift, ci->sorts); + } lock_unlock_blind(&ci->lock); - - /* Create the sort for cj. */ lock_lock(&cj->lock); if (cj->sorts == NULL) @@ -637,13 +635,13 @@ static void scheduler_splittask(struct task *t, struct scheduler *s) { scheduler_addunlock(s, cj->sorts, t); - /* Create the drift for cj. */ - if (cj->drift == NULL) { - cj->drift = scheduler_addtask(s, task_type_drift, task_subtype_none, + /* Create the drift for cj. */ + if (cj->drift == NULL) { + cj->drift = scheduler_addtask(s, task_type_drift, task_subtype_none, 0, 0, cj, NULL, 0); - scheduler_addunlock(s, cj->drift, cj->sorts); - } - lock_unlock_blind(&cj->lock); + scheduler_addunlock(s, cj->drift, cj->sorts); + } + lock_unlock_blind(&cj->lock); } } /* pair interaction? */ @@ -802,8 +800,9 @@ void scheduler_set_unlocks(struct scheduler *s) { for (int j = i + 1; j < t->nr_unlock_tasks; j++) { if (t->unlock_tasks[i] == t->unlock_tasks[j]) error("duplicate unlock! t->type=%s/%s unlocking type=%s/%s", - taskID_names[t->type], subtaskID_names[t->subtype], - taskID_names[t->unlock_tasks[i]->type], subtaskID_names[t->unlock_tasks[i]->subtype]); + taskID_names[t->type], subtaskID_names[t->subtype], + taskID_names[t->unlock_tasks[i]->type], + subtaskID_names[t->unlock_tasks[i]->subtype]); } } } @@ -1102,10 +1101,11 @@ void scheduler_start(struct scheduler *s) { taskID_names[t->type], subtaskID_names[t->subtype], ti_current, ci->ti_end_min, t->flags); - /* Special treatement for drifts */ + /* Special treatement for drifts */ if (ci->ti_end_min == ti_current && t->skip && - t->type == task_type_drift) - {;} + t->type == task_type_drift) { + ; + } } else { /* pair */ @@ -1174,7 +1174,7 @@ void scheduler_enqueue(struct scheduler *s, struct task *t) { case task_type_sort: case task_type_ghost: case task_type_kick: - case task_type_drift: + case task_type_drift: case task_type_init: qid = t->ci->super->owner; break; diff --git a/src/space.c b/src/space.c index 935677a9e..f3b19ab59 100644 --- a/src/space.c +++ b/src/space.c @@ -409,6 +409,7 @@ void space_regrid(struct space *s, int verbose) { s->cells_top[k].extra_ghost = NULL; s->cells_top[k].ghost = NULL; s->cells_top[k].kick = NULL; + s->cells_top[k].drift = NULL; s->cells_top[k].cooling = NULL; s->cells_top[k].sourceterms = NULL; s->cells_top[k].super = &s->cells_top[k]; diff --git a/src/task.c b/src/task.c index 19a895915..ceea09126 100644 --- a/src/task.c +++ b/src/task.c @@ -48,10 +48,10 @@ /* Task type names. */ const char *taskID_names[task_type_count] = { - "none", "sort", "self", "pair", "sub_self", - "sub_pair", "init", "ghost", "extra_ghost", "drift", "kick", - "send", "recv", "grav_gather_m", "grav_fft", "grav_mm", - "grav_up", "cooling", "sourceterms"}; + "none", "sort", "self", "pair", "sub_self", + "sub_pair", "init", "ghost", "extra_ghost", "drift", + "kick", "send", "recv", "grav_gather_m", "grav_fft", + "grav_mm", "grav_up", "cooling", "sourceterms"}; const char *subtaskID_names[task_subtype_count] = { "none", "density", "gradient", "force", "grav", "external_grav", "tend"}; -- GitLab From 24e1ff733b241ba8819f84e25023e2101a87acbb Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Tue, 6 Dec 2016 10:20:10 +0000 Subject: [PATCH 04/25] Be verbose about rebuilding --- src/space.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/space.c b/src/space.c index f3b19ab59..8665bdbd8 100644 --- a/src/space.c +++ b/src/space.c @@ -295,6 +295,12 @@ void space_regrid(struct space *s, int verbose) { if (s->cells_top == NULL || cdim[0] < s->cdim[0] || cdim[1] < s->cdim[1] || cdim[2] < s->cdim[2]) { +/* Be verbose about this. */ +#ifdef SWIFT_DEBUG_CHECKS + message("re)griding space cdim=(%d %d %d)", cdim[0], cdim[1], cdim[2]); + fflush(stdout); +#endif + /* Free the old cells, if they were allocated. */ if (s->cells_top != NULL) { for (int k = 0; k < s->nr_cells; k++) { @@ -444,8 +450,11 @@ void space_rebuild(struct space *s, int verbose) { const ticks tic = getticks(); - /* Be verbose about this. */ - // message("re)building space..."); fflush(stdout); +/* Be verbose about this. */ +#ifdef SWIFT_DEBUG_CHECKS + message("re)building space"); + fflush(stdout); +#endif /* Re-grid if necessary, or just re-set the cell data. */ space_regrid(s, verbose); -- GitLab From e1b61cbe5e72a5beb70d6066cb042d60c579169b Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Tue, 6 Dec 2016 10:41:28 +0000 Subject: [PATCH 05/25] Rephrased the logic in engine_step() in terms of task-unskipping and cell-drifting --- src/engine.c | 44 +++++++++++++++++++++++++++++++------------- src/engine.h | 3 ++- src/runner.c | 4 ++-- src/runner.h | 3 ++- 4 files changed, 37 insertions(+), 17 deletions(-) diff --git a/src/engine.c b/src/engine.c index e892f674e..be58038f5 100644 --- a/src/engine.c +++ b/src/engine.c @@ -2229,6 +2229,9 @@ void engine_prepare(struct engine *e, int nodrift) { TIMER_TIC; + /* Unskip active tasks and check for rebuild */ + engine_unskip(e); + /* Run through the tasks and mark as skip or not. */ int rebuild = e->forcerebuild; @@ -2246,11 +2249,11 @@ void engine_prepare(struct engine *e, int nodrift) { /* Drift all particles to the current time if needed. */ if (!nodrift) { - e->drift_all = 1; - engine_drift(e); + // e->drift_all = 1; + engine_drift_all(e); /* Restore the default drifting policy */ - e->drift_all = (e->policy & engine_policy_drift_all); + // e->drift_all = (e->policy & engine_policy_drift_all); } #ifdef SWIFT_DEBUG_CHECKS @@ -2603,10 +2606,10 @@ void engine_step(struct engine *e) { /* Drift everybody to the snapshot position */ e->drift_all = 1; - engine_drift(e); + engine_drift_all(e); /* Restore the default drifting policy */ - e->drift_all = (e->policy & engine_policy_drift_all); + // e->drift_all = (e->policy & engine_policy_drift_all); /* Dump... */ engine_dump_snapshot(e); @@ -2638,17 +2641,17 @@ void engine_step(struct engine *e) { /* Drift only the necessary particles, that means all particles * if we are about to repartition. */ const int repart = (e->forcerepart != REPART_NONE); - e->drift_all = repart || e->drift_all; - engine_drift(e); + // e->drift_all = repart || e->drift_all; + if (repart) engine_drift_all(e); /* Re-distribute the particles amongst the nodes? */ if (repart) engine_repartition(e); /* Prepare the space. */ - engine_prepare(e, e->drift_all); + engine_prepare(e, (e->drift_all || repart)); /* Restore the default drifting policy */ - e->drift_all = (e->policy & engine_policy_drift_all); + // e->drift_all = (e->policy & engine_policy_drift_all); if (e->verbose) engine_print_task_counts(e); @@ -2683,15 +2686,30 @@ int engine_is_done(struct engine *e) { * * @param e The #engine. */ -void engine_drift(struct engine *e) { +void engine_unskip(struct engine *e) { const ticks tic = getticks(); - threadpool_map(&e->threadpool, runner_do_drift_mapper, e->s->cells_top, + threadpool_map(&e->threadpool, runner_do_unskip_mapper, e->s->cells_top, e->s->nr_cells, sizeof(struct cell), 1, e); if (e->verbose) - message("took %.3f %s (including task unskipping).", - clocks_from_ticks(getticks() - tic), clocks_getunit()); + message("took %.3f %s.", clocks_from_ticks(getticks() - tic), + clocks_getunit()); +} + +void engine_drift_all(struct engine *e) { + + e->drift_all = 1; + + const ticks tic = getticks(); + threadpool_map(&e->threadpool, runner_do_unskip_mapper, e->s->cells_top, + e->s->nr_cells, sizeof(struct cell), 1, e); + + e->drift_all = e->policy & engine_policy_drift_all; + + if (e->verbose) + message("took %.3f %s.", clocks_from_ticks(getticks() - tic), + clocks_getunit()); } /** diff --git a/src/engine.h b/src/engine.h index 861e32162..1960b3dc4 100644 --- a/src/engine.h +++ b/src/engine.h @@ -218,7 +218,8 @@ struct engine { /* Function prototypes. */ void engine_barrier(struct engine *e, int tid); void engine_compute_next_snapshot_time(struct engine *e); -void engine_drift(struct engine *e); +void engine_unskip(struct engine *e); +void engine_drift_all(struct engine *e); void engine_dump_snapshot(struct engine *e); void engine_init(struct engine *e, struct space *s, const struct swift_params *params, int nr_nodes, int nodeID, diff --git a/src/runner.c b/src/runner.c index c5975c12a..fa42af4e9 100644 --- a/src/runner.c +++ b/src/runner.c @@ -877,8 +877,8 @@ static void runner_do_unskip(struct cell *c, struct engine *e, int drift) { * @param extra_data Pointer to an #engine. */ -void runner_do_drift_mapper(void *map_data, int num_elements, - void *extra_data) { +void runner_do_unskip_mapper(void *map_data, int num_elements, + void *extra_data) { struct engine *e = (struct engine *)extra_data; struct cell *cells = (struct cell *)map_data; diff --git a/src/runner.h b/src/runner.h index a8caf2424..5c89c97c6 100644 --- a/src/runner.h +++ b/src/runner.h @@ -56,6 +56,7 @@ void runner_do_init(struct runner *r, struct cell *c, int timer); void runner_do_cooling(struct runner *r, struct cell *c, int timer); void runner_do_grav_external(struct runner *r, struct cell *c, int timer); void *runner_main(void *data); -void runner_do_drift_mapper(void *map_data, int num_elements, void *extra_data); +void runner_do_unskip_mapper(void *map_data, int num_elements, + void *extra_data); #endif /* SWIFT_RUNNER_H */ -- GitLab From 1f5f33bb5c5c609bdebaf5a4bcf649309402ed88 Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Tue, 6 Dec 2016 12:15:54 +0000 Subject: [PATCH 06/25] Make the drift task recurse and updated all its children's ti_old --- src/cell.c | 107 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/cell.h | 3 +- src/engine.c | 2 +- src/runner.c | 43 +++++++++++++++------ src/runner.h | 1 + 5 files changed, 142 insertions(+), 14 deletions(-) diff --git a/src/cell.c b/src/cell.c index 9bd2a104c..5557dd15d 100644 --- a/src/cell.c +++ b/src/cell.c @@ -49,6 +49,7 @@ /* Local headers. */ #include "active.h" #include "atomic.h" +#include "drift.h" #include "error.h" #include "gravity.h" #include "hydro.h" @@ -985,3 +986,109 @@ void cell_set_super(struct cell *c, struct cell *super) { for (int k = 0; k < 8; k++) if (c->progeny[k] != NULL) cell_set_super(c->progeny[k], super); } + +void cell_drift(struct cell *c, struct engine *e) { + + const double timeBase = e->timeBase; + const int ti_old = c->ti_old; + const int ti_current = e->ti_current; + struct part *const parts = c->parts; + struct xpart *const xparts = c->xparts; + struct gpart *const gparts = c->gparts; + + /* Drift from the last time the cell was drifted to the current time */ + const double dt = (ti_current - ti_old) * timeBase; + float dx_max = 0.f, dx2_max = 0.f, h_max = 0.f; + + /* Check that we are actually going to move forward. */ + if (ti_current <= ti_old) return; + + /* Are we in a leaf ? */ + if (c->split) { + + /* Loop over the progeny and collect their data. */ + for (int k = 0; k < 8; k++) + if (c->progeny[k] != NULL) { + struct cell *cp = c->progeny[k]; + cell_drift(cp, e); + dx_max = max(dx_max, cp->dx_max); + h_max = max(h_max, cp->h_max); + } + + } else { + + /* Loop over all the g-particles in the cell */ + const size_t nr_gparts = c->gcount; + for (size_t k = 0; k < nr_gparts; k++) { + + /* Get a handle on the gpart. */ + struct gpart *const gp = &gparts[k]; + + /* Drift... */ + drift_gpart(gp, dt, timeBase, ti_old, ti_current); + + /* Compute (square of) motion since last cell construction */ + const float dx2 = gp->x_diff[0] * gp->x_diff[0] + + gp->x_diff[1] * gp->x_diff[1] + + gp->x_diff[2] * gp->x_diff[2]; + dx2_max = (dx2_max > dx2) ? dx2_max : dx2; + } + + /* Loop over all the particles in the cell */ + const size_t nr_parts = c->count; + for (size_t k = 0; k < nr_parts; k++) { + + /* Get a handle on the part. */ + struct part *const p = &parts[k]; + struct xpart *const xp = &xparts[k]; + + /* Drift... */ + drift_part(p, xp, dt, timeBase, ti_old, ti_current); + + /* Compute (square of) motion since last cell construction */ + const float dx2 = xp->x_diff[0] * xp->x_diff[0] + + xp->x_diff[1] * xp->x_diff[1] + + xp->x_diff[2] * xp->x_diff[2]; + dx2_max = (dx2_max > dx2) ? dx2_max : dx2; + + /* Maximal smoothing length */ + h_max = (h_max > p->h) ? h_max : p->h; + } + + /* Now, get the maximal particle motion from its square */ + dx_max = sqrtf(dx2_max); + + /* Set ti_old on the sub-cells */ + cell_set_ti_old(c, e->ti_current); + + } /* Check that we are actually going to move forward. */ + + /* Store the values */ + c->h_max = h_max; + c->dx_max = dx_max; + + /* Update the time of the last drift */ + c->ti_old = ti_current; +} + +/** + * Set ti_old of a #cell and all its progenies to a new value. + * + * @param c The #cell. + * @param ti_current The new value of ti_old. + */ +void cell_set_ti_old(struct cell *c, int ti_current) { + + /* Set this cell */ + c->ti_old = ti_current; + + /* Recurse */ + if (c->split) { + for (int k = 0; k < 8; ++k) { + if (c->progeny[k] != NULL) { + struct cell *cp = c->progeny[k]; + cell_set_ti_old(cp, ti_current); + } + } + } +} diff --git a/src/cell.h b/src/cell.h index d978de71f..8a7301a57 100644 --- a/src/cell.h +++ b/src/cell.h @@ -298,5 +298,6 @@ void cell_check_drift_point(struct cell *c, void *data); int cell_is_drift_needed(struct cell *c, const struct engine *e); int cell_unskip_tasks(struct cell *c, struct scheduler *s); void cell_set_super(struct cell *c, struct cell *super); - +void cell_drift(struct cell *c, struct engine *e); +void cell_set_ti_old(struct cell *c, int ti_current); #endif /* SWIFT_CELL_H */ diff --git a/src/engine.c b/src/engine.c index be58038f5..d6a1dcf8b 100644 --- a/src/engine.c +++ b/src/engine.c @@ -2702,7 +2702,7 @@ void engine_drift_all(struct engine *e) { e->drift_all = 1; const ticks tic = getticks(); - threadpool_map(&e->threadpool, runner_do_unskip_mapper, e->s->cells_top, + threadpool_map(&e->threadpool, runner_do_drift_mapper, e->s->cells_top, e->s->nr_cells, sizeof(struct cell), 1, e); e->drift_all = e->policy & engine_policy_drift_all; diff --git a/src/runner.c b/src/runner.c index fa42af4e9..334aa5560 100644 --- a/src/runner.c +++ b/src/runner.c @@ -779,23 +779,23 @@ static void runner_do_unskip(struct cell *c, struct engine *e, int drift) { /* Now, we can drift */ /* Get some information first */ - const double timeBase = e->timeBase; + // const double timeBase = e->timeBase; const int ti_old = c->ti_old; const int ti_current = e->ti_current; - struct part *const parts = c->parts; - struct xpart *const xparts = c->xparts; - struct gpart *const gparts = c->gparts; + // struct part *const parts = c->parts; + // struct xpart *const xparts = c->xparts; + // struct gpart *const gparts = c->gparts; /* Drift from the last time the cell was drifted to the current time */ - const double dt = (ti_current - ti_old) * timeBase; - float dx_max = 0.f, dx2_max = 0.f, h_max = 0.f; + // const double dt = (ti_current - ti_old) * timeBase; + // float dx_max = 0.f, dx2_max = 0.f, h_max = 0.f; /* No children? */ if (!c->split) { /* Check that we are actually going to move forward. */ if (ti_current > ti_old) { - +#if 0 /* Loop over all the g-particles in the cell */ const size_t nr_gparts = c->gcount; for (size_t k = 0; k < nr_gparts; k++) { @@ -836,13 +836,15 @@ static void runner_do_unskip(struct cell *c, struct engine *e, int drift) { /* Now, get the maximal particle motion from its square */ dx_max = sqrtf(dx2_max); - +#endif } /* Check that we are actually going to move forward. */ else { +#if 0 /* ti_old == ti_current, just keep the current cell values. */ h_max = c->h_max; dx_max = c->dx_max; +#endif } } @@ -853,20 +855,23 @@ static void runner_do_unskip(struct cell *c, struct engine *e, int drift) { for (int k = 0; k < 8; k++) if (c->progeny[k] != NULL) { struct cell *cp = c->progeny[k]; - + message("aaa"); /* Recurse. */ runner_do_unskip(cp, e, drift); +#if 0 dx_max = max(dx_max, cp->dx_max); h_max = max(h_max, cp->h_max); +#endif } } - +#if 0 /* Store the values */ c->h_max = h_max; c->dx_max = dx_max; /* Update the time of the last drift */ c->ti_old = ti_current; +#endif } /** @@ -876,7 +881,6 @@ static void runner_do_unskip(struct cell *c, struct engine *e, int drift) { * @param num_elements Chunk size. * @param extra_data Pointer to an #engine. */ - void runner_do_unskip_mapper(void *map_data, int num_elements, void *extra_data) { @@ -893,7 +897,22 @@ void runner_do_unskip_mapper(void *map_data, int num_elements, } } -void runner_do_drift(struct runner *r, struct cell *c, int timer) {} +void runner_do_drift(struct runner *r, struct cell *c, int timer) { + + cell_drift(c, r->e); +} + +void runner_do_drift_mapper(void *map_data, int num_elements, + void *extra_data) { + + struct engine *e = (struct engine *)extra_data; + struct cell *cells = (struct cell *)map_data; + + for (int ind = 0; ind < num_elements; ind++) { + struct cell *c = &cells[ind]; + if (c != NULL && c->nodeID == e->nodeID) cell_drift(c, e); + } +} /** * @brief Kick particles in momentum space and collect statistics (floating diff --git a/src/runner.h b/src/runner.h index 5c89c97c6..6aab8469a 100644 --- a/src/runner.h +++ b/src/runner.h @@ -58,5 +58,6 @@ void runner_do_grav_external(struct runner *r, struct cell *c, int timer); void *runner_main(void *data); void runner_do_unskip_mapper(void *map_data, int num_elements, void *extra_data); +void runner_do_drift_mapper(void *map_data, int num_elements, void *extra_data); #endif /* SWIFT_RUNNER_H */ -- GitLab From fe4019ec0495d18e67a9d3a007917df6a1ac7e51 Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Tue, 6 Dec 2016 14:44:37 +0000 Subject: [PATCH 07/25] Correct locking/unlocking of drift task --- src/task.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/task.c b/src/task.c index ceea09126..d787e92af 100644 --- a/src/task.c +++ b/src/task.c @@ -261,6 +261,11 @@ void task_unlock(struct task *t) { /* Act based on task type. */ switch (type) { + case task_type_drift: + cell_unlocktree(ci); + cell_gunlocktree(ci); + break; + case task_type_sort: cell_unlocktree(ci); break; @@ -328,6 +333,15 @@ int task_lock(struct task *t) { #endif break; + case task_type_drift: + if (ci->hold || ci->ghold) return 0; + if (cell_locktree(ci) != 0) return 0; + if (cell_glocktree(ci) != 0) { + cell_unlocktree(ci); + return 0; + } + break; + case task_type_sort: if (cell_locktree(ci) != 0) return 0; break; -- GitLab From 2d03f5ad2515375c8f11635e6cfb68b73e2bec66 Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Tue, 6 Dec 2016 14:45:20 +0000 Subject: [PATCH 08/25] Don't recurse in cell_drift if not necessary --- src/cell.c | 34 ++++++++++++++++++---------------- src/cell.h | 2 +- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/src/cell.c b/src/cell.c index 5557dd15d..120aa9d07 100644 --- a/src/cell.c +++ b/src/cell.c @@ -880,8 +880,8 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) { } /* Activate the drift on both sides */ - if (ci == c && cj != NULL) scheduler_activate(s, cj->drift); - if (cj == c && ci != NULL) scheduler_activate(s, ci->drift); + if (ci == c && cj != NULL && cj->drift != NULL) scheduler_activate(s, cj->drift); + if (cj == c && ci != NULL && ci->drift != NULL) scheduler_activate(s, ci->drift); /* Check whether there was too much particle motion */ if (t->type == task_type_pair || t->type == task_type_sub_pair) { @@ -987,7 +987,7 @@ void cell_set_super(struct cell *c, struct cell *super) { if (c->progeny[k] != NULL) cell_set_super(c->progeny[k], super); } -void cell_drift(struct cell *c, struct engine *e) { +void cell_drift(struct cell *c, const struct engine *e) { const double timeBase = e->timeBase; const int ti_old = c->ti_old; @@ -999,41 +999,41 @@ void cell_drift(struct cell *c, struct engine *e) { /* Drift from the last time the cell was drifted to the current time */ const double dt = (ti_current - ti_old) * timeBase; float dx_max = 0.f, dx2_max = 0.f, h_max = 0.f; - + /* Check that we are actually going to move forward. */ if (ti_current <= ti_old) return; - /* Are we in a leaf ? */ + /* Are we not in a leaf ? */ if (c->split) { /* Loop over the progeny and collect their data. */ for (int k = 0; k < 8; k++) if (c->progeny[k] != NULL) { struct cell *cp = c->progeny[k]; - cell_drift(cp, e); - dx_max = max(dx_max, cp->dx_max); + cell_drift(cp, e); + dx_max = max(dx_max, cp->dx_max); h_max = max(h_max, cp->h_max); } - + } else { - + /* Loop over all the g-particles in the cell */ const size_t nr_gparts = c->gcount; for (size_t k = 0; k < nr_gparts; k++) { - + /* Get a handle on the gpart. */ struct gpart *const gp = &gparts[k]; - + /* Drift... */ drift_gpart(gp, dt, timeBase, ti_old, ti_current); - + /* Compute (square of) motion since last cell construction */ const float dx2 = gp->x_diff[0] * gp->x_diff[0] + - gp->x_diff[1] * gp->x_diff[1] + - gp->x_diff[2] * gp->x_diff[2]; + gp->x_diff[1] * gp->x_diff[1] + + gp->x_diff[2] * gp->x_diff[2]; dx2_max = (dx2_max > dx2) ? dx2_max : dx2; } - + /* Loop over all the particles in the cell */ const size_t nr_parts = c->count; for (size_t k = 0; k < nr_parts; k++) { @@ -1045,6 +1045,8 @@ void cell_drift(struct cell *c, struct engine *e) { /* Drift... */ drift_part(p, xp, dt, timeBase, ti_old, ti_current); + p->ti_old = ti_current; + /* Compute (square of) motion since last cell construction */ const float dx2 = xp->x_diff[0] * xp->x_diff[0] + xp->x_diff[1] * xp->x_diff[1] + @@ -1059,7 +1061,7 @@ void cell_drift(struct cell *c, struct engine *e) { dx_max = sqrtf(dx2_max); /* Set ti_old on the sub-cells */ - cell_set_ti_old(c, e->ti_current); + //cell_set_ti_old(c, e->ti_current); } /* Check that we are actually going to move forward. */ diff --git a/src/cell.h b/src/cell.h index 8a7301a57..d55d15c88 100644 --- a/src/cell.h +++ b/src/cell.h @@ -298,6 +298,6 @@ void cell_check_drift_point(struct cell *c, void *data); int cell_is_drift_needed(struct cell *c, const struct engine *e); int cell_unskip_tasks(struct cell *c, struct scheduler *s); void cell_set_super(struct cell *c, struct cell *super); -void cell_drift(struct cell *c, struct engine *e); +void cell_drift(struct cell *c, const struct engine *e); void cell_set_ti_old(struct cell *c, int ti_current); #endif /* SWIFT_CELL_H */ -- GitLab From 0b6ee9461aa797a8c2cf4fd7e1376b4ebdcf1224 Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Wed, 7 Dec 2016 11:00:18 +0000 Subject: [PATCH 09/25] drift on demand working --- examples/SedovBlast_3D/sedov.yml | 3 ++ src/active.h | 37 +++++++++++++++++----- src/cell.c | 29 +++++++++++++---- src/engine.c | 12 ++++++-- src/hydro/Gadget2/hydro.h | 1 + src/hydro/Gadget2/hydro_part.h | 18 ++++++----- src/runner.c | 16 +++++++--- src/runner_doiact.h | 53 +++++++++++++++++++++++--------- src/scheduler.c | 38 ++++++++++++++++++----- src/space.c | 32 +++++++++++++++++-- src/space.h | 1 + src/task.c | 6 ++-- 12 files changed, 189 insertions(+), 57 deletions(-) diff --git a/examples/SedovBlast_3D/sedov.yml b/examples/SedovBlast_3D/sedov.yml index 1cc4aced9..dc2295037 100644 --- a/examples/SedovBlast_3D/sedov.yml +++ b/examples/SedovBlast_3D/sedov.yml @@ -6,6 +6,9 @@ InternalUnitSystem: UnitCurrent_in_cgs: 1 # Amperes UnitTemp_in_cgs: 1 # Kelvin +#Scheduler: +# max_top_level_cells: 3 + # Parameters governing the time integration TimeIntegration: time_begin: 0. # The starting time of the simulation (in internal units). diff --git a/src/active.h b/src/active.h index e33f8baf6..903b3c092 100644 --- a/src/active.h +++ b/src/active.h @@ -36,7 +36,7 @@ * @param c The #cell. * @param e The #engine containing information about the current time. */ -__attribute__((always_inline)) INLINE static void cell_is_drifted( +__attribute__((always_inline)) INLINE static int cell_is_drifted( const struct cell *c, const struct engine *e) { #ifdef SWIFT_DEBUG_CHECKS @@ -45,14 +45,35 @@ __attribute__((always_inline)) INLINE static void cell_is_drifted( "Cell has been drifted too far forward in time! c->ti_old=%d " "e->ti_current=%d", c->ti_old, e->ti_current); - - if (c->ti_old != e->ti_current) { - error( - "Cell has not been drifted to the current time c->ti_old=%d, " - "e->ti_current=%d", - c->ti_old, e->ti_current); - } #endif + + /* if (c->ti_old != e->ti_current) { */ + + /* int wrong = 0; */ + /* for (int i = 0; i < c->count; ++i) { */ + /* if (c->parts[i].ti_old < e->ti_current) ++wrong; */ + /* } */ + + /* message( */ + /* "Cell has not been drifted to the current time c->ti_old=%d, " */ + /* "e->ti_current=%d wrong=%d c->count=%d c->drift=%p, c->depth=%d, + * c=%p, c->super=%p, c->parent=%p ", */ + /* c->ti_old, e->ti_current, wrong, c->count, c->drift, c->depth, c, + * c->super, c->parent); */ + + /* cell_drift((struct cell*)c, e); */ + + /* message( */ + /* "Cell has not been drifted to the current time c->ti_old=%d, " */ + /* "e->ti_current=%d wrong=%d c->count=%d c->drift=%p, c->depth=%d, + * c=%p, c->super=%p, c->parent=%p ", */ + /* c->ti_old, e->ti_current, wrong, c->count, c->drift, c->depth, c, + * c->super, c->parent); */ + + /* error("AAAAA"); */ + /* } */ + + return (c->ti_old == e->ti_current); } /** diff --git a/src/cell.c b/src/cell.c index 120aa9d07..5b7941634 100644 --- a/src/cell.c +++ b/src/cell.c @@ -717,6 +717,13 @@ void cell_check_drift_point(struct cell *c, void *data) { if (c->ti_old != ti_current) error("Cell in an incorrect time-zone! c->ti_old=%d ti_current=%d", c->ti_old, ti_current); + + /* for (int i = 0; i < c->count; ++i) */ + /* if (c->parts[i].ti_old != ti_current) */ + /* error( */ + /* "Particle in an incorrect time-zone! part->ti_old=%d c->ti_old=%d " */ + /* "ti_current=%d", */ + /* c->parts[i].ti_old, c->ti_old, ti_current); */ } /** @@ -880,8 +887,10 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) { } /* Activate the drift on both sides */ - if (ci == c && cj != NULL && cj->drift != NULL) scheduler_activate(s, cj->drift); - if (cj == c && ci != NULL && ci->drift != NULL) scheduler_activate(s, ci->drift); + if (ci == c && cj != NULL && cj->drift != NULL) + scheduler_activate(s, cj->drift); + if (cj == c && ci != NULL && ci->drift != NULL) + scheduler_activate(s, ci->drift); /* Check whether there was too much particle motion */ if (t->type == task_type_pair || t->type == task_type_sub_pair) { @@ -1000,8 +1009,11 @@ void cell_drift(struct cell *c, const struct engine *e) { const double dt = (ti_current - ti_old) * timeBase; float dx_max = 0.f, dx2_max = 0.f, h_max = 0.f; + //message("DRFIT ! ti_old=%d ti_current=%d", ti_old, ti_current); + /* Check that we are actually going to move forward. */ - if (ti_current <= ti_old) return; + if (ti_current < ti_old) error("Attempt to drift to the past"); + //if (ti_current == ti_old) return; /* Are we not in a leaf ? */ if (c->split) { @@ -1015,7 +1027,7 @@ void cell_drift(struct cell *c, const struct engine *e) { h_max = max(h_max, cp->h_max); } - } else { + } else if (ti_current >= ti_old) { /* Loop over all the g-particles in the cell */ const size_t nr_gparts = c->gcount; @@ -1045,7 +1057,7 @@ void cell_drift(struct cell *c, const struct engine *e) { /* Drift... */ drift_part(p, xp, dt, timeBase, ti_old, ti_current); - p->ti_old = ti_current; + //p->ti_old = ti_current; /* Compute (square of) motion since last cell construction */ const float dx2 = xp->x_diff[0] * xp->x_diff[0] + @@ -1061,9 +1073,14 @@ void cell_drift(struct cell *c, const struct engine *e) { dx_max = sqrtf(dx2_max); /* Set ti_old on the sub-cells */ - //cell_set_ti_old(c, e->ti_current); + cell_set_ti_old(c, e->ti_current); } /* Check that we are actually going to move forward. */ + else { + + h_max = c->h_max; + dx_max = c->dx_max; + } /* Store the values */ c->h_max = h_max; diff --git a/src/engine.c b/src/engine.c index d6a1dcf8b..065054711 100644 --- a/src/engine.c +++ b/src/engine.c @@ -2262,6 +2262,11 @@ void engine_prepare(struct engine *e, int nodrift) { #endif engine_rebuild(e); + +#ifdef SWIFT_DEBUG_CHECKS + /* Check that all cells have been drifted to the current time */ + space_check_drift_point(e->s, e->ti_current); +#endif } /* Re-rank the tasks every now and then. */ @@ -2605,7 +2610,7 @@ void engine_step(struct engine *e) { snapshot_drift_time = e->timeStep; /* Drift everybody to the snapshot position */ - e->drift_all = 1; + // e->drift_all = 1; engine_drift_all(e); /* Restore the default drifting policy */ @@ -2629,8 +2634,9 @@ void engine_step(struct engine *e) { if (e->nodeID == 0) { /* Print some information to the screen */ - printf(" %6d %14e %14e %10zu %10zu %21.3f\n", e->step, e->time, - e->timeStep, e->updates, e->g_updates, e->wallclock_time); + printf(" %6d %14e %d %14e %10zu %10zu %21.3f\n", e->step, e->time, + e->ti_current, e->timeStep, e->updates, e->g_updates, + e->wallclock_time); fflush(stdout); fprintf(e->file_timesteps, " %6d %14e %14e %10zu %10zu %21.3f\n", e->step, diff --git a/src/hydro/Gadget2/hydro.h b/src/hydro/Gadget2/hydro.h index 157893bc9..a1321da89 100644 --- a/src/hydro/Gadget2/hydro.h +++ b/src/hydro/Gadget2/hydro.h @@ -212,6 +212,7 @@ __attribute__((always_inline)) INLINE static float hydro_compute_timestep( __attribute__((always_inline)) INLINE static void hydro_first_init_part( struct part *restrict p, struct xpart *restrict xp) { + //p->ti_old = 0; p->ti_begin = 0; p->ti_end = 0; xp->v_full[0] = p->v[0]; diff --git a/src/hydro/Gadget2/hydro_part.h b/src/hydro/Gadget2/hydro_part.h index 4bbbf0aed..ee8114b8e 100644 --- a/src/hydro/Gadget2/hydro_part.h +++ b/src/hydro/Gadget2/hydro_part.h @@ -50,6 +50,12 @@ struct xpart { /* Data of a single particle. */ struct part { + /* Particle ID. */ + long long id; + + /* Pointer to corresponding gravity part. */ + struct gpart* gpart; + /* Particle position. */ double x[3]; @@ -71,6 +77,8 @@ struct part { /* Particle time of end of time-step. */ int ti_end; + //int ti_old; + /* Particle density. */ float rho; @@ -80,7 +88,7 @@ struct part { /* Entropy time derivative */ float entropy_dt; - union { + //union { struct { @@ -122,13 +130,7 @@ struct part { float h_dt; } force; - }; - - /* Particle ID. */ - long long id; - - /* Pointer to corresponding gravity part. */ - struct gpart* gpart; + //}; } SWIFT_STRUCT_ALIGN; diff --git a/src/runner.c b/src/runner.c index 334aa5560..975068f34 100644 --- a/src/runner.c +++ b/src/runner.c @@ -733,9 +733,17 @@ void runner_do_ghost(struct runner *r, struct cell *c, int timer) { } } - if (count) + if (count) { message("Smoothing length failed to converge on %i particles.", count); + for(int i=0; iprogeny[k] != NULL) { struct cell *cp = c->progeny[k]; - message("aaa"); + // message("aaa"); /* Recurse. */ - runner_do_unskip(cp, e, drift); + runner_do_unskip(cp, e, 0); #if 0 dx_max = max(dx_max, cp->dx_max); h_max = max(h_max, cp->h_max); @@ -892,7 +900,7 @@ void runner_do_unskip_mapper(void *map_data, int num_elements, #ifdef WITH_MPI if (c != NULL) runner_do_unskip(c, e, (c->nodeID == e->nodeID)); #else - if (c != NULL) runner_do_unskip(c, e, 1); + if (c != NULL) runner_do_unskip(c, e, 0); #endif } } diff --git a/src/runner_doiact.h b/src/runner_doiact.h index 6bc8f2da8..bfc7bcf40 100644 --- a/src/runner_doiact.h +++ b/src/runner_doiact.h @@ -721,10 +721,8 @@ void DOPAIR1(struct runner *r, struct cell *ci, struct cell *cj) { /* Anything to do here? */ if (!cell_is_active(ci, e) && !cell_is_active(cj, e)) return; -#ifdef SWIFT_DEBUG_CHECKS - cell_is_drifted(ci, e); - cell_is_drifted(cj, e); -#endif + if(!cell_is_drifted(ci, e)) cell_drift(ci, e); + if(!cell_is_drifted(cj, e)) cell_drift(cj, e); /* Get the sort ID. */ double shift[3] = {0.0, 0.0, 0.0}; @@ -764,6 +762,8 @@ void DOPAIR1(struct runner *r, struct cell *ci, struct cell *cj) { const double di = sort_i[pid].d + hi * kernel_gamma + dx_max - rshift; if (di < dj_min) continue; + + double pix[3]; for (int k = 0; k < 3; k++) pix[k] = pi->x[k] - shift[k]; const float hig2 = hi * hi * kernel_gamma2; @@ -774,6 +774,7 @@ void DOPAIR1(struct runner *r, struct cell *ci, struct cell *cj) { /* Get a pointer to the jth particle. */ struct part *restrict pj = &parts_j[sort_j[pjd].i]; + /* Compute the pairwise distance. */ float r2 = 0.0f; float dx[3]; @@ -826,6 +827,8 @@ void DOPAIR1(struct runner *r, struct cell *ci, struct cell *cj) { const double dj = sort_j[pjd].d - hj * kernel_gamma - dx_max - rshift; if (dj > di_max) continue; + + double pjx[3]; for (int k = 0; k < 3; k++) pjx[k] = pj->x[k] + shift[k]; const float hjg2 = hj * hj * kernel_gamma2; @@ -836,6 +839,8 @@ void DOPAIR1(struct runner *r, struct cell *ci, struct cell *cj) { /* Get a pointer to the jth particle. */ struct part *restrict pi = &parts_i[sort_i[pid].i]; + + /* Compute the pairwise distance. */ float r2 = 0.0f; float dx[3]; @@ -918,11 +923,9 @@ void DOPAIR2(struct runner *r, struct cell *ci, struct cell *cj) { /* Anything to do here? */ if (!cell_is_active(ci, e) && !cell_is_active(cj, e)) return; -#ifdef SWIFT_DEBUG_CHECKS - cell_is_drifted(ci, e); - cell_is_drifted(cj, e); -#endif - + if(!cell_is_drifted(ci, e)) error("Cell ci not drifted"); + if(!cell_is_drifted(cj, e)) error("Cell cj not drifted"); + /* Get the shift ID. */ double shift[3] = {0.0, 0.0, 0.0}; const int sid = space_getsid(e->s, &ci, &cj, shift); @@ -990,6 +993,8 @@ void DOPAIR2(struct runner *r, struct cell *ci, struct cell *cj) { const double di = sort_i[pid].d + hi * kernel_gamma + dx_max - rshift; if (di < dj_min) continue; + + double pix[3]; for (int k = 0; k < 3; k++) pix[k] = pi->x[k] - shift[k]; const float hig2 = hi * hi * kernel_gamma2; @@ -1004,6 +1009,8 @@ void DOPAIR2(struct runner *r, struct cell *ci, struct cell *cj) { struct part *restrict pj = &parts_j[sortdt_j[pjd].i]; const float hj = pj->h; + + /* Compute the pairwise distance. */ float r2 = 0.0f; float dx[3]; @@ -1055,6 +1062,8 @@ void DOPAIR2(struct runner *r, struct cell *ci, struct cell *cj) { struct part *restrict pj = &parts_j[sort_j[pjd].i]; const float hj = pj->h; + + /* Compute the pairwise distance. */ float r2 = 0.0f; float dx[3]; @@ -1134,6 +1143,8 @@ void DOPAIR2(struct runner *r, struct cell *ci, struct cell *cj) { const double dj = sort_j[pjd].d - hj * kernel_gamma - dx_max - rshift; if (dj > di_max) continue; + + double pjx[3]; for (int k = 0; k < 3; k++) pjx[k] = pj->x[k] + shift[k]; const float hjg2 = hj * hj * kernel_gamma2; @@ -1148,6 +1159,8 @@ void DOPAIR2(struct runner *r, struct cell *ci, struct cell *cj) { struct part *restrict pi = &parts_i[sortdt_i[pid].i]; const float hi = pi->h; + + /* Compute the pairwise distance. */ float r2 = 0.0f; float dx[3]; @@ -1198,6 +1211,8 @@ void DOPAIR2(struct runner *r, struct cell *ci, struct cell *cj) { struct part *restrict pi = &parts_i[sort_i[pid].i]; const float hi = pi->h; + + /* Compute the pairwise distance. */ float r2 = 0.0f; float dx[3]; @@ -1313,9 +1328,7 @@ void DOSELF1(struct runner *r, struct cell *restrict c) { if (!cell_is_active(c, e)) return; -#ifdef SWIFT_DEBUG_CHECKS - cell_is_drifted(c, e); -#endif + if(!cell_is_drifted(c, e)) cell_drift(c, e); struct part *restrict parts = c->parts; const int count = c->count; @@ -1344,6 +1357,8 @@ void DOSELF1(struct runner *r, struct cell *restrict c) { const float hi = pi->h; const float hig2 = hi * hi * kernel_gamma2; + + /* Is the ith particle inactive? */ if (!part_is_active(pi, e)) { @@ -1354,6 +1369,8 @@ void DOSELF1(struct runner *r, struct cell *restrict c) { struct part *restrict pj = &parts[indt[pjd]]; const float hj = pj->h; + + /* Compute the pairwise distance. */ float r2 = 0.0f; float dx[3]; @@ -1408,6 +1425,8 @@ void DOSELF1(struct runner *r, struct cell *restrict c) { struct part *restrict pj = &parts[pjd]; const float hj = pj->h; + + /* Compute the pairwise distance. */ float r2 = 0.0f; float dx[3]; @@ -1548,9 +1567,7 @@ void DOSELF2(struct runner *r, struct cell *restrict c) { if (!cell_is_active(c, e)) return; -#ifdef SWIFT_DEBUG_CHECKS - cell_is_drifted(c, e); -#endif + if(!cell_is_drifted(c, e)) error("Cell is not drifted"); struct part *restrict parts = c->parts; const int count = c->count; @@ -1579,6 +1596,8 @@ void DOSELF2(struct runner *r, struct cell *restrict c) { const float hi = pi->h; const float hig2 = hi * hi * kernel_gamma2; + + /* Is the ith particle not active? */ if (!part_is_active(pi, e)) { @@ -1589,6 +1608,8 @@ void DOSELF2(struct runner *r, struct cell *restrict c) { struct part *restrict pj = &parts[indt[pjd]]; const float hj = pj->h; + + /* Compute the pairwise distance. */ float r2 = 0.0f; float dx[3]; @@ -1643,6 +1664,8 @@ void DOSELF2(struct runner *r, struct cell *restrict c) { struct part *restrict pj = &parts[pjd]; const float hj = pj->h; + + /* Compute the pairwise distance. */ float r2 = 0.0f; float dx[3]; diff --git a/src/scheduler.c b/src/scheduler.c index 55ec764e5..b530b24ae 100644 --- a/src/scheduler.c +++ b/src/scheduler.c @@ -215,7 +215,8 @@ static void scheduler_splittask(struct task *t, struct scheduler *s) { /* Get the sort ID, use space_getsid and not t->flags to make sure we get ci and cj swapped if needed. */ double shift[3]; - int sid = space_getsid(s->space, &ci, &cj, shift); + const int sid = space_getsid(s->space, &ci, &cj, shift); + const int did = space_getdid(s->space, ci, cj); /* Should this task be split-up? */ if (ci->split && cj->split && @@ -584,6 +585,7 @@ static void scheduler_splittask(struct task *t, struct scheduler *s) { } /* Otherwise, break it up if it is too large? */ + } else if (scheduler_doforcesplit && ci->split && cj->split && (ci->count > space_maxsize / cj->count)) { @@ -605,6 +607,7 @@ static void scheduler_splittask(struct task *t, struct scheduler *s) { } /* Otherwise, if not spilt, stitch-up the sorting and drift. */ + } else { /* Create the sort for ci. */ @@ -619,10 +622,21 @@ static void scheduler_splittask(struct task *t, struct scheduler *s) { /* Create the drift for ci. */ if (ci->drift == NULL) { - ci->drift = scheduler_addtask(s, task_type_drift, task_subtype_none, - 0, 0, ci, NULL, 0); - scheduler_addunlock(s, ci->drift, ci->sorts); + // ci->drift = scheduler_addtask(s, task_type_drift, + // task_subtype_none, + // 1 << sid, 0, ci, NULL, 0); + // scheduler_addunlock(s, ci->drift, ci->sorts); + + // scheduler_addunlock(s, ci->drift, t); } + + /* if(!(ci->drift->flags & (1 << sid))) { */ + /* scheduler_addunlock(s, ci->drift, ci->sorts); */ + /* ci->drift->flags |= (1 << sid); */ + /* } */ + + if (did == 0 || did > 31) message("did=%d 1<lock); /* Create the sort for cj. */ @@ -637,10 +651,20 @@ static void scheduler_splittask(struct task *t, struct scheduler *s) { /* Create the drift for cj. */ if (cj->drift == NULL) { - cj->drift = scheduler_addtask(s, task_type_drift, task_subtype_none, - 0, 0, cj, NULL, 0); - scheduler_addunlock(s, cj->drift, cj->sorts); + // cj->drift = scheduler_addtask(s, task_type_drift, + // task_subtype_none, + // 1 << sid, 0, cj, NULL, 0); + // scheduler_addunlock(s, cj->drift, cj->sorts); + + // scheduler_addunlock(s, cj->drift, t); + // scheduler_addunlock(s, cj->drift, cj->init); } + + /* if(!(cj->drift->flags & (1 << sid))) { */ + /* scheduler_addunlock(s, cj->drift, cj->sorts); */ + /* cj->drift->flags |= (1 << sid); */ + /* } */ + lock_unlock_blind(&cj->lock); } diff --git a/src/space.c b/src/space.c index 8665bdbd8..2c67cfce2 100644 --- a/src/space.c +++ b/src/space.c @@ -168,6 +168,32 @@ int space_getsid(struct space *s, struct cell **ci, struct cell **cj, return sid; } +int space_getdid(struct space *s, struct cell *ci, struct cell *cj) { + + /* Get the relative distance between the pairs, wrapping. */ + const int periodic = s->periodic; + double dx[3]; + double shift[3]; + for (int k = 0; k < 3; k++) { + dx[k] = cj->loc[k] - ci->loc[k]; + if (periodic && dx[k] < -s->dim[k] / 2) + shift[k] = s->dim[k]; + else if (periodic && dx[k] > s->dim[k] / 2) + shift[k] = -s->dim[k]; + else + shift[k] = 0.0; + dx[k] += shift[k]; + } + + /* Get the drift index. */ + int did = 0; + for (int k = 0; k < 3; k++) + did = 3 * did + ((dx[k] < 0.0) ? 0 : ((dx[k] > 0.0) ? 2 : 1)); + + /* Return the drift ID. */ + return did; +} + /** * @brief Recursively dismantle a cell tree. * @@ -346,7 +372,7 @@ void space_regrid(struct space *s, int verbose) { c->depth = 0; c->count = 0; c->gcount = 0; - c->super = c; + // c->super = c; c->ti_old = ti_current; lock_init(&c->lock); } @@ -419,6 +445,7 @@ void space_regrid(struct space *s, int verbose) { s->cells_top[k].cooling = NULL; s->cells_top[k].sourceterms = NULL; s->cells_top[k].super = &s->cells_top[k]; + s->cells_top[k].ti_old = 0; #if WITH_MPI s->cells_top[k].recv_xv = NULL; s->cells_top[k].recv_rho = NULL; @@ -1464,7 +1491,6 @@ void space_split_recursive(struct space *s, struct cell *c, int *buff) { struct part *parts = c->parts; struct gpart *gparts = c->gparts; struct xpart *xparts = c->xparts; - struct engine *e = s->e; /* If the buff is NULL, allocate it, and remember to free it. */ const int allocate_buffer = (buff == NULL); @@ -1494,7 +1520,7 @@ void space_split_recursive(struct space *s, struct cell *c, int *buff) { temp = space_getcell(s); temp->count = 0; temp->gcount = 0; - temp->ti_old = e->ti_current; + temp->ti_old = c->ti_old; temp->loc[0] = c->loc[0]; temp->loc[1] = c->loc[1]; temp->loc[2] = c->loc[2]; diff --git a/src/space.h b/src/space.h index 53cf2d0c8..2f53e24ad 100644 --- a/src/space.h +++ b/src/space.h @@ -150,6 +150,7 @@ void space_gparts_sort(struct space *s, int *ind, size_t N, int min, int max, struct cell *space_getcell(struct space *s); int space_getsid(struct space *s, struct cell **ci, struct cell **cj, double *shift); +int space_getdid(struct space *s, struct cell *ci, struct cell *cj); void space_init(struct space *s, const struct swift_params *params, double dim[3], struct part *parts, struct gpart *gparts, size_t Npart, size_t Ngpart, int periodic, int gravity, diff --git a/src/task.c b/src/task.c index d787e92af..71fb0af8a 100644 --- a/src/task.c +++ b/src/task.c @@ -265,7 +265,7 @@ void task_unlock(struct task *t) { cell_unlocktree(ci); cell_gunlocktree(ci); break; - + case task_type_sort: cell_unlocktree(ci); break; @@ -337,8 +337,8 @@ int task_lock(struct task *t) { if (ci->hold || ci->ghold) return 0; if (cell_locktree(ci) != 0) return 0; if (cell_glocktree(ci) != 0) { - cell_unlocktree(ci); - return 0; + cell_unlocktree(ci); + return 0; } break; -- GitLab From ffb2e88d1f6b0cbfc3f06187283d86511a1bc627 Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Fri, 9 Dec 2016 12:31:05 +0000 Subject: [PATCH 10/25] Drift before init for active cells --- src/cell.c | 9 ++++--- src/engine.c | 21 ++++++++++++---- src/hydro/Gadget2/hydro.h | 2 +- src/hydro/Gadget2/hydro_part.h | 10 ++++---- src/runner.c | 6 ++--- src/runner_doiact.h | 45 ++++++---------------------------- src/scheduler.c | 40 ++---------------------------- src/space.c | 26 -------------------- src/space.h | 1 - 9 files changed, 39 insertions(+), 121 deletions(-) diff --git a/src/cell.c b/src/cell.c index 5b7941634..2b34bfcad 100644 --- a/src/cell.c +++ b/src/cell.c @@ -721,7 +721,8 @@ void cell_check_drift_point(struct cell *c, void *data) { /* for (int i = 0; i < c->count; ++i) */ /* if (c->parts[i].ti_old != ti_current) */ /* error( */ - /* "Particle in an incorrect time-zone! part->ti_old=%d c->ti_old=%d " */ + /* "Particle in an incorrect time-zone! part->ti_old=%d c->ti_old=%d " + */ /* "ti_current=%d", */ /* c->parts[i].ti_old, c->ti_old, ti_current); */ } @@ -1009,11 +1010,11 @@ void cell_drift(struct cell *c, const struct engine *e) { const double dt = (ti_current - ti_old) * timeBase; float dx_max = 0.f, dx2_max = 0.f, h_max = 0.f; - //message("DRFIT ! ti_old=%d ti_current=%d", ti_old, ti_current); + // message("DRFIT ! ti_old=%d ti_current=%d", ti_old, ti_current); /* Check that we are actually going to move forward. */ if (ti_current < ti_old) error("Attempt to drift to the past"); - //if (ti_current == ti_old) return; + // if (ti_current == ti_old) return; /* Are we not in a leaf ? */ if (c->split) { @@ -1057,7 +1058,7 @@ void cell_drift(struct cell *c, const struct engine *e) { /* Drift... */ drift_part(p, xp, dt, timeBase, ti_old, ti_current); - //p->ti_old = ti_current; + // p->ti_old = ti_current; /* Compute (square of) motion since last cell construction */ const float dx2 = xp->x_diff[0] * xp->x_diff[0] + diff --git a/src/engine.c b/src/engine.c index 065054711..08811e350 100644 --- a/src/engine.c +++ b/src/engine.c @@ -142,6 +142,11 @@ void engine_make_hierarchical_tasks(struct engine *e, struct cell *c) { c->kick = scheduler_addtask(s, task_type_kick, task_subtype_none, 0, 0, c, NULL, 0); + c->drift = scheduler_addtask(s, task_type_drift, task_subtype_none, 0, 0, + c, NULL, 0); + + scheduler_addunlock(s, c->drift, c->init); + /* Generate the ghost task. */ if (is_hydro) c->ghost = scheduler_addtask(s, task_type_ghost, task_subtype_none, 0, @@ -1987,10 +1992,10 @@ void engine_marktasks_mapper(void *map_data, int num_elements, else if (t->type == task_type_self || t->type == task_type_sub_self) { /* Local pointers. */ - const struct cell *ci = t->ci; + // const struct cell *ci = t->ci; /* Activate the drift */ - if (ci->drift) scheduler_activate(s, ci->drift); + // if (ci->drift) scheduler_activate(s, ci->drift); } /* Pair? */ @@ -2001,8 +2006,8 @@ void engine_marktasks_mapper(void *map_data, int num_elements, const struct cell *cj = t->cj; /* Activate the drift on both sides */ - if (ci->drift) scheduler_activate(s, ci->drift); - if (cj->drift) scheduler_activate(s, cj->drift); + // if (ci->drift) scheduler_activate(s, ci->drift); + // if (cj->drift) scheduler_activate(s, cj->drift); /* Too much particle movement? */ if (t->tight && @@ -2100,6 +2105,11 @@ void engine_marktasks_mapper(void *map_data, int num_elements, if (t->ci->ti_end_min <= ti_end) scheduler_activate(s, t); } + /* Drift? */ + else if (t->type == task_type_drift) { + if (t->ci->ti_end_min <= ti_end) scheduler_activate(s, t); + } + /* Init? */ else if (t->type == task_type_init) { if (t->ci->ti_end_min <= ti_end) scheduler_activate(s, t); @@ -2482,7 +2492,8 @@ void engine_skip_force_and_kick(struct engine *e) { /* Skip everything that updates the particles */ if (t->subtype == task_subtype_force || t->type == task_type_kick || - t->type == task_type_cooling || t->type == task_type_sourceterms) + t->type == task_type_cooling || t->type == task_type_sourceterms || + t->type == task_type_drift) t->skip = 1; } } diff --git a/src/hydro/Gadget2/hydro.h b/src/hydro/Gadget2/hydro.h index a1321da89..c681db18a 100644 --- a/src/hydro/Gadget2/hydro.h +++ b/src/hydro/Gadget2/hydro.h @@ -212,7 +212,7 @@ __attribute__((always_inline)) INLINE static float hydro_compute_timestep( __attribute__((always_inline)) INLINE static void hydro_first_init_part( struct part *restrict p, struct xpart *restrict xp) { - //p->ti_old = 0; + // p->ti_old = 0; p->ti_begin = 0; p->ti_end = 0; xp->v_full[0] = p->v[0]; diff --git a/src/hydro/Gadget2/hydro_part.h b/src/hydro/Gadget2/hydro_part.h index ee8114b8e..61158a71d 100644 --- a/src/hydro/Gadget2/hydro_part.h +++ b/src/hydro/Gadget2/hydro_part.h @@ -55,7 +55,7 @@ struct part { /* Pointer to corresponding gravity part. */ struct gpart* gpart; - + /* Particle position. */ double x[3]; @@ -77,8 +77,8 @@ struct part { /* Particle time of end of time-step. */ int ti_end; - //int ti_old; - + // int ti_old; + /* Particle density. */ float rho; @@ -88,7 +88,7 @@ struct part { /* Entropy time derivative */ float entropy_dt; - //union { + union { struct { @@ -130,7 +130,7 @@ struct part { float h_dt; } force; - //}; + }; } SWIFT_STRUCT_ALIGN; diff --git a/src/runner.c b/src/runner.c index 975068f34..35148d506 100644 --- a/src/runner.c +++ b/src/runner.c @@ -736,11 +736,11 @@ void runner_do_ghost(struct runner *r, struct cell *c, int timer) { if (count) { message("Smoothing length failed to converge on %i particles.", count); - for(int i=0; ix[k] - shift[k]; const float hig2 = hi * hi * kernel_gamma2; @@ -774,7 +772,6 @@ void DOPAIR1(struct runner *r, struct cell *ci, struct cell *cj) { /* Get a pointer to the jth particle. */ struct part *restrict pj = &parts_j[sort_j[pjd].i]; - /* Compute the pairwise distance. */ float r2 = 0.0f; float dx[3]; @@ -827,8 +824,6 @@ void DOPAIR1(struct runner *r, struct cell *ci, struct cell *cj) { const double dj = sort_j[pjd].d - hj * kernel_gamma - dx_max - rshift; if (dj > di_max) continue; - - double pjx[3]; for (int k = 0; k < 3; k++) pjx[k] = pj->x[k] + shift[k]; const float hjg2 = hj * hj * kernel_gamma2; @@ -839,8 +834,6 @@ void DOPAIR1(struct runner *r, struct cell *ci, struct cell *cj) { /* Get a pointer to the jth particle. */ struct part *restrict pi = &parts_i[sort_i[pid].i]; - - /* Compute the pairwise distance. */ float r2 = 0.0f; float dx[3]; @@ -923,9 +916,9 @@ void DOPAIR2(struct runner *r, struct cell *ci, struct cell *cj) { /* Anything to do here? */ if (!cell_is_active(ci, e) && !cell_is_active(cj, e)) return; - if(!cell_is_drifted(ci, e)) error("Cell ci not drifted"); - if(!cell_is_drifted(cj, e)) error("Cell cj not drifted"); - + if (!cell_is_drifted(ci, e)) error("Cell ci not drifted"); + if (!cell_is_drifted(cj, e)) error("Cell cj not drifted"); + /* Get the shift ID. */ double shift[3] = {0.0, 0.0, 0.0}; const int sid = space_getsid(e->s, &ci, &cj, shift); @@ -993,8 +986,6 @@ void DOPAIR2(struct runner *r, struct cell *ci, struct cell *cj) { const double di = sort_i[pid].d + hi * kernel_gamma + dx_max - rshift; if (di < dj_min) continue; - - double pix[3]; for (int k = 0; k < 3; k++) pix[k] = pi->x[k] - shift[k]; const float hig2 = hi * hi * kernel_gamma2; @@ -1009,8 +1000,6 @@ void DOPAIR2(struct runner *r, struct cell *ci, struct cell *cj) { struct part *restrict pj = &parts_j[sortdt_j[pjd].i]; const float hj = pj->h; - - /* Compute the pairwise distance. */ float r2 = 0.0f; float dx[3]; @@ -1062,8 +1051,6 @@ void DOPAIR2(struct runner *r, struct cell *ci, struct cell *cj) { struct part *restrict pj = &parts_j[sort_j[pjd].i]; const float hj = pj->h; - - /* Compute the pairwise distance. */ float r2 = 0.0f; float dx[3]; @@ -1143,8 +1130,6 @@ void DOPAIR2(struct runner *r, struct cell *ci, struct cell *cj) { const double dj = sort_j[pjd].d - hj * kernel_gamma - dx_max - rshift; if (dj > di_max) continue; - - double pjx[3]; for (int k = 0; k < 3; k++) pjx[k] = pj->x[k] + shift[k]; const float hjg2 = hj * hj * kernel_gamma2; @@ -1159,8 +1144,6 @@ void DOPAIR2(struct runner *r, struct cell *ci, struct cell *cj) { struct part *restrict pi = &parts_i[sortdt_i[pid].i]; const float hi = pi->h; - - /* Compute the pairwise distance. */ float r2 = 0.0f; float dx[3]; @@ -1211,8 +1194,6 @@ void DOPAIR2(struct runner *r, struct cell *ci, struct cell *cj) { struct part *restrict pi = &parts_i[sort_i[pid].i]; const float hi = pi->h; - - /* Compute the pairwise distance. */ float r2 = 0.0f; float dx[3]; @@ -1328,7 +1309,7 @@ void DOSELF1(struct runner *r, struct cell *restrict c) { if (!cell_is_active(c, e)) return; - if(!cell_is_drifted(c, e)) cell_drift(c, e); + if (!cell_is_drifted(c, e)) cell_drift(c, e); struct part *restrict parts = c->parts; const int count = c->count; @@ -1357,8 +1338,6 @@ void DOSELF1(struct runner *r, struct cell *restrict c) { const float hi = pi->h; const float hig2 = hi * hi * kernel_gamma2; - - /* Is the ith particle inactive? */ if (!part_is_active(pi, e)) { @@ -1369,8 +1348,6 @@ void DOSELF1(struct runner *r, struct cell *restrict c) { struct part *restrict pj = &parts[indt[pjd]]; const float hj = pj->h; - - /* Compute the pairwise distance. */ float r2 = 0.0f; float dx[3]; @@ -1425,8 +1402,6 @@ void DOSELF1(struct runner *r, struct cell *restrict c) { struct part *restrict pj = &parts[pjd]; const float hj = pj->h; - - /* Compute the pairwise distance. */ float r2 = 0.0f; float dx[3]; @@ -1567,7 +1542,7 @@ void DOSELF2(struct runner *r, struct cell *restrict c) { if (!cell_is_active(c, e)) return; - if(!cell_is_drifted(c, e)) error("Cell is not drifted"); + if (!cell_is_drifted(c, e)) error("Cell is not drifted"); struct part *restrict parts = c->parts; const int count = c->count; @@ -1596,8 +1571,6 @@ void DOSELF2(struct runner *r, struct cell *restrict c) { const float hi = pi->h; const float hig2 = hi * hi * kernel_gamma2; - - /* Is the ith particle not active? */ if (!part_is_active(pi, e)) { @@ -1608,8 +1581,6 @@ void DOSELF2(struct runner *r, struct cell *restrict c) { struct part *restrict pj = &parts[indt[pjd]]; const float hj = pj->h; - - /* Compute the pairwise distance. */ float r2 = 0.0f; float dx[3]; @@ -1664,8 +1635,6 @@ void DOSELF2(struct runner *r, struct cell *restrict c) { struct part *restrict pj = &parts[pjd]; const float hj = pj->h; - - /* Compute the pairwise distance. */ float r2 = 0.0f; float dx[3]; diff --git a/src/scheduler.c b/src/scheduler.c index b530b24ae..82ea453b6 100644 --- a/src/scheduler.c +++ b/src/scheduler.c @@ -216,7 +216,6 @@ static void scheduler_splittask(struct task *t, struct scheduler *s) { to make sure we get ci and cj swapped if needed. */ double shift[3]; const int sid = space_getsid(s->space, &ci, &cj, shift); - const int did = space_getdid(s->space, ci, cj); /* Should this task be split-up? */ if (ci->split && cj->split && @@ -617,28 +616,10 @@ static void scheduler_splittask(struct task *t, struct scheduler *s) { 1 << sid, 0, ci, NULL, 0); else ci->sorts->flags |= (1 << sid); + lock_unlock_blind(&ci->lock); scheduler_addunlock(s, ci->sorts, t); - /* Create the drift for ci. */ - if (ci->drift == NULL) { - // ci->drift = scheduler_addtask(s, task_type_drift, - // task_subtype_none, - // 1 << sid, 0, ci, NULL, 0); - // scheduler_addunlock(s, ci->drift, ci->sorts); - - // scheduler_addunlock(s, ci->drift, t); - } - - /* if(!(ci->drift->flags & (1 << sid))) { */ - /* scheduler_addunlock(s, ci->drift, ci->sorts); */ - /* ci->drift->flags |= (1 << sid); */ - /* } */ - - if (did == 0 || did > 31) message("did=%d 1<lock); - /* Create the sort for cj. */ lock_lock(&cj->lock); if (cj->sorts == NULL) @@ -646,26 +627,9 @@ static void scheduler_splittask(struct task *t, struct scheduler *s) { 1 << sid, 0, cj, NULL, 0); else cj->sorts->flags |= (1 << sid); + lock_unlock_blind(&cj->lock); scheduler_addunlock(s, cj->sorts, t); - - /* Create the drift for cj. */ - if (cj->drift == NULL) { - // cj->drift = scheduler_addtask(s, task_type_drift, - // task_subtype_none, - // 1 << sid, 0, cj, NULL, 0); - // scheduler_addunlock(s, cj->drift, cj->sorts); - - // scheduler_addunlock(s, cj->drift, t); - // scheduler_addunlock(s, cj->drift, cj->init); - } - - /* if(!(cj->drift->flags & (1 << sid))) { */ - /* scheduler_addunlock(s, cj->drift, cj->sorts); */ - /* cj->drift->flags |= (1 << sid); */ - /* } */ - - lock_unlock_blind(&cj->lock); } } /* pair interaction? */ diff --git a/src/space.c b/src/space.c index 2c67cfce2..145b0bf8b 100644 --- a/src/space.c +++ b/src/space.c @@ -168,32 +168,6 @@ int space_getsid(struct space *s, struct cell **ci, struct cell **cj, return sid; } -int space_getdid(struct space *s, struct cell *ci, struct cell *cj) { - - /* Get the relative distance between the pairs, wrapping. */ - const int periodic = s->periodic; - double dx[3]; - double shift[3]; - for (int k = 0; k < 3; k++) { - dx[k] = cj->loc[k] - ci->loc[k]; - if (periodic && dx[k] < -s->dim[k] / 2) - shift[k] = s->dim[k]; - else if (periodic && dx[k] > s->dim[k] / 2) - shift[k] = -s->dim[k]; - else - shift[k] = 0.0; - dx[k] += shift[k]; - } - - /* Get the drift index. */ - int did = 0; - for (int k = 0; k < 3; k++) - did = 3 * did + ((dx[k] < 0.0) ? 0 : ((dx[k] > 0.0) ? 2 : 1)); - - /* Return the drift ID. */ - return did; -} - /** * @brief Recursively dismantle a cell tree. * diff --git a/src/space.h b/src/space.h index 2f53e24ad..53cf2d0c8 100644 --- a/src/space.h +++ b/src/space.h @@ -150,7 +150,6 @@ void space_gparts_sort(struct space *s, int *ind, size_t N, int min, int max, struct cell *space_getcell(struct space *s); int space_getsid(struct space *s, struct cell **ci, struct cell **cj, double *shift); -int space_getdid(struct space *s, struct cell *ci, struct cell *cj); void space_init(struct space *s, const struct swift_params *params, double dim[3], struct part *parts, struct gpart *gparts, size_t Npart, size_t Ngpart, int periodic, int gravity, -- GitLab From cb30d6f1cb478dd7bc3b0ca1dc00ee5a8ba7127e Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Fri, 9 Dec 2016 12:48:21 +0000 Subject: [PATCH 11/25] Simplified unskiping recursion --- src/runner.c | 115 +++------------------------------------------------ 1 file changed, 6 insertions(+), 109 deletions(-) diff --git a/src/runner.c b/src/runner.c index 35148d506..baacd5c5a 100644 --- a/src/runner.c +++ b/src/runner.c @@ -768,118 +768,15 @@ static void runner_do_unskip(struct cell *c, struct engine *e, int drift) { if (forcerebuild) atomic_inc(&e->forcerebuild); } - /* Do we really need to drift? */ - if (drift) { - if (!e->drift_all && !cell_is_drift_needed(c, e)) return; - } else { - - /* Not drifting, but may still need to recurse for task un-skipping. */ - if (c->split) { - for (int k = 0; k < 8; k++) { - if (c->progeny[k] != NULL) { - struct cell *cp = c->progeny[k]; - runner_do_unskip(cp, e, 0); - } - } - } - return; - } - - /* Now, we can drift */ - /* Get some information first */ - // const double timeBase = e->timeBase; - const int ti_old = c->ti_old; - const int ti_current = e->ti_current; - // struct part *const parts = c->parts; - // struct xpart *const xparts = c->xparts; - // struct gpart *const gparts = c->gparts; - - /* Drift from the last time the cell was drifted to the current time */ - // const double dt = (ti_current - ti_old) * timeBase; - // float dx_max = 0.f, dx2_max = 0.f, h_max = 0.f; - - /* No children? */ - if (!c->split) { - - /* Check that we are actually going to move forward. */ - if (ti_current > ti_old) { -#if 0 - /* Loop over all the g-particles in the cell */ - const size_t nr_gparts = c->gcount; - for (size_t k = 0; k < nr_gparts; k++) { - - /* Get a handle on the gpart. */ - struct gpart *const gp = &gparts[k]; - - /* Drift... */ - drift_gpart(gp, dt, timeBase, ti_old, ti_current); - - /* Compute (square of) motion since last cell construction */ - const float dx2 = gp->x_diff[0] * gp->x_diff[0] + - gp->x_diff[1] * gp->x_diff[1] + - gp->x_diff[2] * gp->x_diff[2]; - dx2_max = (dx2_max > dx2) ? dx2_max : dx2; - } - - /* Loop over all the particles in the cell */ - const size_t nr_parts = c->count; - for (size_t k = 0; k < nr_parts; k++) { - - /* Get a handle on the part. */ - struct part *const p = &parts[k]; - struct xpart *const xp = &xparts[k]; - - /* Drift... */ - drift_part(p, xp, dt, timeBase, ti_old, ti_current); - - /* Compute (square of) motion since last cell construction */ - const float dx2 = xp->x_diff[0] * xp->x_diff[0] + - xp->x_diff[1] * xp->x_diff[1] + - xp->x_diff[2] * xp->x_diff[2]; - dx2_max = (dx2_max > dx2) ? dx2_max : dx2; - - /* Maximal smoothing length */ - h_max = (h_max > p->h) ? h_max : p->h; - } - - /* Now, get the maximal particle motion from its square */ - dx_max = sqrtf(dx2_max); -#endif - } /* Check that we are actually going to move forward. */ - - else { -#if 0 - /* ti_old == ti_current, just keep the current cell values. */ - h_max = c->h_max; - dx_max = c->dx_max; -#endif - } - } - - /* Otherwise, aggregate data from children. */ - else { - - /* Loop over the progeny and collect their data. */ - for (int k = 0; k < 8; k++) + /* Not drifting, but may still need to recurse for task un-skipping. */ + if (c->split) { + for (int k = 0; k < 8; k++) { if (c->progeny[k] != NULL) { - struct cell *cp = c->progeny[k]; - // message("aaa"); - /* Recurse. */ - runner_do_unskip(cp, e, 0); -#if 0 - dx_max = max(dx_max, cp->dx_max); - h_max = max(h_max, cp->h_max); -#endif + struct cell *cp = c->progeny[k]; + runner_do_unskip(cp, e, 0); } + } } -#if 0 - /* Store the values */ - c->h_max = h_max; - c->dx_max = dx_max; - - /* Update the time of the last drift */ - c->ti_old = ti_current; -#endif } /** -- GitLab From b3dbade26b4de9cb90ef770c627473a54c47f5d8 Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Fri, 9 Dec 2016 12:55:31 +0000 Subject: [PATCH 12/25] Cleaning up --- examples/SedovBlast_3D/sedov.yml | 3 --- src/active.h | 26 ------------------ src/cell.c | 15 +---------- src/engine.c | 45 +++----------------------------- src/engine.h | 3 --- src/runner.c | 14 +++------- 6 files changed, 8 insertions(+), 98 deletions(-) diff --git a/examples/SedovBlast_3D/sedov.yml b/examples/SedovBlast_3D/sedov.yml index dc2295037..1cc4aced9 100644 --- a/examples/SedovBlast_3D/sedov.yml +++ b/examples/SedovBlast_3D/sedov.yml @@ -6,9 +6,6 @@ InternalUnitSystem: UnitCurrent_in_cgs: 1 # Amperes UnitTemp_in_cgs: 1 # Kelvin -#Scheduler: -# max_top_level_cells: 3 - # Parameters governing the time integration TimeIntegration: time_begin: 0. # The starting time of the simulation (in internal units). diff --git a/src/active.h b/src/active.h index 903b3c092..df7cbd6a6 100644 --- a/src/active.h +++ b/src/active.h @@ -47,32 +47,6 @@ __attribute__((always_inline)) INLINE static int cell_is_drifted( c->ti_old, e->ti_current); #endif - /* if (c->ti_old != e->ti_current) { */ - - /* int wrong = 0; */ - /* for (int i = 0; i < c->count; ++i) { */ - /* if (c->parts[i].ti_old < e->ti_current) ++wrong; */ - /* } */ - - /* message( */ - /* "Cell has not been drifted to the current time c->ti_old=%d, " */ - /* "e->ti_current=%d wrong=%d c->count=%d c->drift=%p, c->depth=%d, - * c=%p, c->super=%p, c->parent=%p ", */ - /* c->ti_old, e->ti_current, wrong, c->count, c->drift, c->depth, c, - * c->super, c->parent); */ - - /* cell_drift((struct cell*)c, e); */ - - /* message( */ - /* "Cell has not been drifted to the current time c->ti_old=%d, " */ - /* "e->ti_current=%d wrong=%d c->count=%d c->drift=%p, c->depth=%d, - * c=%p, c->super=%p, c->parent=%p ", */ - /* c->ti_old, e->ti_current, wrong, c->count, c->drift, c->depth, c, - * c->super, c->parent); */ - - /* error("AAAAA"); */ - /* } */ - return (c->ti_old == e->ti_current); } diff --git a/src/cell.c b/src/cell.c index 2b34bfcad..e6fa623e0 100644 --- a/src/cell.c +++ b/src/cell.c @@ -717,14 +717,6 @@ void cell_check_drift_point(struct cell *c, void *data) { if (c->ti_old != ti_current) error("Cell in an incorrect time-zone! c->ti_old=%d ti_current=%d", c->ti_old, ti_current); - - /* for (int i = 0; i < c->count; ++i) */ - /* if (c->parts[i].ti_old != ti_current) */ - /* error( */ - /* "Particle in an incorrect time-zone! part->ti_old=%d c->ti_old=%d " - */ - /* "ti_current=%d", */ - /* c->parts[i].ti_old, c->ti_old, ti_current); */ } /** @@ -887,12 +879,6 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) { } } - /* Activate the drift on both sides */ - if (ci == c && cj != NULL && cj->drift != NULL) - scheduler_activate(s, cj->drift); - if (cj == c && ci != NULL && ci->drift != NULL) - scheduler_activate(s, ci->drift); - /* Check whether there was too much particle motion */ if (t->type == task_type_pair || t->type == task_type_sub_pair) { if (t->tight && @@ -970,6 +956,7 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) { if (c->extra_ghost != NULL) scheduler_activate(s, c->extra_ghost); if (c->ghost != NULL) scheduler_activate(s, c->ghost); if (c->init != NULL) scheduler_activate(s, c->init); + if (c->drift != NULL) scheduler_activate(s, c->drift); if (c->kick != NULL) scheduler_activate(s, c->kick); if (c->cooling != NULL) scheduler_activate(s, c->cooling); if (c->sourceterms != NULL) scheduler_activate(s, c->sourceterms); diff --git a/src/engine.c b/src/engine.c index 08811e350..1215e789c 100644 --- a/src/engine.c +++ b/src/engine.c @@ -1988,16 +1988,6 @@ void engine_marktasks_mapper(void *map_data, int num_elements, if (t->ci->ti_end_min <= ti_end) scheduler_activate(s, t); } - /* Self? */ - else if (t->type == task_type_self || t->type == task_type_sub_self) { - - /* Local pointers. */ - // const struct cell *ci = t->ci; - - /* Activate the drift */ - // if (ci->drift) scheduler_activate(s, ci->drift); - } - /* Pair? */ else if (t->type == task_type_pair || t->type == task_type_sub_pair) { @@ -2005,10 +1995,6 @@ void engine_marktasks_mapper(void *map_data, int num_elements, const struct cell *ci = t->ci; const struct cell *cj = t->cj; - /* Activate the drift on both sides */ - // if (ci->drift) scheduler_activate(s, ci->drift); - // if (cj->drift) scheduler_activate(s, cj->drift); - /* Too much particle movement? */ if (t->tight && (max(ci->h_max, cj->h_max) + ci->dx_max + cj->dx_max > cj->dmin || @@ -2258,13 +2244,7 @@ void engine_prepare(struct engine *e, int nodrift) { if (rebuild) { /* Drift all particles to the current time if needed. */ - if (!nodrift) { - // e->drift_all = 1; - engine_drift_all(e); - - /* Restore the default drifting policy */ - // e->drift_all = (e->policy & engine_policy_drift_all); - } + if (!nodrift) engine_drift_all(e); #ifdef SWIFT_DEBUG_CHECKS /* Check that all cells have been drifted to the current time */ @@ -2272,11 +2252,6 @@ void engine_prepare(struct engine *e, int nodrift) { #endif engine_rebuild(e); - -#ifdef SWIFT_DEBUG_CHECKS - /* Check that all cells have been drifted to the current time */ - space_check_drift_point(e->s, e->ti_current); -#endif } /* Re-rank the tasks every now and then. */ @@ -2621,12 +2596,8 @@ void engine_step(struct engine *e) { snapshot_drift_time = e->timeStep; /* Drift everybody to the snapshot position */ - // e->drift_all = 1; engine_drift_all(e); - /* Restore the default drifting policy */ - // e->drift_all = (e->policy & engine_policy_drift_all); - /* Dump... */ engine_dump_snapshot(e); @@ -2658,17 +2629,14 @@ void engine_step(struct engine *e) { /* Drift only the necessary particles, that means all particles * if we are about to repartition. */ const int repart = (e->forcerepart != REPART_NONE); - // e->drift_all = repart || e->drift_all; - if (repart) engine_drift_all(e); + const int drift_all = (e->policy & engine_policy_drift_all); + if (repart || drift_all) engine_drift_all(e); /* Re-distribute the particles amongst the nodes? */ if (repart) engine_repartition(e); /* Prepare the space. */ - engine_prepare(e, (e->drift_all || repart)); - - /* Restore the default drifting policy */ - // e->drift_all = (e->policy & engine_policy_drift_all); + engine_prepare(e, (drift_all || repart)); if (e->verbose) engine_print_task_counts(e); @@ -2716,14 +2684,10 @@ void engine_unskip(struct engine *e) { void engine_drift_all(struct engine *e) { - e->drift_all = 1; - const ticks tic = getticks(); threadpool_map(&e->threadpool, runner_do_drift_mapper, e->s->cells_top, e->s->nr_cells, sizeof(struct cell), 1, e); - e->drift_all = e->policy & engine_policy_drift_all; - if (e->verbose) message("took %.3f %s.", clocks_from_ticks(getticks() - tic), clocks_getunit()); @@ -3064,7 +3028,6 @@ void engine_init(struct engine *e, struct space *s, e->timeStep = 0.; e->timeBase = 0.; e->timeBase_inv = 0.; - e->drift_all = (policy & engine_policy_drift_all); e->internalUnits = internal_units; e->timeFirstSnapshot = parser_get_param_double(params, "Snapshots:time_first"); diff --git a/src/engine.h b/src/engine.h index 1960b3dc4..5d72c23cd 100644 --- a/src/engine.h +++ b/src/engine.h @@ -133,9 +133,6 @@ struct engine { /* Minimal ti_end for the next time-step */ int ti_end_min; - /* Are we drifting all particles now ? */ - int drift_all; - /* Number of particles updated */ size_t updates, g_updates; diff --git a/src/runner.c b/src/runner.c index baacd5c5a..3865b1b2a 100644 --- a/src/runner.c +++ b/src/runner.c @@ -733,17 +733,9 @@ void runner_do_ghost(struct runner *r, struct cell *c, int timer) { } } - if (count) { + if (count) message("Smoothing length failed to converge on %i particles.", count); - for (int i = 0; i < count; ++i) { - struct part *restrict p = &parts[pid[i]]; - struct xpart *restrict xp = &xparts[pid[i]]; - - printParticle_single(p, xp); - } - } - /* Be clean */ free(pid); } @@ -772,8 +764,8 @@ static void runner_do_unskip(struct cell *c, struct engine *e, int drift) { if (c->split) { for (int k = 0; k < 8; k++) { if (c->progeny[k] != NULL) { - struct cell *cp = c->progeny[k]; - runner_do_unskip(cp, e, 0); + struct cell *cp = c->progeny[k]; + runner_do_unskip(cp, e, 0); } } } -- GitLab From a7485b7b7a4270ea52d203acc9cee801438a91ae Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Fri, 9 Dec 2016 15:50:26 +0100 Subject: [PATCH 13/25] More cleaning-up, documentation and no unwanted changes. --- src/cell.c | 6 +++++ src/engine.c | 5 ++-- src/hydro/Gadget2/hydro.h | 1 - src/hydro/Gadget2/hydro_part.h | 14 +++++------ src/runner.c | 45 ++++++++++++++++++++-------------- src/scheduler.c | 14 ++--------- 6 files changed, 42 insertions(+), 43 deletions(-) diff --git a/src/cell.c b/src/cell.c index e6fa623e0..f9b6fbedc 100644 --- a/src/cell.c +++ b/src/cell.c @@ -984,6 +984,12 @@ void cell_set_super(struct cell *c, struct cell *super) { if (c->progeny[k] != NULL) cell_set_super(c->progeny[k], super); } +/** + * @brief Recursively drifts all particles and g-particles in a cell hierarchy. + * + * @param c The #cell. + * @param e The #engine (to get ti_current). + */ void cell_drift(struct cell *c, const struct engine *e) { const double timeBase = e->timeBase; diff --git a/src/engine.c b/src/engine.c index 1215e789c..a7f4d7d60 100644 --- a/src/engine.c +++ b/src/engine.c @@ -2616,9 +2616,8 @@ void engine_step(struct engine *e) { if (e->nodeID == 0) { /* Print some information to the screen */ - printf(" %6d %14e %d %14e %10zu %10zu %21.3f\n", e->step, e->time, - e->ti_current, e->timeStep, e->updates, e->g_updates, - e->wallclock_time); + printf(" %6d %14e %14e %10zu %10zu %21.3f\n", e->step, e->time, + e->timeStep, e->updates, e->g_updates, e->wallclock_time); fflush(stdout); fprintf(e->file_timesteps, " %6d %14e %14e %10zu %10zu %21.3f\n", e->step, diff --git a/src/hydro/Gadget2/hydro.h b/src/hydro/Gadget2/hydro.h index c681db18a..157893bc9 100644 --- a/src/hydro/Gadget2/hydro.h +++ b/src/hydro/Gadget2/hydro.h @@ -212,7 +212,6 @@ __attribute__((always_inline)) INLINE static float hydro_compute_timestep( __attribute__((always_inline)) INLINE static void hydro_first_init_part( struct part *restrict p, struct xpart *restrict xp) { - // p->ti_old = 0; p->ti_begin = 0; p->ti_end = 0; xp->v_full[0] = p->v[0]; diff --git a/src/hydro/Gadget2/hydro_part.h b/src/hydro/Gadget2/hydro_part.h index 61158a71d..4bbbf0aed 100644 --- a/src/hydro/Gadget2/hydro_part.h +++ b/src/hydro/Gadget2/hydro_part.h @@ -50,12 +50,6 @@ struct xpart { /* Data of a single particle. */ struct part { - /* Particle ID. */ - long long id; - - /* Pointer to corresponding gravity part. */ - struct gpart* gpart; - /* Particle position. */ double x[3]; @@ -77,8 +71,6 @@ struct part { /* Particle time of end of time-step. */ int ti_end; - // int ti_old; - /* Particle density. */ float rho; @@ -132,6 +124,12 @@ struct part { } force; }; + /* Particle ID. */ + long long id; + + /* Pointer to corresponding gravity part. */ + struct gpart* gpart; + } SWIFT_STRUCT_ALIGN; #endif /* SWIFT_GADGET2_HYDRO_PART_H */ diff --git a/src/runner.c b/src/runner.c index 3865b1b2a..11e66b0e6 100644 --- a/src/runner.c +++ b/src/runner.c @@ -744,15 +744,12 @@ void runner_do_ghost(struct runner *r, struct cell *c, int timer) { } /** - * @brief Drift particles and g-particles in a cell forward in time, - * unskipping any tasks associated with active cells. + * @brief Unskip any tasks associated with active cells. * * @param c The cell. * @param e The engine. - * @param drift whether to actually drift the particles, will not be - * necessary for non-local cells. */ -static void runner_do_unskip(struct cell *c, struct engine *e, int drift) { +static void runner_do_unskip(struct cell *c, struct engine *e) { /* Unskip any active tasks. */ if (cell_is_active(c, e)) { @@ -765,14 +762,14 @@ static void runner_do_unskip(struct cell *c, struct engine *e, int drift) { for (int k = 0; k < 8; k++) { if (c->progeny[k] != NULL) { struct cell *cp = c->progeny[k]; - runner_do_unskip(cp, e, 0); + runner_do_unskip(cp, e); } } } } /** - * @brief Mapper function to drift particles and g-particles forward in time. + * @brief Mapper function to unskip active tasks. * * @param map_data An array of #cell%s. * @param num_elements Chunk size. @@ -787,18 +784,35 @@ void runner_do_unskip_mapper(void *map_data, int num_elements, for (int ind = 0; ind < num_elements; ind++) { struct cell *c = &cells[ind]; #ifdef WITH_MPI - if (c != NULL) runner_do_unskip(c, e, (c->nodeID == e->nodeID)); + if (c != NULL) runner_do_unskip(c, e); #else - if (c != NULL) runner_do_unskip(c, e, 0); + if (c != NULL) runner_do_unskip(c, e); #endif } } - +/** + * @brief Drift particles in real space. + * + * @param r The runner thread. + * @param c The cell. + * @param timer Are we timing this ? + */ void runner_do_drift(struct runner *r, struct cell *c, int timer) { + TIMER_TIC; + cell_drift(c, r->e); + + if (timer) TIMER_TOC(timer_drift); } +/** + * @brief Mapper function to drift ALL particle and g-particles forward in time. + * + * @param map_data An array of #cell%s. + * @param num_elements Chunk size. + * @param extra_data Pointer to an #engine. + */ void runner_do_drift_mapper(void *map_data, int num_elements, void *extra_data) { @@ -812,8 +826,7 @@ void runner_do_drift_mapper(void *map_data, int num_elements, } /** - * @brief Kick particles in momentum space and collect statistics (floating - * time-step case) + * @brief Kick particles in momentum space and collect statistics. * * @param r The runner thread. * @param c The cell. @@ -1043,8 +1056,7 @@ void *runner_main(void *data) { /* Check that we haven't scheduled an inactive task */ #ifdef SWIFT_DEBUG_CHECKS if (cj == NULL) { /* self */ - if (!cell_is_active(ci, e) && t->type != task_type_sort && - t->type != task_type_drift) + if (!cell_is_active(ci, e) && t->type != task_type_sort) error( "Task (type='%s/%s') should have been skipped ti_current=%d " "c->ti_end_min=%d", @@ -1060,11 +1072,6 @@ void *runner_main(void *data) { taskID_names[t->type], subtaskID_names[t->subtype], e->ti_current, ci->ti_end_min, t->flags); - /* Special treatement for drifts */ - if (!cell_is_active(ci, e) && t->type == task_type_drift) { - ; - } - } else { /* pair */ if (!cell_is_active(ci, e) && !cell_is_active(cj, e)) error( diff --git a/src/scheduler.c b/src/scheduler.c index 82ea453b6..df50f8525 100644 --- a/src/scheduler.c +++ b/src/scheduler.c @@ -584,7 +584,6 @@ static void scheduler_splittask(struct task *t, struct scheduler *s) { } /* Otherwise, break it up if it is too large? */ - } else if (scheduler_doforcesplit && ci->split && cj->split && (ci->count > space_maxsize / cj->count)) { @@ -605,8 +604,7 @@ static void scheduler_splittask(struct task *t, struct scheduler *s) { tl->flags = space_getsid(s->space, &t->ci, &t->cj, shift); } - /* Otherwise, if not spilt, stitch-up the sorting and drift. */ - + /* Otherwise, if not spilt, stitch-up the sorting. */ } else { /* Create the sort for ci. */ @@ -617,7 +615,6 @@ static void scheduler_splittask(struct task *t, struct scheduler *s) { else ci->sorts->flags |= (1 << sid); lock_unlock_blind(&ci->lock); - scheduler_addunlock(s, ci->sorts, t); /* Create the sort for cj. */ @@ -628,7 +625,6 @@ static void scheduler_splittask(struct task *t, struct scheduler *s) { else cj->sorts->flags |= (1 << sid); lock_unlock_blind(&cj->lock); - scheduler_addunlock(s, cj->sorts, t); } @@ -1073,7 +1069,7 @@ void scheduler_start(struct scheduler *s) { if (cj == NULL) { /* self */ if (ci->ti_end_min == ti_current && t->skip && - t->type != task_type_sort && t->type != task_type_drift) + t->type != task_type_sort && t->type) error( "Task (type='%s/%s') should not have been skipped ti_current=%d " "c->ti_end_min=%d", @@ -1089,12 +1085,6 @@ void scheduler_start(struct scheduler *s) { taskID_names[t->type], subtaskID_names[t->subtype], ti_current, ci->ti_end_min, t->flags); - /* Special treatement for drifts */ - if (ci->ti_end_min == ti_current && t->skip && - t->type == task_type_drift) { - ; - } - } else { /* pair */ if ((ci->ti_end_min == ti_current || cj->ti_end_min == ti_current) && -- GitLab From 6676d2615203a04025acbf72982f82f0a6e7539b Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Fri, 9 Dec 2016 15:59:18 +0100 Subject: [PATCH 14/25] No unused funtion --- src/cell.c | 33 +-------------------------------- src/cell.h | 2 +- src/space.c | 2 +- 3 files changed, 3 insertions(+), 34 deletions(-) diff --git a/src/cell.c b/src/cell.c index f9b6fbedc..0dbd38a6a 100644 --- a/src/cell.c +++ b/src/cell.c @@ -1003,11 +1003,8 @@ void cell_drift(struct cell *c, const struct engine *e) { const double dt = (ti_current - ti_old) * timeBase; float dx_max = 0.f, dx2_max = 0.f, h_max = 0.f; - // message("DRFIT ! ti_old=%d ti_current=%d", ti_old, ti_current); - /* Check that we are actually going to move forward. */ if (ti_current < ti_old) error("Attempt to drift to the past"); - // if (ti_current == ti_old) return; /* Are we not in a leaf ? */ if (c->split) { @@ -1051,8 +1048,6 @@ void cell_drift(struct cell *c, const struct engine *e) { /* Drift... */ drift_part(p, xp, dt, timeBase, ti_old, ti_current); - // p->ti_old = ti_current; - /* Compute (square of) motion since last cell construction */ const float dx2 = xp->x_diff[0] * xp->x_diff[0] + xp->x_diff[1] * xp->x_diff[1] + @@ -1066,11 +1061,7 @@ void cell_drift(struct cell *c, const struct engine *e) { /* Now, get the maximal particle motion from its square */ dx_max = sqrtf(dx2_max); - /* Set ti_old on the sub-cells */ - cell_set_ti_old(c, e->ti_current); - - } /* Check that we are actually going to move forward. */ - else { + } else { h_max = c->h_max; dx_max = c->dx_max; @@ -1083,25 +1074,3 @@ void cell_drift(struct cell *c, const struct engine *e) { /* Update the time of the last drift */ c->ti_old = ti_current; } - -/** - * Set ti_old of a #cell and all its progenies to a new value. - * - * @param c The #cell. - * @param ti_current The new value of ti_old. - */ -void cell_set_ti_old(struct cell *c, int ti_current) { - - /* Set this cell */ - c->ti_old = ti_current; - - /* Recurse */ - if (c->split) { - for (int k = 0; k < 8; ++k) { - if (c->progeny[k] != NULL) { - struct cell *cp = c->progeny[k]; - cell_set_ti_old(cp, ti_current); - } - } - } -} diff --git a/src/cell.h b/src/cell.h index d55d15c88..08dc568c9 100644 --- a/src/cell.h +++ b/src/cell.h @@ -299,5 +299,5 @@ int cell_is_drift_needed(struct cell *c, const struct engine *e); int cell_unskip_tasks(struct cell *c, struct scheduler *s); void cell_set_super(struct cell *c, struct cell *super); void cell_drift(struct cell *c, const struct engine *e); -void cell_set_ti_old(struct cell *c, int ti_current); + #endif /* SWIFT_CELL_H */ diff --git a/src/space.c b/src/space.c index 145b0bf8b..48f9fd353 100644 --- a/src/space.c +++ b/src/space.c @@ -346,7 +346,7 @@ void space_regrid(struct space *s, int verbose) { c->depth = 0; c->count = 0; c->gcount = 0; - // c->super = c; + c->super = c; c->ti_old = ti_current; lock_init(&c->lock); } -- GitLab From c4af3c0da93a86fefd2c834733b08e48a96ac2ee Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Wed, 14 Dec 2016 13:58:54 +0000 Subject: [PATCH 15/25] Corrected documentation and removed MPI exception. --- src/active.h | 8 +++++--- src/engine.c | 7 ++++++- src/runner.c | 6 +----- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/active.h b/src/active.h index df7cbd6a6..e34ba0961 100644 --- a/src/active.h +++ b/src/active.h @@ -30,11 +30,9 @@ /** * @brief Check that a cell been drifted to the current time. * - * Only used for debugging. Calls error() if the cell has not - * been drifted. Does nothing if SWIFT_DEBUG_CHECKS is not defined. - * * @param c The #cell. * @param e The #engine containing information about the current time. + * @return 1 if the #cell has been drifted to the current time, 0 otherwise. */ __attribute__((always_inline)) INLINE static int cell_is_drifted( const struct cell *c, const struct engine *e) { @@ -55,6 +53,7 @@ __attribute__((always_inline)) INLINE static int cell_is_drifted( * * @param c The #cell. * @param e The #engine containing information about the current time. + * @param 1 if the #cell contains at least an active particle, 0 otherwise. */ __attribute__((always_inline)) INLINE static int cell_is_active( const struct cell *c, const struct engine *e) { @@ -73,6 +72,7 @@ __attribute__((always_inline)) INLINE static int cell_is_active( * * @param c The #cell. * @param e The #engine containing information about the current time. + * @param 1 if all particles in a #cell are active, 0 otherwise. */ __attribute__((always_inline)) INLINE static int cell_is_all_active( const struct cell *c, const struct engine *e) { @@ -91,6 +91,7 @@ __attribute__((always_inline)) INLINE static int cell_is_all_active( * * @param p The #part. * @param e The #engine containing information about the current time. + * @return 1 if the #part is active, 0 otherwise. */ __attribute__((always_inline)) INLINE static int part_is_active( const struct part *p, const struct engine *e) { @@ -109,6 +110,7 @@ __attribute__((always_inline)) INLINE static int part_is_active( * * @param gp The #gpart. * @param e The #engine containing information about the current time. + * @return 1 if the #gpart is active, 0 otherwise. */ __attribute__((always_inline)) INLINE static int gpart_is_active( const struct gpart *gp, const struct engine *e) { diff --git a/src/engine.c b/src/engine.c index fb14f07b9..fb5f24eb4 100644 --- a/src/engine.c +++ b/src/engine.c @@ -2684,7 +2684,7 @@ int engine_is_done(struct engine *e) { } /** - * @brief Drift particles using the current engine drift policy. + * @brief Unskip all the tasks that act on active cells at this time. * * @param e The #engine. */ @@ -2699,6 +2699,11 @@ void engine_unskip(struct engine *e) { clocks_getunit()); } +/** + * @brief Drift *all* particles forward to the current time. + * + * @param e The #engine. + */ void engine_drift_all(struct engine *e) { const ticks tic = getticks(); diff --git a/src/runner.c b/src/runner.c index 11e66b0e6..24003d960 100644 --- a/src/runner.c +++ b/src/runner.c @@ -757,7 +757,7 @@ static void runner_do_unskip(struct cell *c, struct engine *e) { if (forcerebuild) atomic_inc(&e->forcerebuild); } - /* Not drifting, but may still need to recurse for task un-skipping. */ + /* Recurse */ if (c->split) { for (int k = 0; k < 8; k++) { if (c->progeny[k] != NULL) { @@ -783,11 +783,7 @@ void runner_do_unskip_mapper(void *map_data, int num_elements, for (int ind = 0; ind < num_elements; ind++) { struct cell *c = &cells[ind]; -#ifdef WITH_MPI - if (c != NULL) runner_do_unskip(c, e); -#else if (c != NULL) runner_do_unskip(c, e); -#endif } } /** -- GitLab From 24555216c534e47ffb588c2b5ee535ff3f6be81e Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Wed, 14 Dec 2016 17:43:53 +0000 Subject: [PATCH 16/25] Correct doxygen keywords --- src/active.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/active.h b/src/active.h index e34ba0961..b24ef6792 100644 --- a/src/active.h +++ b/src/active.h @@ -53,7 +53,7 @@ __attribute__((always_inline)) INLINE static int cell_is_drifted( * * @param c The #cell. * @param e The #engine containing information about the current time. - * @param 1 if the #cell contains at least an active particle, 0 otherwise. + * @return 1 if the #cell contains at least an active particle, 0 otherwise. */ __attribute__((always_inline)) INLINE static int cell_is_active( const struct cell *c, const struct engine *e) { @@ -72,7 +72,7 @@ __attribute__((always_inline)) INLINE static int cell_is_active( * * @param c The #cell. * @param e The #engine containing information about the current time. - * @param 1 if all particles in a #cell are active, 0 otherwise. + * @return 1 if all particles in a #cell are active, 0 otherwise. */ __attribute__((always_inline)) INLINE static int cell_is_all_active( const struct cell *c, const struct engine *e) { -- GitLab From ab72a00ce54fcb7117d1d9d6b685e91eae8252dc Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Fri, 16 Dec 2016 10:58:14 +0000 Subject: [PATCH 17/25] Add drift-->send dependency and update ti_old on reception --- src/cell.c | 5 ++++- src/cell.h | 2 +- src/engine.c | 8 ++++++++ src/runner.c | 9 +++++++-- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/cell.c b/src/cell.c index 0dbd38a6a..ae79e1400 100644 --- a/src/cell.c +++ b/src/cell.c @@ -99,6 +99,7 @@ int cell_unpack(struct pcell *pc, struct cell *c, struct space *s) { c->h_max = pc->h_max; c->ti_end_min = pc->ti_end_min; c->ti_end_max = pc->ti_end_max; + c->ti_old = pc->ti_old; c->count = pc->count; c->gcount = pc->gcount; c->tag = pc->tag; @@ -127,6 +128,7 @@ int cell_unpack(struct pcell *pc, struct cell *c, struct space *s) { temp->dx_max = 0.f; temp->nodeID = c->nodeID; temp->parent = c; + temp->ti_old = c->ti_old; c->progeny[k] = temp; c->split = 1; count += cell_unpack(&pc[pc->progeny[k]], temp, s); @@ -209,6 +211,7 @@ int cell_pack(struct cell *c, struct pcell *pc) { pc->h_max = c->h_max; pc->ti_end_min = c->ti_end_min; pc->ti_end_max = c->ti_end_max; + pc->ti_old = c->ti_old; pc->count = c->count; pc->gcount = c->gcount; c->tag = pc->tag = atomic_inc(&cell_next_tag) % cell_max_tag; @@ -714,7 +717,7 @@ void cell_check_drift_point(struct cell *c, void *data) { const int ti_current = *(int *)data; - if (c->ti_old != ti_current) + if (c->ti_old != ti_current && c->nodeID == engine_rank) error("Cell in an incorrect time-zone! c->ti_old=%d ti_current=%d", c->ti_old, ti_current); } diff --git a/src/cell.h b/src/cell.h index 08dc568c9..3772cfe92 100644 --- a/src/cell.h +++ b/src/cell.h @@ -67,7 +67,7 @@ struct pcell { /* Stats on this cell's particles. */ double h_max; - int ti_end_min, ti_end_max; + int ti_end_min, ti_end_max, ti_old; /* Number of particles in this cell. */ int count, gcount; diff --git a/src/engine.c b/src/engine.c index fb5f24eb4..4aa0ddb1b 100644 --- a/src/engine.c +++ b/src/engine.c @@ -143,6 +143,7 @@ void engine_make_hierarchical_tasks(struct engine *e, struct cell *c) { c->kick = scheduler_addtask(s, task_type_kick, task_subtype_none, 0, 0, c, NULL, 0); + /* Add the drift task and dependency. */ c->drift = scheduler_addtask(s, task_type_drift, task_subtype_none, 0, 0, c, NULL, 0); @@ -673,6 +674,11 @@ void engine_addtasks_send(struct engine *e, struct cell *ci, struct cell *cj, /* Create the tasks and their dependencies? */ if (t_xv == NULL) { + + if(ci->super->drift == NULL) + ci->super->drift = scheduler_addtask(s, task_type_drift, task_subtype_none, 0, 0, + ci->super, NULL, 0); + t_xv = scheduler_addtask(s, task_type_send, task_subtype_none, 4 * ci->tag, 0, ci, cj, 0); t_rho = scheduler_addtask(s, task_type_send, task_subtype_none, @@ -708,6 +714,8 @@ void engine_addtasks_send(struct engine *e, struct cell *ci, struct cell *cj, /* The send_xv task should unlock the super-cell's ghost task. */ scheduler_addunlock(s, t_xv, ci->super->ghost); + + scheduler_addunlock(s, ci->super->drift, t_xv); #endif /* The super-cell's kick task should unlock the send_ti task. */ diff --git a/src/runner.c b/src/runner.c index 24003d960..52f91d620 100644 --- a/src/runner.c +++ b/src/runner.c @@ -958,7 +958,7 @@ void runner_do_recv_cell(struct runner *r, struct cell *c, int timer) { const struct gpart *restrict gparts = c->gparts; const size_t nr_parts = c->count; const size_t nr_gparts = c->gcount; - // const int ti_current = r->e->ti_current; + const int ti_current = r->e->ti_current; TIMER_TIC; @@ -1001,6 +1001,7 @@ void runner_do_recv_cell(struct runner *r, struct cell *c, int timer) { /* ... and store. */ c->ti_end_min = ti_end_min; c->ti_end_max = ti_end_max; + c->ti_old = ti_current; c->h_max = h_max; if (timer) TIMER_TOC(timer_dorecv_cell); @@ -1051,8 +1052,9 @@ void *runner_main(void *data) { /* Check that we haven't scheduled an inactive task */ #ifdef SWIFT_DEBUG_CHECKS +#ifndef WITH_MPI if (cj == NULL) { /* self */ - if (!cell_is_active(ci, e) && t->type != task_type_sort) + if (!cell_is_active(ci, e) && t->type != task_type_sort && t->type != task_type_send && t->type != task_type_recv) error( "Task (type='%s/%s') should have been skipped ti_current=%d " "c->ti_end_min=%d", @@ -1070,12 +1072,15 @@ void *runner_main(void *data) { } else { /* pair */ if (!cell_is_active(ci, e) && !cell_is_active(cj, e)) + + if(t->type != task_type_send && t->type != task_type_recv) error( "Task (type='%s/%s') should have been skipped ti_current=%d " "ci->ti_end_min=%d cj->ti_end_min=%d", taskID_names[t->type], subtaskID_names[t->subtype], e->ti_current, ci->ti_end_min, cj->ti_end_min); } +#endif #endif /* Different types of tasks... */ -- GitLab From f89460ea3967ed9a034f4a6f73212d3d2eb2eb51 Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Mon, 19 Dec 2016 07:52:50 +0000 Subject: [PATCH 18/25] More debugging info --- src/cell.c | 6 +++++- src/engine.c | 2 ++ src/hydro/Gadget2/hydro.h | 1 + src/hydro/Gadget2/hydro_part.h | 2 ++ src/runner.c | 21 +++++++++++++++++++++ 5 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/cell.c b/src/cell.c index ae79e1400..0e5c6672e 100644 --- a/src/cell.c +++ b/src/cell.c @@ -906,6 +906,7 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) { ; if (l == NULL) error("Missing link to send_xv task."); scheduler_activate(s, l->t); + if(l->t->cj->drift != NULL) scheduler_activate(s, l->t->cj->drift); for (l = cj->send_rho; l != NULL && l->t->cj->nodeID != ci->nodeID; l = l->next) @@ -932,6 +933,7 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) { ; if (l == NULL) error("Missing link to send_xv task."); scheduler_activate(s, l->t); + if(l->t->ci->drift != NULL) scheduler_activate(s, l->t->ci->drift); for (l = ci->send_rho; l != NULL && l->t->cj->nodeID != cj->nodeID; l = l->next) @@ -1021,7 +1023,7 @@ void cell_drift(struct cell *c, const struct engine *e) { h_max = max(h_max, cp->h_max); } - } else if (ti_current >= ti_old) { + } else if (ti_current > ti_old) { /* Loop over all the g-particles in the cell */ const size_t nr_gparts = c->gcount; @@ -1051,6 +1053,8 @@ void cell_drift(struct cell *c, const struct engine *e) { /* Drift... */ drift_part(p, xp, dt, timeBase, ti_old, ti_current); + p->ti_old = ti_current; + /* Compute (square of) motion since last cell construction */ const float dx2 = xp->x_diff[0] * xp->x_diff[0] + xp->x_diff[1] * xp->x_diff[1] + diff --git a/src/engine.c b/src/engine.c index 4aa0ddb1b..e281d9122 100644 --- a/src/engine.c +++ b/src/engine.c @@ -2066,6 +2066,7 @@ void engine_marktasks_mapper(void *map_data, int num_elements, ; if (l == NULL) error("Missing link to send_xv task."); scheduler_activate(s, l->t); + if(l->t->cj->drift) scheduler_activate(s, l->t->cj->drift); for (l = cj->send_rho; l != NULL && l->t->cj->nodeID != ci->nodeID; l = l->next) @@ -2093,6 +2094,7 @@ void engine_marktasks_mapper(void *map_data, int num_elements, ; if (l == NULL) error("Missing link to send_xv task."); scheduler_activate(s, l->t); + if(l->t->ci->drift != NULL) scheduler_activate(s, l->t->ci->drift); for (l = ci->send_rho; l != NULL && l->t->cj->nodeID != cj->nodeID; l = l->next) diff --git a/src/hydro/Gadget2/hydro.h b/src/hydro/Gadget2/hydro.h index 157893bc9..66c699805 100644 --- a/src/hydro/Gadget2/hydro.h +++ b/src/hydro/Gadget2/hydro.h @@ -212,6 +212,7 @@ __attribute__((always_inline)) INLINE static float hydro_compute_timestep( __attribute__((always_inline)) INLINE static void hydro_first_init_part( struct part *restrict p, struct xpart *restrict xp) { + p->ti_old = 0; p->ti_begin = 0; p->ti_end = 0; xp->v_full[0] = p->v[0]; diff --git a/src/hydro/Gadget2/hydro_part.h b/src/hydro/Gadget2/hydro_part.h index 4bbbf0aed..15545f480 100644 --- a/src/hydro/Gadget2/hydro_part.h +++ b/src/hydro/Gadget2/hydro_part.h @@ -71,6 +71,8 @@ struct part { /* Particle time of end of time-step. */ int ti_end; + int ti_old; + /* Particle density. */ float rho; diff --git a/src/runner.c b/src/runner.c index 52f91d620..6bf9dc4e2 100644 --- a/src/runner.c +++ b/src/runner.c @@ -973,6 +973,8 @@ void runner_do_recv_cell(struct runner *r, struct cell *c, int timer) { for (size_t k = 0; k < nr_parts; k++) { const int ti_end = parts[k].ti_end; // if(ti_end < ti_current) error("Received invalid particle !"); + if(parts[k].ti_old != ti_current) error("Received undrifted particles p->ti_old=%d ti_current=%d c->nodeID=%d loc=(%f %f %f) c->drift=%p, c->drift->skip=%d", + parts[k].ti_old, ti_current, c->nodeID, c->loc[0], c->loc[1], c->loc[2], c->drift, c->drift ? c->drift->skip : -1); ti_end_min = min(ti_end_min, ti_end); ti_end_max = max(ti_end_max, ti_end); h_max = max(h_max, parts[k].h); @@ -1007,6 +1009,24 @@ void runner_do_recv_cell(struct runner *r, struct cell *c, int timer) { if (timer) TIMER_TOC(timer_dorecv_cell); } +void runner_check_drift(struct runner *r, struct cell *c) { + + + const struct part *restrict parts = c->parts; + const size_t nr_parts = c->count; + const int ti_current = r->e->ti_current; + + for (size_t k = 0; k < nr_parts; k++) { + + if(parts[k].ti_old != ti_current) { + message("Sending undrifted particles p->ti_old=%d ti_current=%d c->nodeID=%d loc=(%f %f %f) c->drift=%p, c->drift->skip=%d", + parts[k].ti_old, ti_current, c->nodeID, c->loc[0], c->loc[1], c->loc[2], + c->drift, c->drift ? c->drift->skip : -1); + break; + } + } +} + /** * @brief The #runner main thread routine. * @@ -1170,6 +1190,7 @@ void *runner_main(void *data) { break; #ifdef WITH_MPI case task_type_send: + runner_check_drift(r, ci); if (t->subtype == task_subtype_tend) { free(t->buff); } -- GitLab From 0656604c0b63949737f5fbffd60e6f842983efa5 Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Tue, 20 Dec 2016 16:29:56 +0000 Subject: [PATCH 19/25] No debugging code any more --- src/runner.c | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/runner.c b/src/runner.c index 6bf9dc4e2..52f91d620 100644 --- a/src/runner.c +++ b/src/runner.c @@ -973,8 +973,6 @@ void runner_do_recv_cell(struct runner *r, struct cell *c, int timer) { for (size_t k = 0; k < nr_parts; k++) { const int ti_end = parts[k].ti_end; // if(ti_end < ti_current) error("Received invalid particle !"); - if(parts[k].ti_old != ti_current) error("Received undrifted particles p->ti_old=%d ti_current=%d c->nodeID=%d loc=(%f %f %f) c->drift=%p, c->drift->skip=%d", - parts[k].ti_old, ti_current, c->nodeID, c->loc[0], c->loc[1], c->loc[2], c->drift, c->drift ? c->drift->skip : -1); ti_end_min = min(ti_end_min, ti_end); ti_end_max = max(ti_end_max, ti_end); h_max = max(h_max, parts[k].h); @@ -1009,24 +1007,6 @@ void runner_do_recv_cell(struct runner *r, struct cell *c, int timer) { if (timer) TIMER_TOC(timer_dorecv_cell); } -void runner_check_drift(struct runner *r, struct cell *c) { - - - const struct part *restrict parts = c->parts; - const size_t nr_parts = c->count; - const int ti_current = r->e->ti_current; - - for (size_t k = 0; k < nr_parts; k++) { - - if(parts[k].ti_old != ti_current) { - message("Sending undrifted particles p->ti_old=%d ti_current=%d c->nodeID=%d loc=(%f %f %f) c->drift=%p, c->drift->skip=%d", - parts[k].ti_old, ti_current, c->nodeID, c->loc[0], c->loc[1], c->loc[2], - c->drift, c->drift ? c->drift->skip : -1); - break; - } - } -} - /** * @brief The #runner main thread routine. * @@ -1190,7 +1170,6 @@ void *runner_main(void *data) { break; #ifdef WITH_MPI case task_type_send: - runner_check_drift(r, ci); if (t->subtype == task_subtype_tend) { free(t->buff); } -- GitLab From 4e460da89a6a4036a052b3892114030454692078 Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Tue, 20 Dec 2016 16:36:46 +0000 Subject: [PATCH 20/25] Ready for merge --- src/cell.c | 13 +++++++++++-- src/engine.c | 12 ++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/cell.c b/src/cell.c index 0e5c6672e..e8f8cbffb 100644 --- a/src/cell.c +++ b/src/cell.c @@ -906,7 +906,11 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) { ; if (l == NULL) error("Missing link to send_xv task."); scheduler_activate(s, l->t); - if(l->t->cj->drift != NULL) scheduler_activate(s, l->t->cj->drift); + + if(cj->super->drift) + scheduler_activate(s, cj->super->drift); + else + error("Drift task missing !"); for (l = cj->send_rho; l != NULL && l->t->cj->nodeID != ci->nodeID; l = l->next) @@ -926,6 +930,7 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) { scheduler_activate(s, cj->recv_xv); scheduler_activate(s, cj->recv_rho); scheduler_activate(s, cj->recv_ti); + /* Look for the local cell ci's send tasks. */ struct link *l = NULL; for (l = ci->send_xv; l != NULL && l->t->cj->nodeID != cj->nodeID; @@ -933,7 +938,11 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) { ; if (l == NULL) error("Missing link to send_xv task."); scheduler_activate(s, l->t); - if(l->t->ci->drift != NULL) scheduler_activate(s, l->t->ci->drift); + + if(ci->super->drift) + scheduler_activate(s, ci->super->drift); + else + error("Drift task missing !"); for (l = ci->send_rho; l != NULL && l->t->cj->nodeID != cj->nodeID; l = l->next) diff --git a/src/engine.c b/src/engine.c index e281d9122..82ff7b595 100644 --- a/src/engine.c +++ b/src/engine.c @@ -2066,8 +2066,12 @@ void engine_marktasks_mapper(void *map_data, int num_elements, ; if (l == NULL) error("Missing link to send_xv task."); scheduler_activate(s, l->t); - if(l->t->cj->drift) scheduler_activate(s, l->t->cj->drift); + if(cj->super->drift) + scheduler_activate(s, cj->super->drift); + else + error("Drift task missing !"); + for (l = cj->send_rho; l != NULL && l->t->cj->nodeID != ci->nodeID; l = l->next) ; @@ -2094,7 +2098,11 @@ void engine_marktasks_mapper(void *map_data, int num_elements, ; if (l == NULL) error("Missing link to send_xv task."); scheduler_activate(s, l->t); - if(l->t->ci->drift != NULL) scheduler_activate(s, l->t->ci->drift); + + if(ci->super->drift) + scheduler_activate(s, ci->super->drift); + else + error("Drift task missing !"); for (l = ci->send_rho; l != NULL && l->t->cj->nodeID != cj->nodeID; l = l->next) -- GitLab From e9d7520bb1a847ea16b456f98318fc453eae55c5 Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Tue, 20 Dec 2016 17:38:16 +0100 Subject: [PATCH 21/25] Code formatting --- src/cell.c | 16 ++++++++-------- src/engine.c | 26 +++++++++++++------------- src/runner.c | 15 ++++++++------- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/src/cell.c b/src/cell.c index e8f8cbffb..f0668f48f 100644 --- a/src/cell.c +++ b/src/cell.c @@ -907,10 +907,10 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) { if (l == NULL) error("Missing link to send_xv task."); scheduler_activate(s, l->t); - if(cj->super->drift) - scheduler_activate(s, cj->super->drift); - else - error("Drift task missing !"); + if (cj->super->drift) + scheduler_activate(s, cj->super->drift); + else + error("Drift task missing !"); for (l = cj->send_rho; l != NULL && l->t->cj->nodeID != ci->nodeID; l = l->next) @@ -939,10 +939,10 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) { if (l == NULL) error("Missing link to send_xv task."); scheduler_activate(s, l->t); - if(ci->super->drift) - scheduler_activate(s, ci->super->drift); - else - error("Drift task missing !"); + if (ci->super->drift) + scheduler_activate(s, ci->super->drift); + else + error("Drift task missing !"); for (l = ci->send_rho; l != NULL && l->t->cj->nodeID != cj->nodeID; l = l->next) diff --git a/src/engine.c b/src/engine.c index 82ff7b595..60ca48bd3 100644 --- a/src/engine.c +++ b/src/engine.c @@ -674,10 +674,10 @@ void engine_addtasks_send(struct engine *e, struct cell *ci, struct cell *cj, /* Create the tasks and their dependencies? */ if (t_xv == NULL) { - - if(ci->super->drift == NULL) - ci->super->drift = scheduler_addtask(s, task_type_drift, task_subtype_none, 0, 0, - ci->super, NULL, 0); + + if (ci->super->drift == NULL) + ci->super->drift = scheduler_addtask( + s, task_type_drift, task_subtype_none, 0, 0, ci->super, NULL, 0); t_xv = scheduler_addtask(s, task_type_send, task_subtype_none, 4 * ci->tag, 0, ci, cj, 0); @@ -2067,11 +2067,11 @@ void engine_marktasks_mapper(void *map_data, int num_elements, if (l == NULL) error("Missing link to send_xv task."); scheduler_activate(s, l->t); - if(cj->super->drift) - scheduler_activate(s, cj->super->drift); - else - error("Drift task missing !"); - + if (cj->super->drift) + scheduler_activate(s, cj->super->drift); + else + error("Drift task missing !"); + for (l = cj->send_rho; l != NULL && l->t->cj->nodeID != ci->nodeID; l = l->next) ; @@ -2099,10 +2099,10 @@ void engine_marktasks_mapper(void *map_data, int num_elements, if (l == NULL) error("Missing link to send_xv task."); scheduler_activate(s, l->t); - if(ci->super->drift) - scheduler_activate(s, ci->super->drift); - else - error("Drift task missing !"); + if (ci->super->drift) + scheduler_activate(s, ci->super->drift); + else + error("Drift task missing !"); for (l = ci->send_rho; l != NULL && l->t->cj->nodeID != cj->nodeID; l = l->next) diff --git a/src/runner.c b/src/runner.c index 52f91d620..269ec2229 100644 --- a/src/runner.c +++ b/src/runner.c @@ -1054,7 +1054,8 @@ void *runner_main(void *data) { #ifdef SWIFT_DEBUG_CHECKS #ifndef WITH_MPI if (cj == NULL) { /* self */ - if (!cell_is_active(ci, e) && t->type != task_type_sort && t->type != task_type_send && t->type != task_type_recv) + if (!cell_is_active(ci, e) && t->type != task_type_sort && + t->type != task_type_send && t->type != task_type_recv) error( "Task (type='%s/%s') should have been skipped ti_current=%d " "c->ti_end_min=%d", @@ -1073,12 +1074,12 @@ void *runner_main(void *data) { } else { /* pair */ if (!cell_is_active(ci, e) && !cell_is_active(cj, e)) - if(t->type != task_type_send && t->type != task_type_recv) - error( - "Task (type='%s/%s') should have been skipped ti_current=%d " - "ci->ti_end_min=%d cj->ti_end_min=%d", - taskID_names[t->type], subtaskID_names[t->subtype], e->ti_current, - ci->ti_end_min, cj->ti_end_min); + if (t->type != task_type_send && t->type != task_type_recv) + error( + "Task (type='%s/%s') should have been skipped ti_current=%d " + "ci->ti_end_min=%d cj->ti_end_min=%d", + taskID_names[t->type], subtaskID_names[t->subtype], + e->ti_current, ci->ti_end_min, cj->ti_end_min); } #endif #endif -- GitLab From 048cc0af45b8ec5f8349f0b02e65a275f1fe0d26 Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Tue, 20 Dec 2016 17:42:48 +0100 Subject: [PATCH 22/25] Remove all debugging information --- src/cell.c | 2 -- src/hydro/Gadget2/hydro.h | 1 - src/hydro/Gadget2/hydro_part.h | 2 -- 3 files changed, 5 deletions(-) diff --git a/src/cell.c b/src/cell.c index f0668f48f..c7b992bb7 100644 --- a/src/cell.c +++ b/src/cell.c @@ -1062,8 +1062,6 @@ void cell_drift(struct cell *c, const struct engine *e) { /* Drift... */ drift_part(p, xp, dt, timeBase, ti_old, ti_current); - p->ti_old = ti_current; - /* Compute (square of) motion since last cell construction */ const float dx2 = xp->x_diff[0] * xp->x_diff[0] + xp->x_diff[1] * xp->x_diff[1] + diff --git a/src/hydro/Gadget2/hydro.h b/src/hydro/Gadget2/hydro.h index 66c699805..157893bc9 100644 --- a/src/hydro/Gadget2/hydro.h +++ b/src/hydro/Gadget2/hydro.h @@ -212,7 +212,6 @@ __attribute__((always_inline)) INLINE static float hydro_compute_timestep( __attribute__((always_inline)) INLINE static void hydro_first_init_part( struct part *restrict p, struct xpart *restrict xp) { - p->ti_old = 0; p->ti_begin = 0; p->ti_end = 0; xp->v_full[0] = p->v[0]; diff --git a/src/hydro/Gadget2/hydro_part.h b/src/hydro/Gadget2/hydro_part.h index 15545f480..4bbbf0aed 100644 --- a/src/hydro/Gadget2/hydro_part.h +++ b/src/hydro/Gadget2/hydro_part.h @@ -71,8 +71,6 @@ struct part { /* Particle time of end of time-step. */ int ti_end; - int ti_old; - /* Particle density. */ float rho; -- GitLab From de558d53a2330ab63b48385358835b0e4962f6dd Mon Sep 17 00:00:00 2001 From: "Peter W. Draper" Date: Thu, 22 Dec 2016 13:05:05 +0000 Subject: [PATCH 23/25] Move unskip to after rebuild, probably needed this way after repartitioning --- src/engine.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/engine.c b/src/engine.c index 1ff7a7d46..f437af521 100644 --- a/src/engine.c +++ b/src/engine.c @@ -2261,10 +2261,7 @@ void engine_prepare(struct engine *e, int nodrift) { TIMER_TIC; - /* Unskip active tasks and check for rebuild */ - engine_unskip(e); - - /* Run through the tasks and mark as skip or not. */ + /* Check for rebuild */ int rebuild = e->forcerebuild; /* Collect the values of rebuild from all nodes. */ @@ -2290,6 +2287,9 @@ void engine_prepare(struct engine *e, int nodrift) { engine_rebuild(e); } + /* Run through the tasks and mark as skip or not (after rebuild). */ + engine_unskip(e); + /* Re-rank the tasks every now and then. */ if (e->tasks_age % engine_tasksreweight == 1) { scheduler_reweight(&e->sched, e->verbose); -- GitLab From 6d35c4e9fba4dc89e5708b88d29d11d82b8a4f48 Mon Sep 17 00:00:00 2001 From: "Peter W. Draper" Date: Thu, 22 Dec 2016 13:12:50 +0000 Subject: [PATCH 24/25] Revert "Move unskip to after rebuild, probably needed this way after repartitioning" This reverts commit de558d53a2330ab63b48385358835b0e4962f6dd. The non-MPI SodShock test starting using 0 dt steps. --- src/engine.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/engine.c b/src/engine.c index f437af521..1ff7a7d46 100644 --- a/src/engine.c +++ b/src/engine.c @@ -2261,7 +2261,10 @@ void engine_prepare(struct engine *e, int nodrift) { TIMER_TIC; - /* Check for rebuild */ + /* Unskip active tasks and check for rebuild */ + engine_unskip(e); + + /* Run through the tasks and mark as skip or not. */ int rebuild = e->forcerebuild; /* Collect the values of rebuild from all nodes. */ @@ -2287,9 +2290,6 @@ void engine_prepare(struct engine *e, int nodrift) { engine_rebuild(e); } - /* Run through the tasks and mark as skip or not (after rebuild). */ - engine_unskip(e); - /* Re-rank the tasks every now and then. */ if (e->tasks_age % engine_tasksreweight == 1) { scheduler_reweight(&e->sched, e->verbose); -- GitLab From 598ae8c0b45403dca476c8ef1b22ec4311b86c85 Mon Sep 17 00:00:00 2001 From: "Peter W. Draper" Date: Thu, 22 Dec 2016 15:11:33 +0000 Subject: [PATCH 25/25] Need to unskip before rebuild, except when following a repartition, so make that work --- src/engine.c | 15 +++++++++------ src/engine.h | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/engine.c b/src/engine.c index 1ff7a7d46..443be97cb 100644 --- a/src/engine.c +++ b/src/engine.c @@ -2253,16 +2253,18 @@ void engine_rebuild(struct engine *e) { * @brief Prepare the #engine by re-building the cells and tasks. * * @param e The #engine to prepare. - * @param nodrift Whether to drift particles before rebuilding or not. Will + * @param drift_all Whether to drift particles before rebuilding or not. Will * not be necessary if all particles have already been * drifted (before repartitioning for instance). + * @param deferskip Whether to defer the skip until after the rebuild. + * Needed after a repartition. */ -void engine_prepare(struct engine *e, int nodrift) { +void engine_prepare(struct engine *e, int drift_all, int deferskip) { TIMER_TIC; /* Unskip active tasks and check for rebuild */ - engine_unskip(e); + if (!deferskip) engine_unskip(e); /* Run through the tasks and mark as skip or not. */ int rebuild = e->forcerebuild; @@ -2280,7 +2282,7 @@ void engine_prepare(struct engine *e, int nodrift) { if (rebuild) { /* Drift all particles to the current time if needed. */ - if (!nodrift) engine_drift_all(e); + if (drift_all) engine_drift_all(e); #ifdef SWIFT_DEBUG_CHECKS /* Check that all cells have been drifted to the current time */ @@ -2289,6 +2291,7 @@ void engine_prepare(struct engine *e, int nodrift) { engine_rebuild(e); } + if (deferskip) engine_unskip(e); /* Re-rank the tasks every now and then. */ if (e->tasks_age % engine_tasksreweight == 1) { @@ -2566,7 +2569,7 @@ void engine_init_particles(struct engine *e, int flag_entropy_ICs) { if (e->nodeID == 0) message("Running initialisation fake time-step."); - engine_prepare(e, 1); + engine_prepare(e, 0, 0); engine_marktasks(e); @@ -2671,7 +2674,7 @@ void engine_step(struct engine *e) { if (repart) engine_repartition(e); /* Prepare the space. */ - engine_prepare(e, (drift_all || repart)); + engine_prepare(e, !(drift_all || repart), repart); if (e->verbose) engine_print_task_counts(e); diff --git a/src/engine.h b/src/engine.h index 5d72c23cd..87c2ea0a9 100644 --- a/src/engine.h +++ b/src/engine.h @@ -228,7 +228,7 @@ void engine_init(struct engine *e, struct space *s, const struct cooling_function_data *cooling, struct sourceterms *sourceterms); void engine_launch(struct engine *e, int nr_runners); -void engine_prepare(struct engine *e, int nodrift); +void engine_prepare(struct engine *e, int drift_all, int deferskip); void engine_print(struct engine *e); void engine_init_particles(struct engine *e, int flag_entropy_ICs); void engine_step(struct engine *e); -- GitLab