diff --git a/src/active.h b/src/active.h index e33f8baf6e5bd5d799e122e4e04610a7cab443bf..b24ef6792303315ea395b38912cae5cf279e0fe8 100644 --- a/src/active.h +++ b/src/active.h @@ -30,13 +30,11 @@ /** * @brief Check that a cell been drifted to the current time. * - * Only used for debugging. Calls error() if the cell has not - * been drifted. Does nothing if SWIFT_DEBUG_CHECKS is not defined. - * * @param c The #cell. * @param e The #engine containing information about the current time. + * @return 1 if the #cell has been drifted to the current time, 0 otherwise. */ -__attribute__((always_inline)) INLINE static void cell_is_drifted( +__attribute__((always_inline)) INLINE static int cell_is_drifted( const struct cell *c, const struct engine *e) { #ifdef SWIFT_DEBUG_CHECKS @@ -45,14 +43,9 @@ __attribute__((always_inline)) INLINE static void cell_is_drifted( "Cell has been drifted too far forward in time! c->ti_old=%d " "e->ti_current=%d", c->ti_old, e->ti_current); - - if (c->ti_old != e->ti_current) { - error( - "Cell has not been drifted to the current time c->ti_old=%d, " - "e->ti_current=%d", - c->ti_old, e->ti_current); - } #endif + + return (c->ti_old == e->ti_current); } /** @@ -60,6 +53,7 @@ __attribute__((always_inline)) INLINE static void cell_is_drifted( * * @param c The #cell. * @param e The #engine containing information about the current time. + * @return 1 if the #cell contains at least an active particle, 0 otherwise. */ __attribute__((always_inline)) INLINE static int cell_is_active( const struct cell *c, const struct engine *e) { @@ -78,6 +72,7 @@ __attribute__((always_inline)) INLINE static int cell_is_active( * * @param c The #cell. * @param e The #engine containing information about the current time. + * @return 1 if all particles in a #cell are active, 0 otherwise. */ __attribute__((always_inline)) INLINE static int cell_is_all_active( const struct cell *c, const struct engine *e) { @@ -96,6 +91,7 @@ __attribute__((always_inline)) INLINE static int cell_is_all_active( * * @param p The #part. * @param e The #engine containing information about the current time. + * @return 1 if the #part is active, 0 otherwise. */ __attribute__((always_inline)) INLINE static int part_is_active( const struct part *p, const struct engine *e) { @@ -114,6 +110,7 @@ __attribute__((always_inline)) INLINE static int part_is_active( * * @param gp The #gpart. * @param e The #engine containing information about the current time. + * @return 1 if the #gpart is active, 0 otherwise. */ __attribute__((always_inline)) INLINE static int gpart_is_active( const struct gpart *gp, const struct engine *e) { diff --git a/src/cell.c b/src/cell.c index e2767cdaa9e1189ec87b5ef51cc578c91f8cfe4c..c7b992bb72f5431c267c19fad6355d31c663b20b 100644 --- a/src/cell.c +++ b/src/cell.c @@ -49,6 +49,7 @@ /* Local headers. */ #include "active.h" #include "atomic.h" +#include "drift.h" #include "error.h" #include "gravity.h" #include "hydro.h" @@ -98,6 +99,7 @@ int cell_unpack(struct pcell *pc, struct cell *c, struct space *s) { c->h_max = pc->h_max; c->ti_end_min = pc->ti_end_min; c->ti_end_max = pc->ti_end_max; + c->ti_old = pc->ti_old; c->count = pc->count; c->gcount = pc->gcount; c->tag = pc->tag; @@ -126,6 +128,7 @@ int cell_unpack(struct pcell *pc, struct cell *c, struct space *s) { temp->dx_max = 0.f; temp->nodeID = c->nodeID; temp->parent = c; + temp->ti_old = c->ti_old; c->progeny[k] = temp; c->split = 1; count += cell_unpack(&pc[pc->progeny[k]], temp, s); @@ -208,6 +211,7 @@ int cell_pack(struct cell *c, struct pcell *pc) { pc->h_max = c->h_max; pc->ti_end_min = c->ti_end_min; pc->ti_end_max = c->ti_end_max; + pc->ti_old = c->ti_old; pc->count = c->count; pc->gcount = c->gcount; c->tag = pc->tag = atomic_inc(&cell_next_tag) % cell_max_tag; @@ -713,7 +717,7 @@ void cell_check_drift_point(struct cell *c, void *data) { const int ti_current = *(int *)data; - if (c->ti_old != ti_current) + if (c->ti_old != ti_current && c->nodeID == engine_rank) error("Cell in an incorrect time-zone! c->ti_old=%d ti_current=%d", c->ti_old, ti_current); } @@ -903,6 +907,11 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) { if (l == NULL) error("Missing link to send_xv task."); scheduler_activate(s, l->t); + if (cj->super->drift) + scheduler_activate(s, cj->super->drift); + else + error("Drift task missing !"); + for (l = cj->send_rho; l != NULL && l->t->cj->nodeID != ci->nodeID; l = l->next) ; @@ -921,6 +930,7 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) { scheduler_activate(s, cj->recv_xv); scheduler_activate(s, cj->recv_rho); scheduler_activate(s, cj->recv_ti); + /* Look for the local cell ci's send tasks. */ struct link *l = NULL; for (l = ci->send_xv; l != NULL && l->t->cj->nodeID != cj->nodeID; @@ -929,6 +939,11 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) { if (l == NULL) error("Missing link to send_xv task."); scheduler_activate(s, l->t); + if (ci->super->drift) + scheduler_activate(s, ci->super->drift); + else + error("Drift task missing !"); + for (l = ci->send_rho; l != NULL && l->t->cj->nodeID != cj->nodeID; l = l->next) ; @@ -955,6 +970,7 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) { if (c->extra_ghost != NULL) scheduler_activate(s, c->extra_ghost); if (c->ghost != NULL) scheduler_activate(s, c->ghost); if (c->init != NULL) scheduler_activate(s, c->init); + if (c->drift != NULL) scheduler_activate(s, c->drift); if (c->kick != NULL) scheduler_activate(s, c->kick); if (c->cooling != NULL) scheduler_activate(s, c->cooling); if (c->sourceterms != NULL) scheduler_activate(s, c->sourceterms); @@ -981,3 +997,94 @@ void cell_set_super(struct cell *c, struct cell *super) { for (int k = 0; k < 8; k++) if (c->progeny[k] != NULL) cell_set_super(c->progeny[k], super); } + +/** + * @brief Recursively drifts all particles and g-particles in a cell hierarchy. + * + * @param c The #cell. + * @param e The #engine (to get ti_current). + */ +void cell_drift(struct cell *c, const struct engine *e) { + + const double timeBase = e->timeBase; + const int ti_old = c->ti_old; + const int ti_current = e->ti_current; + struct part *const parts = c->parts; + struct xpart *const xparts = c->xparts; + struct gpart *const gparts = c->gparts; + + /* Drift from the last time the cell was drifted to the current time */ + const double dt = (ti_current - ti_old) * timeBase; + float dx_max = 0.f, dx2_max = 0.f, h_max = 0.f; + + /* Check that we are actually going to move forward. */ + if (ti_current < ti_old) error("Attempt to drift to the past"); + + /* Are we not in a leaf ? */ + if (c->split) { + + /* Loop over the progeny and collect their data. */ + for (int k = 0; k < 8; k++) + if (c->progeny[k] != NULL) { + struct cell *cp = c->progeny[k]; + cell_drift(cp, e); + dx_max = max(dx_max, cp->dx_max); + h_max = max(h_max, cp->h_max); + } + + } else if (ti_current > ti_old) { + + /* Loop over all the g-particles in the cell */ + const size_t nr_gparts = c->gcount; + for (size_t k = 0; k < nr_gparts; k++) { + + /* Get a handle on the gpart. */ + struct gpart *const gp = &gparts[k]; + + /* Drift... */ + drift_gpart(gp, dt, timeBase, ti_old, ti_current); + + /* Compute (square of) motion since last cell construction */ + const float dx2 = gp->x_diff[0] * gp->x_diff[0] + + gp->x_diff[1] * gp->x_diff[1] + + gp->x_diff[2] * gp->x_diff[2]; + dx2_max = (dx2_max > dx2) ? dx2_max : dx2; + } + + /* Loop over all the particles in the cell */ + const size_t nr_parts = c->count; + for (size_t k = 0; k < nr_parts; k++) { + + /* Get a handle on the part. */ + struct part *const p = &parts[k]; + struct xpart *const xp = &xparts[k]; + + /* Drift... */ + drift_part(p, xp, dt, timeBase, ti_old, ti_current); + + /* Compute (square of) motion since last cell construction */ + const float dx2 = xp->x_diff[0] * xp->x_diff[0] + + xp->x_diff[1] * xp->x_diff[1] + + xp->x_diff[2] * xp->x_diff[2]; + dx2_max = (dx2_max > dx2) ? dx2_max : dx2; + + /* Maximal smoothing length */ + h_max = (h_max > p->h) ? h_max : p->h; + } + + /* Now, get the maximal particle motion from its square */ + dx_max = sqrtf(dx2_max); + + } else { + + h_max = c->h_max; + dx_max = c->dx_max; + } + + /* Store the values */ + c->h_max = h_max; + c->dx_max = dx_max; + + /* Update the time of the last drift */ + c->ti_old = ti_current; +} diff --git a/src/cell.h b/src/cell.h index 2cd13cf2ab6b934f6aab84bcbacf510270892866..3772cfe929a52601f407614f7f94929126f27514 100644 --- a/src/cell.h +++ b/src/cell.h @@ -67,7 +67,7 @@ struct pcell { /* Stats on this cell's particles. */ double h_max; - int ti_end_min, ti_end_max; + int ti_end_min, ti_end_max, ti_old; /* Number of particles in this cell. */ int count, gcount; @@ -147,6 +147,9 @@ struct cell { /*! The extra ghost task for complex hydro schemes */ struct task *extra_ghost; + /*! The drift task */ + struct task *drift; + /*! The kick task */ struct task *kick; @@ -295,5 +298,6 @@ void cell_check_drift_point(struct cell *c, void *data); int cell_is_drift_needed(struct cell *c, const struct engine *e); int cell_unskip_tasks(struct cell *c, struct scheduler *s); void cell_set_super(struct cell *c, struct cell *super); +void cell_drift(struct cell *c, const struct engine *e); #endif /* SWIFT_CELL_H */ diff --git a/src/engine.c b/src/engine.c index 6df1fd09ea68adb066cb5b1363c005ee28b3f643..443be97cb4adf53051523a7a8f8af750f477da33 100644 --- a/src/engine.c +++ b/src/engine.c @@ -143,6 +143,12 @@ void engine_make_hierarchical_tasks(struct engine *e, struct cell *c) { c->kick = scheduler_addtask(s, task_type_kick, task_subtype_none, 0, 0, c, NULL, 0); + /* Add the drift task and dependency. */ + c->drift = scheduler_addtask(s, task_type_drift, task_subtype_none, 0, 0, + c, NULL, 0); + + scheduler_addunlock(s, c->drift, c->init); + /* Generate the ghost task. */ if (is_hydro) c->ghost = scheduler_addtask(s, task_type_ghost, task_subtype_none, 0, @@ -668,6 +674,11 @@ void engine_addtasks_send(struct engine *e, struct cell *ci, struct cell *cj, /* Create the tasks and their dependencies? */ if (t_xv == NULL) { + + if (ci->super->drift == NULL) + ci->super->drift = scheduler_addtask( + s, task_type_drift, task_subtype_none, 0, 0, ci->super, NULL, 0); + t_xv = scheduler_addtask(s, task_type_send, task_subtype_none, 4 * ci->tag, 0, ci, cj, 0); t_rho = scheduler_addtask(s, task_type_send, task_subtype_none, @@ -703,6 +714,8 @@ void engine_addtasks_send(struct engine *e, struct cell *ci, struct cell *cj, /* The send_xv task should unlock the super-cell's ghost task. */ scheduler_addunlock(s, t_xv, ci->super->ghost); + + scheduler_addunlock(s, ci->super->drift, t_xv); #endif /* The super-cell's kick task should unlock the send_ti task. */ @@ -2054,6 +2067,11 @@ void engine_marktasks_mapper(void *map_data, int num_elements, if (l == NULL) error("Missing link to send_xv task."); scheduler_activate(s, l->t); + if (cj->super->drift) + scheduler_activate(s, cj->super->drift); + else + error("Drift task missing !"); + for (l = cj->send_rho; l != NULL && l->t->cj->nodeID != ci->nodeID; l = l->next) ; @@ -2081,6 +2099,11 @@ void engine_marktasks_mapper(void *map_data, int num_elements, if (l == NULL) error("Missing link to send_xv task."); scheduler_activate(s, l->t); + if (ci->super->drift) + scheduler_activate(s, ci->super->drift); + else + error("Drift task missing !"); + for (l = ci->send_rho; l != NULL && l->t->cj->nodeID != cj->nodeID; l = l->next) ; @@ -2104,6 +2127,11 @@ void engine_marktasks_mapper(void *map_data, int num_elements, if (t->ci->ti_end_min <= ti_end) scheduler_activate(s, t); } + /* Drift? */ + else if (t->type == task_type_drift) { + if (t->ci->ti_end_min <= ti_end) scheduler_activate(s, t); + } + /* Init? */ else if (t->type == task_type_init) { if (t->ci->ti_end_min <= ti_end) scheduler_activate(s, t); @@ -2225,14 +2253,19 @@ void engine_rebuild(struct engine *e) { * @brief Prepare the #engine by re-building the cells and tasks. * * @param e The #engine to prepare. - * @param nodrift Whether to drift particles before rebuilding or not. Will + * @param drift_all Whether to drift particles before rebuilding or not. Will * not be necessary if all particles have already been * drifted (before repartitioning for instance). + * @param deferskip Whether to defer the skip until after the rebuild. + * Needed after a repartition. */ -void engine_prepare(struct engine *e, int nodrift) { +void engine_prepare(struct engine *e, int drift_all, int deferskip) { TIMER_TIC; + /* Unskip active tasks and check for rebuild */ + if (!deferskip) engine_unskip(e); + /* Run through the tasks and mark as skip or not. */ int rebuild = e->forcerebuild; @@ -2249,13 +2282,7 @@ void engine_prepare(struct engine *e, int nodrift) { if (rebuild) { /* Drift all particles to the current time if needed. */ - if (!nodrift) { - e->drift_all = 1; - engine_drift(e); - - /* Restore the default drifting policy */ - e->drift_all = (e->policy & engine_policy_drift_all); - } + if (drift_all) engine_drift_all(e); #ifdef SWIFT_DEBUG_CHECKS /* Check that all cells have been drifted to the current time */ @@ -2264,6 +2291,7 @@ void engine_prepare(struct engine *e, int nodrift) { engine_rebuild(e); } + if (deferskip) engine_unskip(e); /* Re-rank the tasks every now and then. */ if (e->tasks_age % engine_tasksreweight == 1) { @@ -2478,7 +2506,8 @@ void engine_skip_force_and_kick(struct engine *e) { /* Skip everything that updates the particles */ if (t->subtype == task_subtype_force || t->type == task_type_kick || - t->type == task_type_cooling || t->type == task_type_sourceterms) + t->type == task_type_cooling || t->type == task_type_sourceterms || + t->type == task_type_drift) t->skip = 1; } } @@ -2540,7 +2569,7 @@ void engine_init_particles(struct engine *e, int flag_entropy_ICs) { if (e->nodeID == 0) message("Running initialisation fake time-step."); - engine_prepare(e, 1); + engine_prepare(e, 0, 0); engine_marktasks(e); @@ -2606,11 +2635,7 @@ void engine_step(struct engine *e) { snapshot_drift_time = e->timeStep; /* Drift everybody to the snapshot position */ - e->drift_all = 1; - engine_drift(e); - - /* Restore the default drifting policy */ - e->drift_all = (e->policy & engine_policy_drift_all); + engine_drift_all(e); /* Dump... */ engine_dump_snapshot(e); @@ -2642,17 +2667,14 @@ void engine_step(struct engine *e) { /* Drift only the necessary particles, that means all particles * if we are about to repartition. */ const int repart = (e->forcerepart != REPART_NONE); - e->drift_all = repart || e->drift_all; - engine_drift(e); + const int drift_all = (e->policy & engine_policy_drift_all); + if (repart || drift_all) engine_drift_all(e); /* Re-distribute the particles amongst the nodes? */ if (repart) engine_repartition(e); /* Prepare the space. */ - engine_prepare(e, e->drift_all); - - /* Restore the default drifting policy */ - e->drift_all = (e->policy & engine_policy_drift_all); + engine_prepare(e, !(drift_all || repart), repart); if (e->verbose) engine_print_task_counts(e); @@ -2683,19 +2705,35 @@ int engine_is_done(struct engine *e) { } /** - * @brief Drift particles using the current engine drift policy. + * @brief Unskip all the tasks that act on active cells at this time. * * @param e The #engine. */ -void engine_drift(struct engine *e) { +void engine_unskip(struct engine *e) { + + const ticks tic = getticks(); + threadpool_map(&e->threadpool, runner_do_unskip_mapper, e->s->cells_top, + e->s->nr_cells, sizeof(struct cell), 1, e); + + if (e->verbose) + message("took %.3f %s.", clocks_from_ticks(getticks() - tic), + clocks_getunit()); +} + +/** + * @brief Drift *all* particles forward to the current time. + * + * @param e The #engine. + */ +void engine_drift_all(struct engine *e) { const ticks tic = getticks(); threadpool_map(&e->threadpool, runner_do_drift_mapper, e->s->cells_top, e->s->nr_cells, sizeof(struct cell), 1, e); if (e->verbose) - message("took %.3f %s (including task unskipping).", - clocks_from_ticks(getticks() - tic), clocks_getunit()); + message("took %.3f %s.", clocks_from_ticks(getticks() - tic), + clocks_getunit()); } /** @@ -3033,7 +3071,6 @@ void engine_init(struct engine *e, struct space *s, e->timeStep = 0.; e->timeBase = 0.; e->timeBase_inv = 0.; - e->drift_all = (policy & engine_policy_drift_all); e->internalUnits = internal_units; e->timeFirstSnapshot = parser_get_param_double(params, "Snapshots:time_first"); diff --git a/src/engine.h b/src/engine.h index 861e321627032eb1f773992a9c123249c013daa0..87c2ea0a94058fe5a0af39c0eb02a72676d0794c 100644 --- a/src/engine.h +++ b/src/engine.h @@ -133,9 +133,6 @@ struct engine { /* Minimal ti_end for the next time-step */ int ti_end_min; - /* Are we drifting all particles now ? */ - int drift_all; - /* Number of particles updated */ size_t updates, g_updates; @@ -218,7 +215,8 @@ struct engine { /* Function prototypes. */ void engine_barrier(struct engine *e, int tid); void engine_compute_next_snapshot_time(struct engine *e); -void engine_drift(struct engine *e); +void engine_unskip(struct engine *e); +void engine_drift_all(struct engine *e); void engine_dump_snapshot(struct engine *e); void engine_init(struct engine *e, struct space *s, const struct swift_params *params, int nr_nodes, int nodeID, @@ -230,7 +228,7 @@ void engine_init(struct engine *e, struct space *s, const struct cooling_function_data *cooling, struct sourceterms *sourceterms); void engine_launch(struct engine *e, int nr_runners); -void engine_prepare(struct engine *e, int nodrift); +void engine_prepare(struct engine *e, int drift_all, int deferskip); void engine_print(struct engine *e); void engine_init_particles(struct engine *e, int flag_entropy_ICs); void engine_step(struct engine *e); diff --git a/src/runner.c b/src/runner.c index 2d6da4e4aedc9c40d1dade243e605e9aeda86dbe..269ec2229caf0c72bd23fdc7bbfa3752156244dd 100644 --- a/src/runner.c +++ b/src/runner.c @@ -744,15 +744,12 @@ void runner_do_ghost(struct runner *r, struct cell *c, int timer) { } /** - * @brief Drift particles and g-particles in a cell forward in time, - * unskipping any tasks associated with active cells. + * @brief Unskip any tasks associated with active cells. * * @param c The cell. * @param e The engine. - * @param drift whether to actually drift the particles, will not be - * necessary for non-local cells. */ -static void runner_do_drift(struct cell *c, struct engine *e, int drift) { +static void runner_do_unskip(struct cell *c, struct engine *e) { /* Unskip any active tasks. */ if (cell_is_active(c, e)) { @@ -760,124 +757,58 @@ static void runner_do_drift(struct cell *c, struct engine *e, int drift) { if (forcerebuild) atomic_inc(&e->forcerebuild); } - /* Do we really need to drift? */ - if (drift) { - if (!e->drift_all && !cell_is_drift_needed(c, e)) return; - } else { - - /* Not drifting, but may still need to recurse for task un-skipping. */ - if (c->split) { - for (int k = 0; k < 8; k++) { - if (c->progeny[k] != NULL) { - struct cell *cp = c->progeny[k]; - runner_do_drift(cp, e, 0); - } + /* Recurse */ + if (c->split) { + for (int k = 0; k < 8; k++) { + if (c->progeny[k] != NULL) { + struct cell *cp = c->progeny[k]; + runner_do_unskip(cp, e); } } - return; } +} - /* Now, we can drift */ - - /* Get some information first */ - const double timeBase = e->timeBase; - const int ti_old = c->ti_old; - const int ti_current = e->ti_current; - struct part *const parts = c->parts; - struct xpart *const xparts = c->xparts; - struct gpart *const gparts = c->gparts; - - /* Drift from the last time the cell was drifted to the current time */ - const double dt = (ti_current - ti_old) * timeBase; - float dx_max = 0.f, dx2_max = 0.f, h_max = 0.f; - - /* No children? */ - if (!c->split) { - - /* Check that we are actually going to move forward. */ - if (ti_current > ti_old) { - - /* Loop over all the g-particles in the cell */ - const size_t nr_gparts = c->gcount; - for (size_t k = 0; k < nr_gparts; k++) { - - /* Get a handle on the gpart. */ - struct gpart *const gp = &gparts[k]; - - /* Drift... */ - drift_gpart(gp, dt, timeBase, ti_old, ti_current); - - /* Compute (square of) motion since last cell construction */ - const float dx2 = gp->x_diff[0] * gp->x_diff[0] + - gp->x_diff[1] * gp->x_diff[1] + - gp->x_diff[2] * gp->x_diff[2]; - dx2_max = (dx2_max > dx2) ? dx2_max : dx2; - } - - /* Loop over all the particles in the cell */ - const size_t nr_parts = c->count; - for (size_t k = 0; k < nr_parts; k++) { - - /* Get a handle on the part. */ - struct part *const p = &parts[k]; - struct xpart *const xp = &xparts[k]; - - /* Drift... */ - drift_part(p, xp, dt, timeBase, ti_old, ti_current); - - /* Compute (square of) motion since last cell construction */ - const float dx2 = xp->x_diff[0] * xp->x_diff[0] + - xp->x_diff[1] * xp->x_diff[1] + - xp->x_diff[2] * xp->x_diff[2]; - dx2_max = (dx2_max > dx2) ? dx2_max : dx2; - - /* Maximal smoothing length */ - h_max = (h_max > p->h) ? h_max : p->h; - } - - /* Now, get the maximal particle motion from its square */ - dx_max = sqrtf(dx2_max); +/** + * @brief Mapper function to unskip active tasks. + * + * @param map_data An array of #cell%s. + * @param num_elements Chunk size. + * @param extra_data Pointer to an #engine. + */ +void runner_do_unskip_mapper(void *map_data, int num_elements, + void *extra_data) { - } /* Check that we are actually going to move forward. */ + struct engine *e = (struct engine *)extra_data; + struct cell *cells = (struct cell *)map_data; - else { - /* ti_old == ti_current, just keep the current cell values. */ - h_max = c->h_max; - dx_max = c->dx_max; - } + for (int ind = 0; ind < num_elements; ind++) { + struct cell *c = &cells[ind]; + if (c != NULL) runner_do_unskip(c, e); } +} +/** + * @brief Drift particles in real space. + * + * @param r The runner thread. + * @param c The cell. + * @param timer Are we timing this ? + */ +void runner_do_drift(struct runner *r, struct cell *c, int timer) { - /* Otherwise, aggregate data from children. */ - else { - - /* Loop over the progeny and collect their data. */ - for (int k = 0; k < 8; k++) - if (c->progeny[k] != NULL) { - struct cell *cp = c->progeny[k]; - - /* Recurse. */ - runner_do_drift(cp, e, drift); - dx_max = max(dx_max, cp->dx_max); - h_max = max(h_max, cp->h_max); - } - } + TIMER_TIC; - /* Store the values */ - c->h_max = h_max; - c->dx_max = dx_max; + cell_drift(c, r->e); - /* Update the time of the last drift */ - c->ti_old = ti_current; + if (timer) TIMER_TOC(timer_drift); } /** - * @brief Mapper function to drift particles and g-particles forward in time. + * @brief Mapper function to drift ALL particle and g-particles forward in time. * * @param map_data An array of #cell%s. * @param num_elements Chunk size. * @param extra_data Pointer to an #engine. */ - void runner_do_drift_mapper(void *map_data, int num_elements, void *extra_data) { @@ -886,17 +817,12 @@ void runner_do_drift_mapper(void *map_data, int num_elements, for (int ind = 0; ind < num_elements; ind++) { struct cell *c = &cells[ind]; -#ifdef WITH_MPI - if (c != NULL) runner_do_drift(c, e, (c->nodeID == e->nodeID)); -#else - if (c != NULL) runner_do_drift(c, e, 1); -#endif + if (c != NULL && c->nodeID == e->nodeID) cell_drift(c, e); } } /** - * @brief Kick particles in momentum space and collect statistics (floating - * time-step case) + * @brief Kick particles in momentum space and collect statistics. * * @param r The runner thread. * @param c The cell. @@ -1032,7 +958,7 @@ void runner_do_recv_cell(struct runner *r, struct cell *c, int timer) { const struct gpart *restrict gparts = c->gparts; const size_t nr_parts = c->count; const size_t nr_gparts = c->gcount; - // const int ti_current = r->e->ti_current; + const int ti_current = r->e->ti_current; TIMER_TIC; @@ -1075,6 +1001,7 @@ void runner_do_recv_cell(struct runner *r, struct cell *c, int timer) { /* ... and store. */ c->ti_end_min = ti_end_min; c->ti_end_max = ti_end_max; + c->ti_old = ti_current; c->h_max = h_max; if (timer) TIMER_TOC(timer_dorecv_cell); @@ -1125,8 +1052,10 @@ void *runner_main(void *data) { /* Check that we haven't scheduled an inactive task */ #ifdef SWIFT_DEBUG_CHECKS +#ifndef WITH_MPI if (cj == NULL) { /* self */ - if (!cell_is_active(ci, e) && t->type != task_type_sort) + if (!cell_is_active(ci, e) && t->type != task_type_sort && + t->type != task_type_send && t->type != task_type_recv) error( "Task (type='%s/%s') should have been skipped ti_current=%d " "c->ti_end_min=%d", @@ -1144,12 +1073,15 @@ void *runner_main(void *data) { } else { /* pair */ if (!cell_is_active(ci, e) && !cell_is_active(cj, e)) - error( - "Task (type='%s/%s') should have been skipped ti_current=%d " - "ci->ti_end_min=%d cj->ti_end_min=%d", - taskID_names[t->type], subtaskID_names[t->subtype], e->ti_current, - ci->ti_end_min, cj->ti_end_min); + + if (t->type != task_type_send && t->type != task_type_recv) + error( + "Task (type='%s/%s') should have been skipped ti_current=%d " + "ci->ti_end_min=%d cj->ti_end_min=%d", + taskID_names[t->type], subtaskID_names[t->subtype], + e->ti_current, ci->ti_end_min, cj->ti_end_min); } +#endif #endif /* Different types of tasks... */ @@ -1231,6 +1163,9 @@ void *runner_main(void *data) { runner_do_extra_ghost(r, ci, 1); break; #endif + case task_type_drift: + runner_do_drift(r, ci, 1); + break; case task_type_kick: runner_do_kick(r, ci, 1); break; diff --git a/src/runner.h b/src/runner.h index a8caf24248c99438f16729e2cac3e1031535f62b..6aab8469a1eb9aba22dac31553424bbcce8f183f 100644 --- a/src/runner.h +++ b/src/runner.h @@ -56,6 +56,8 @@ void runner_do_init(struct runner *r, struct cell *c, int timer); void runner_do_cooling(struct runner *r, struct cell *c, int timer); void runner_do_grav_external(struct runner *r, struct cell *c, int timer); void *runner_main(void *data); +void runner_do_unskip_mapper(void *map_data, int num_elements, + void *extra_data); void runner_do_drift_mapper(void *map_data, int num_elements, void *extra_data); #endif /* SWIFT_RUNNER_H */ diff --git a/src/runner_doiact.h b/src/runner_doiact.h index 6bc8f2da808cc2d953482b90e9441b833384bc75..8b860a013e7f31d24521fe5edf2cbcd5f0660088 100644 --- a/src/runner_doiact.h +++ b/src/runner_doiact.h @@ -721,10 +721,8 @@ void DOPAIR1(struct runner *r, struct cell *ci, struct cell *cj) { /* Anything to do here? */ if (!cell_is_active(ci, e) && !cell_is_active(cj, e)) return; -#ifdef SWIFT_DEBUG_CHECKS - cell_is_drifted(ci, e); - cell_is_drifted(cj, e); -#endif + if (!cell_is_drifted(ci, e)) cell_drift(ci, e); + if (!cell_is_drifted(cj, e)) cell_drift(cj, e); /* Get the sort ID. */ double shift[3] = {0.0, 0.0, 0.0}; @@ -918,10 +916,8 @@ void DOPAIR2(struct runner *r, struct cell *ci, struct cell *cj) { /* Anything to do here? */ if (!cell_is_active(ci, e) && !cell_is_active(cj, e)) return; -#ifdef SWIFT_DEBUG_CHECKS - cell_is_drifted(ci, e); - cell_is_drifted(cj, e); -#endif + if (!cell_is_drifted(ci, e)) error("Cell ci not drifted"); + if (!cell_is_drifted(cj, e)) error("Cell cj not drifted"); /* Get the shift ID. */ double shift[3] = {0.0, 0.0, 0.0}; @@ -1313,9 +1309,7 @@ void DOSELF1(struct runner *r, struct cell *restrict c) { if (!cell_is_active(c, e)) return; -#ifdef SWIFT_DEBUG_CHECKS - cell_is_drifted(c, e); -#endif + if (!cell_is_drifted(c, e)) cell_drift(c, e); struct part *restrict parts = c->parts; const int count = c->count; @@ -1548,9 +1542,7 @@ void DOSELF2(struct runner *r, struct cell *restrict c) { if (!cell_is_active(c, e)) return; -#ifdef SWIFT_DEBUG_CHECKS - cell_is_drifted(c, e); -#endif + if (!cell_is_drifted(c, e)) error("Cell is not drifted"); struct part *restrict parts = c->parts; const int count = c->count; diff --git a/src/scheduler.c b/src/scheduler.c index 0d7c8c4754bac931c7886200176e3e9441c63c53..df50f8525529d734abde5128ef75bef955d73f55 100644 --- a/src/scheduler.c +++ b/src/scheduler.c @@ -133,6 +133,7 @@ static void scheduler_splittask(struct task *t, struct scheduler *s) { /* Non-splittable task? */ if ((t->ci == NULL || (t->type == task_type_pair && t->cj == NULL)) || ((t->type == task_type_kick) && t->ci->nodeID != s->nodeID) || + ((t->type == task_type_drift) && t->ci->nodeID != s->nodeID) || ((t->type == task_type_init) && t->ci->nodeID != s->nodeID)) { t->type = task_type_none; t->skip = 1; @@ -214,7 +215,7 @@ static void scheduler_splittask(struct task *t, struct scheduler *s) { /* Get the sort ID, use space_getsid and not t->flags to make sure we get ci and cj swapped if needed. */ double shift[3]; - int sid = space_getsid(s->space, &ci, &cj, shift); + const int sid = space_getsid(s->space, &ci, &cj, shift); /* Should this task be split-up? */ if (ci->split && cj->split && @@ -782,7 +783,10 @@ void scheduler_set_unlocks(struct scheduler *s) { for (int i = 0; i < t->nr_unlock_tasks; i++) { for (int j = i + 1; j < t->nr_unlock_tasks; j++) { if (t->unlock_tasks[i] == t->unlock_tasks[j]) - error("duplicate unlock!"); + error("duplicate unlock! t->type=%s/%s unlocking type=%s/%s", + taskID_names[t->type], subtaskID_names[t->subtype], + taskID_names[t->unlock_tasks[i]->type], + subtaskID_names[t->unlock_tasks[i]->subtype]); } } } @@ -1065,7 +1069,7 @@ void scheduler_start(struct scheduler *s) { if (cj == NULL) { /* self */ if (ci->ti_end_min == ti_current && t->skip && - t->type != task_type_sort) + t->type != task_type_sort && t->type) error( "Task (type='%s/%s') should not have been skipped ti_current=%d " "c->ti_end_min=%d", @@ -1148,6 +1152,7 @@ void scheduler_enqueue(struct scheduler *s, struct task *t) { case task_type_sort: case task_type_ghost: case task_type_kick: + case task_type_drift: case task_type_init: qid = t->ci->super->owner; break; diff --git a/src/space.c b/src/space.c index 44bafdd2bb2c7a930c1a6e6691b3ea0beca66683..41e2962bb8790a5a63d9f5ce3ec01f093e8fd8ab 100644 --- a/src/space.c +++ b/src/space.c @@ -214,6 +214,7 @@ void space_rebuild_recycle_mapper(void *map_data, int num_elements, c->extra_ghost = NULL; c->ghost = NULL; c->kick = NULL; + c->drift = NULL; c->cooling = NULL; c->sourceterms = NULL; c->super = c; @@ -345,6 +346,12 @@ void space_regrid(struct space *s, int verbose) { if (s->cells_top == NULL || cdim[0] < s->cdim[0] || cdim[1] < s->cdim[1] || cdim[2] < s->cdim[2]) { +/* Be verbose about this. */ +#ifdef SWIFT_DEBUG_CHECKS + message("re)griding space cdim=(%d %d %d)", cdim[0], cdim[1], cdim[2]); + fflush(stdout); +#endif + /* Free the old cells, if they were allocated. */ if (s->cells_top != NULL) { threadpool_map(&s->e->threadpool, space_rebuild_recycle_mapper, @@ -461,8 +468,11 @@ void space_rebuild(struct space *s, int verbose) { const ticks tic = getticks(); - /* Be verbose about this. */ - // message("re)building space..."); fflush(stdout); +/* Be verbose about this. */ +#ifdef SWIFT_DEBUG_CHECKS + message("re)building space"); + fflush(stdout); +#endif /* Re-grid if necessary, or just re-set the cell data. */ space_regrid(s, verbose); @@ -1471,7 +1481,6 @@ void space_split_recursive(struct space *s, struct cell *c, int *buff) { struct part *parts = c->parts; struct gpart *gparts = c->gparts; struct xpart *xparts = c->xparts; - struct engine *e = s->e; /* If the buff is NULL, allocate it, and remember to free it. */ const int allocate_buffer = (buff == NULL); @@ -1501,7 +1510,7 @@ void space_split_recursive(struct space *s, struct cell *c, int *buff) { temp = space_getcell(s); temp->count = 0; temp->gcount = 0; - temp->ti_old = e->ti_current; + temp->ti_old = c->ti_old; temp->loc[0] = c->loc[0]; temp->loc[1] = c->loc[1]; temp->loc[2] = c->loc[2]; diff --git a/src/task.c b/src/task.c index ea97fdd1bb930d005889fa7c73a3f2cb7b5f054a..71fb0af8a872a10a728d1e13e8eb65a7716558cc 100644 --- a/src/task.c +++ b/src/task.c @@ -48,10 +48,10 @@ /* Task type names. */ const char *taskID_names[task_type_count] = { - "none", "sort", "self", "pair", "sub_self", - "sub_pair", "init", "ghost", "extra_ghost", "kick", - "send", "recv", "grav_gather_m", "grav_fft", "grav_mm", - "grav_up", "cooling", "sourceterms"}; + "none", "sort", "self", "pair", "sub_self", + "sub_pair", "init", "ghost", "extra_ghost", "drift", + "kick", "send", "recv", "grav_gather_m", "grav_fft", + "grav_mm", "grav_up", "cooling", "sourceterms"}; const char *subtaskID_names[task_subtype_count] = { "none", "density", "gradient", "force", "grav", "external_grav", "tend"}; @@ -150,6 +150,7 @@ __attribute__((always_inline)) INLINE static enum task_actions task_acts_on( case task_type_kick: case task_type_send: case task_type_recv: + case task_type_drift: if (t->ci->count > 0 && t->ci->gcount > 0) return task_action_all; else if (t->ci->count > 0) @@ -260,6 +261,11 @@ void task_unlock(struct task *t) { /* Act based on task type. */ switch (type) { + case task_type_drift: + cell_unlocktree(ci); + cell_gunlocktree(ci); + break; + case task_type_sort: cell_unlocktree(ci); break; @@ -327,6 +333,15 @@ int task_lock(struct task *t) { #endif break; + case task_type_drift: + if (ci->hold || ci->ghold) return 0; + if (cell_locktree(ci) != 0) return 0; + if (cell_glocktree(ci) != 0) { + cell_unlocktree(ci); + return 0; + } + break; + case task_type_sort: if (cell_locktree(ci) != 0) return 0; break; diff --git a/src/task.h b/src/task.h index c9425fdd137e2c1708dbd05436d1db685bdd3bfd..b4767f036c40cced9652422da745dd313395d479 100644 --- a/src/task.h +++ b/src/task.h @@ -45,6 +45,7 @@ enum task_types { task_type_init, task_type_ghost, task_type_extra_ghost, + task_type_drift, task_type_kick, task_type_send, task_type_recv,