Commit e7b9d012 authored by Matthieu Schaller's avatar Matthieu Schaller
Browse files

Added MPI tasks for the BH density and feedback.

parent 447086d3
......@@ -511,7 +511,6 @@ int main(int argc, char *argv[]) {
if (with_star_formation && with_feedback)
error("Can't run with star formation and feedback over MPI (yet)");
if (with_limiter) error("Can't run with time-step limiter over MPI (yet)");
if (with_black_holes) error("Can't run with black holes over MPI (yet)");
#endif
/* Temporary early aborts for modes not supported with hand-vec. */
......
......@@ -372,7 +372,7 @@ __attribute__((always_inline)) INLINE static int bpart_is_active(
if (ti_end < ti_current)
error(
"s-particle in an impossible time-zone! bp->ti_end=%lld "
"b-particle in an impossible time-zone! bp->ti_end=%lld "
"e->ti_current=%lld",
ti_end, ti_current);
#endif
......
......@@ -497,12 +497,15 @@ int cell_pack(struct cell *restrict c, struct pcell *restrict pc,
/* Start by packing the data of the current cell. */
pc->hydro.h_max = c->hydro.h_max;
pc->stars.h_max = c->stars.h_max;
pc->black_holes.h_max = c->black_holes.h_max;
pc->hydro.ti_end_min = c->hydro.ti_end_min;
pc->hydro.ti_end_max = c->hydro.ti_end_max;
pc->grav.ti_end_min = c->grav.ti_end_min;
pc->grav.ti_end_max = c->grav.ti_end_max;
pc->stars.ti_end_min = c->stars.ti_end_min;
pc->stars.ti_end_max = c->stars.ti_end_max;
pc->black_holes.ti_end_min = c->black_holes.ti_end_min;
pc->black_holes.ti_end_max = c->black_holes.ti_end_max;
pc->hydro.ti_old_part = c->hydro.ti_old_part;
pc->grav.ti_old_part = c->grav.ti_old_part;
pc->grav.ti_old_multipole = c->grav.ti_old_multipole;
......@@ -510,6 +513,7 @@ int cell_pack(struct cell *restrict c, struct pcell *restrict pc,
pc->hydro.count = c->hydro.count;
pc->grav.count = c->grav.count;
pc->stars.count = c->stars.count;
pc->black_holes.count = c->black_holes.count;
pc->maxdepth = c->maxdepth;
/* Copy the Multipole related information */
......@@ -602,19 +606,24 @@ int cell_unpack(struct pcell *restrict pc, struct cell *restrict c,
/* Unpack the current pcell. */
c->hydro.h_max = pc->hydro.h_max;
c->stars.h_max = pc->stars.h_max;
c->black_holes.h_max = pc->black_holes.h_max;
c->hydro.ti_end_min = pc->hydro.ti_end_min;
c->hydro.ti_end_max = pc->hydro.ti_end_max;
c->grav.ti_end_min = pc->grav.ti_end_min;
c->grav.ti_end_max = pc->grav.ti_end_max;
c->stars.ti_end_min = pc->stars.ti_end_min;
c->stars.ti_end_max = pc->stars.ti_end_max;
c->black_holes.ti_end_min = pc->black_holes.ti_end_min;
c->black_holes.ti_end_max = pc->black_holes.ti_end_max;
c->hydro.ti_old_part = pc->hydro.ti_old_part;
c->grav.ti_old_part = pc->grav.ti_old_part;
c->grav.ti_old_multipole = pc->grav.ti_old_multipole;
c->stars.ti_old_part = pc->stars.ti_old_part;
c->black_holes.ti_old_part = pc->black_holes.ti_old_part;
c->hydro.count = pc->hydro.count;
c->grav.count = pc->grav.count;
c->stars.count = pc->stars.count;
c->black_holes.count = pc->black_holes.count;
c->maxdepth = pc->maxdepth;
#ifdef SWIFT_DEBUG_CHECKS
......@@ -664,6 +673,7 @@ int cell_unpack(struct pcell *restrict pc, struct cell *restrict c,
temp->hydro.dx_max_sort = 0.f;
temp->stars.dx_max_part = 0.f;
temp->stars.dx_max_sort = 0.f;
temp->black_holes.dx_max_part = 0.f;
temp->nodeID = c->nodeID;
temp->parent = c;
c->progeny[k] = temp;
......@@ -2452,7 +2462,6 @@ void cell_activate_stars_sorts_up(struct cell *c, struct scheduler *s) {
#endif
scheduler_activate(s, c->stars.sorts);
if (c->nodeID == engine_rank) {
// MATTHIEU: to do: do we actually need both drifts here?
cell_activate_drift_spart(c, s);
}
} else {
......@@ -3610,22 +3619,18 @@ int cell_unskip_black_holes_tasks(struct cell *c, struct scheduler *s) {
scheduler_activate(s, ci->mpi.hydro.recv_rho);
/* If the local cell is active, more stuff will be needed. */
error("MATTHIEU: needs implementing");
// scheduler_activate_send(s, cj->mpi.black_holes.send, ci_nodeID);
scheduler_activate_send(s, cj->mpi.black_holes.send, ci_nodeID);
cell_activate_drift_bpart(cj, s);
/* If the local cell is active, send its ti_end values. */
error("MATTHIEU: needs implementing");
// scheduler_activate_send(s, cj->mpi.black_holes.send_ti, ci_nodeID);
scheduler_activate_send(s, cj->mpi.black_holes.send_ti, ci_nodeID);
}
if (ci_active) {
error("MATTHIEU: needs implementing");
// scheduler_activate(s, ci->mpi.black_holes.recv);
scheduler_activate(s, ci->mpi.black_holes.recv);
/* If the foreign cell is active, we want its ti_end values. */
error("MATTHIEU: needs implementing");
// scheduler_activate(s, ci->mpi.black_holes.recv_ti);
scheduler_activate(s, ci->mpi.black_holes.recv_ti);
/* Is the foreign cell active and will need stuff from us? */
scheduler_activate_send(s, cj->mpi.hydro.send_xv, ci_nodeID);
......@@ -3643,22 +3648,18 @@ int cell_unskip_black_holes_tasks(struct cell *c, struct scheduler *s) {
scheduler_activate(s, cj->mpi.hydro.recv_rho);
/* If the local cell is active, more stuff will be needed. */
error("MATTHIEU: needs implementing");
// scheduler_activate_send(s, ci->mpi.black_holes.send, cj_nodeID);
scheduler_activate_send(s, ci->mpi.black_holes.send, cj_nodeID);
cell_activate_drift_bpart(ci, s);
/* If the local cell is active, send its ti_end values. */
error("MATTHIEU: needs implementing");
// scheduler_activate_send(s, ci->mpi.black_holes.send_ti, cj_nodeID);
scheduler_activate_send(s, ci->mpi.black_holes.send_ti, cj_nodeID);
}
if (cj_active) {
error("MATTHIEU: needs implementing");
// scheduler_activate(s, cj->mpi.black_holes.recv);
scheduler_activate(s, cj->mpi.black_holes.recv);
/* If the foreign cell is active, we want its ti_end values. */
error("MATTHIEU: needs implementing");
// scheduler_activate(s, cj->mpi.black_holes.recv_ti);
scheduler_activate(s, cj->mpi.black_holes.recv_ti);
/* Is the foreign cell active and will need stuff from us? */
scheduler_activate_send(s, ci->mpi.hydro.send_xv, cj_nodeID);
......@@ -4573,7 +4574,8 @@ void cell_check_timesteps(struct cell *c) {
#ifdef SWIFT_DEBUG_CHECKS
if (c->hydro.ti_end_min == 0 && c->grav.ti_end_min == 0 &&
c->stars.ti_end_min == 0 && c->nr_tasks > 0)
c->stars.ti_end_min == 0 && c->black_holes.ti_end_min == 0 &&
c->nr_tasks > 0)
error("Cell without assigned time-step");
if (c->split) {
......
......@@ -172,6 +172,26 @@ struct pcell {
} stars;
/*! Black hole variables */
struct {
/*! Number of #spart in this cell. */
int count;
/*! Maximal smoothing length. */
double h_max;
/*! Minimal integer end-of-timestep in this cell for black hole tasks */
integertime_t ti_end_min;
/*! Maximal integer end-of-timestep in this cell for black hole tasks */
integertime_t ti_end_max;
/*! Integer time of the last drift of the #spart in this cell */
integertime_t ti_old_part;
} black_holes;
/*! Maximal depth in that part of the tree */
int maxdepth;
......
......@@ -2875,9 +2875,7 @@ void engine_collect_end_of_step_recurse_black_holes(struct cell *c,
/* Skip super-cells (Their values are already set) */
#ifdef WITH_MPI
// MATTHIEU
if (c->timestep != NULL)
return; // || c->mpi.black_holes.recv_ti != NULL) return;
if (c->timestep != NULL || c->mpi.black_holes.recv_ti != NULL) return;
#else
if (c->timestep != NULL) return;
#endif /* WITH_MPI */
......@@ -3305,8 +3303,9 @@ void engine_skip_force_and_kick(struct engine *e) {
/* Skip everything that updates the particles */
if (t->type == task_type_drift_part || t->type == task_type_drift_gpart ||
t->type == task_type_drift_spart || t->type == task_type_kick1 ||
t->type == task_type_kick2 || t->type == task_type_timestep ||
t->type == task_type_drift_spart || t->type == task_type_drift_bpart ||
t->type == task_type_kick1 || t->type == task_type_kick2 ||
t->type == task_type_timestep ||
t->type == task_type_timestep_limiter ||
t->subtype == task_subtype_force ||
t->subtype == task_subtype_limiter || t->subtype == task_subtype_grav ||
......@@ -3324,6 +3323,7 @@ void engine_skip_force_and_kick(struct engine *e) {
t->subtype == task_subtype_tend_part ||
t->subtype == task_subtype_tend_gpart ||
t->subtype == task_subtype_tend_spart ||
t->subtype == task_subtype_tend_bpart ||
t->subtype == task_subtype_rho || t->subtype == task_subtype_gpart)
t->skip = 1;
}
......
......@@ -292,6 +292,75 @@ void engine_addtasks_send_stars(struct engine *e, struct cell *ci,
#endif
}
/**
* @brief Add send tasks for the black holes pairs to a hierarchy of cells.
*
* @param e The #engine.
* @param ci The sending #cell.
* @param cj Dummy cell containing the nodeID of the receiving node.
* @param t_feedback The send_feed #task, if it has already been created.
* @param t_ti The recv_ti_end #task, if it has already been created.
*/
void engine_addtasks_send_black_holes(struct engine *e, struct cell *ci,
struct cell *cj, struct task *t_feedback,
struct task *t_ti) {
#ifdef WITH_MPI
struct link *l = NULL;
struct scheduler *s = &e->sched;
const int nodeID = cj->nodeID;
/* Check if any of the density tasks are for the target node. */
for (l = ci->black_holes.density; l != NULL; l = l->next)
if (l->t->ci->nodeID == nodeID ||
(l->t->cj != NULL && l->t->cj->nodeID == nodeID))
break;
/* If so, attach send tasks. */
if (l != NULL) {
if (t_feedback == NULL) {
/* Make sure this cell is tagged. */
cell_ensure_tagged(ci);
/* Create the tasks and their dependencies? */
t_feedback = scheduler_addtask(s, task_type_send, task_subtype_bpart,
ci->mpi.tag, 0, ci, cj);
t_ti = scheduler_addtask(s, task_type_send, task_subtype_tend_bpart,
ci->mpi.tag, 0, ci, cj);
/* The send_black_holes task should unlock the super_cell's kick task. */
scheduler_addunlock(s, t_feedback,
ci->hydro.super->black_holes.black_holes_out);
/* Ghost before you send */
scheduler_addunlock(s, ci->hydro.super->black_holes.ghost, t_feedback);
/* Drift before you send */
scheduler_addunlock(s, ci->hydro.super->black_holes.drift, t_feedback);
scheduler_addunlock(s, ci->super->timestep, t_ti);
}
engine_addlink(e, &ci->mpi.black_holes.send, t_feedback);
engine_addlink(e, &ci->mpi.black_holes.send_ti, t_ti);
}
/* Recurse? */
if (ci->split)
for (int k = 0; k < 8; k++)
if (ci->progeny[k] != NULL)
engine_addtasks_send_black_holes(e, ci->progeny[k], cj, t_feedback,
t_ti);
#else
error("SWIFT was not compiled with MPI support.");
#endif
}
/**
* @brief Add recv tasks for hydro pairs to a hierarchy of cells.
*
......@@ -445,6 +514,64 @@ void engine_addtasks_recv_stars(struct engine *e, struct cell *c,
#endif
}
/**
* @brief Add recv tasks for black_holes pairs to a hierarchy of cells.
*
* @param e The #engine.
* @param c The foreign #cell.
* @param t_feedback The recv_feed #task, if it has already been created.
* @param t_ti The recv_ti_end #task, if it has already been created.
*/
void engine_addtasks_recv_black_holes(struct engine *e, struct cell *c,
struct task *t_feedback,
struct task *t_ti) {
#ifdef WITH_MPI
struct scheduler *s = &e->sched;
/* Have we reached a level where there are any black_holes tasks ? */
if (t_feedback == NULL && c->black_holes.density != NULL) {
#ifdef SWIFT_DEBUG_CHECKS
/* Make sure this cell has a valid tag. */
if (c->mpi.tag < 0) error("Trying to receive from untagged cell.");
#endif // SWIFT_DEBUG_CHECKS
/* Create the tasks. */
t_feedback = scheduler_addtask(s, task_type_recv, task_subtype_bpart,
c->mpi.tag, 0, c, NULL);
t_ti = scheduler_addtask(s, task_type_recv, task_subtype_tend_bpart,
c->mpi.tag, 0, c, NULL);
}
c->mpi.black_holes.recv = t_feedback;
c->mpi.black_holes.recv_ti = t_ti;
#ifdef SWIFT_DEBUG_CHECKS
if (c->nodeID == e->nodeID) error("Local cell!");
#endif
for (struct link *l = c->black_holes.density; l != NULL; l = l->next) {
scheduler_addunlock(s, l->t, t_feedback);
}
for (struct link *l = c->black_holes.feedback; l != NULL; l = l->next) {
scheduler_addunlock(s, t_feedback, l->t);
scheduler_addunlock(s, l->t, t_ti);
}
/* Recurse? */
if (c->split)
for (int k = 0; k < 8; k++)
if (c->progeny[k] != NULL)
engine_addtasks_recv_black_holes(e, c->progeny[k], t_feedback, t_ti);
#else
error("SWIFT was not compiled with MPI support.");
#endif
}
/**
* @brief Add recv tasks for gravity pairs to a hierarchy of cells.
*
......@@ -1502,7 +1629,7 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
}
/* The black hole feedback tasks */
if (with_feedback) {
if (with_black_holes) {
t_bh_density = scheduler_addtask(
sched, task_type_self, task_subtype_bh_density, flags, 0, ci, NULL);
t_bh_feedback =
......@@ -1563,7 +1690,7 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
ci->hydro.super->stars.stars_out);
}
if (with_feedback) {
if (with_black_holes) {
scheduler_addunlock(sched, ci->hydro.super->black_holes.drift,
t_bh_density);
......@@ -1959,7 +2086,7 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
}
/* The black hole feedback tasks */
if (with_feedback) {
if (with_black_holes) {
t_bh_density =
scheduler_addtask(sched, task_type_sub_pair,
task_subtype_bh_density, flags, 0, ci, cj);
......@@ -2107,7 +2234,7 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
cj->hydro.super->stars.stars_out);
}
if (with_feedback) {
if (with_black_holes) {
scheduler_addunlock(sched, cj->hydro.super->black_holes.drift,
t_bh_density);
......@@ -2160,6 +2287,7 @@ void engine_make_hydroloop_tasks_mapper(void *map_data, int num_elements,
const int periodic = e->s->periodic;
const int with_feedback = (e->policy & engine_policy_feedback);
const int with_stars = (e->policy & engine_policy_stars);
const int with_black_holes = (e->policy & engine_policy_black_holes);
struct space *s = e->s;
struct scheduler *sched = &e->sched;
......@@ -2182,7 +2310,8 @@ void engine_make_hydroloop_tasks_mapper(void *map_data, int num_elements,
struct cell *ci = &cells[cid];
/* Skip cells without hydro or star particles */
if ((ci->hydro.count == 0) && (!with_stars || ci->stars.count == 0))
if ((ci->hydro.count == 0) && (!with_stars || ci->stars.count == 0) &&
(!with_black_holes || ci->black_holes.count == 0))
continue;
/* If the cell is local build a self-interaction */
......@@ -2212,7 +2341,8 @@ void engine_make_hydroloop_tasks_mapper(void *map_data, int num_elements,
/* Is that neighbour local and does it have gas or star particles ? */
if ((cid >= cjd) ||
((cj->hydro.count == 0) &&
(!with_feedback || cj->stars.count == 0)) ||
(!with_feedback || cj->stars.count == 0) &&
(!with_black_holes || cj->black_holes.count == 0)) ||
(ci->nodeID != nodeID && cj->nodeID != nodeID))
continue;
......@@ -2287,9 +2417,6 @@ void engine_addtasks_send_mapper(void *map_data, int num_elements,
struct cell *cj = cell_type_pairs[k].cj;
const int type = cell_type_pairs[k].type;
/* Add the send task for the particle timesteps. */
// engine_addtasks_send_timestep(e, ci, cj, NULL, NULL, with_limiter);
/* Add the send tasks for the cells in the proxy that have a hydro
* connection. */
if ((e->policy & engine_policy_hydro) && (type & proxy_cell_type_hydro))
......@@ -2302,6 +2429,13 @@ void engine_addtasks_send_mapper(void *map_data, int num_elements,
if ((e->policy & engine_policy_feedback) && (type & proxy_cell_type_hydro))
engine_addtasks_send_stars(e, ci, cj, /*t_feedback=*/NULL, /*t_ti=*/NULL);
/* Add the send tasks for the cells in the proxy that have a black holes
* connection. */
if ((e->policy & engine_policy_black_holes) &&
(type & proxy_cell_type_hydro))
engine_addtasks_send_black_holes(e, ci, cj, /*t_feedback=*/NULL,
/*t_ti=*/NULL);
/* Add the send tasks for the cells in the proxy that have a gravity
* connection. */
if ((e->policy & engine_policy_self_gravity) &&
......@@ -2321,9 +2455,6 @@ void engine_addtasks_recv_mapper(void *map_data, int num_elements,
struct cell *ci = cell_type_pairs[k].ci;
const int type = cell_type_pairs[k].type;
/* Add the recv task for the particle timesteps. */
// engine_addtasks_recv_timestep(e, ci, NULL, NULL, with_limiter);
/* Add the recv tasks for the cells in the proxy that have a hydro
* connection. */
if ((e->policy & engine_policy_hydro) && (type & proxy_cell_type_hydro))
......@@ -2334,6 +2465,11 @@ void engine_addtasks_recv_mapper(void *map_data, int num_elements,
if ((e->policy & engine_policy_feedback) && (type & proxy_cell_type_hydro))
engine_addtasks_recv_stars(e, ci, NULL, NULL);
/* Add the recv tasks for the cells in the proxy that have a black holes
* connection. */
if ((e->policy & engine_policy_feedback) && (type & proxy_cell_type_hydro))
engine_addtasks_recv_black_holes(e, ci, NULL, NULL);
/* Add the recv tasks for the cells in the proxy that have a gravity
* connection. */
if ((e->policy & engine_policy_self_gravity) &&
......
......@@ -637,23 +637,18 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
scheduler_activate(s, ci->mpi.hydro.recv_rho);
/* If the local cell is active, more stuff will be needed. */
error("MATTHIEU: needs implementing");
// scheduler_activate_send(s, cj->mpi.black_holes.send, ci_nodeID);
scheduler_activate_send(s, cj->mpi.black_holes.send, ci_nodeID);
cell_activate_drift_bpart(cj, s);
/* If the local cell is active, send its ti_end values. */
error("MATTHIEU: needs implementing");
// scheduler_activate_send(s, cj->mpi.black_holes.send_ti,
// ci_nodeID);
scheduler_activate_send(s, cj->mpi.black_holes.send_ti, ci_nodeID);
}
if (ci_active_black_holes) {
error("MATTHIEU: needs implementing");
// scheduler_activate(s, ci->mpi.black_holes.recv);
scheduler_activate(s, ci->mpi.black_holes.recv);
/* If the foreign cell is active, we want its ti_end values. */
error("MATTHIEU: needs implementing");
// scheduler_activate(s, ci->mpi.black_holes.recv_ti);
scheduler_activate(s, ci->mpi.black_holes.recv_ti);
/* Is the foreign cell active and will need stuff from us? */
scheduler_activate_send(s, cj->mpi.hydro.send_xv, ci_nodeID);
......@@ -672,23 +667,18 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
scheduler_activate(s, cj->mpi.hydro.recv_rho);
/* If the local cell is active, more stuff will be needed. */
error("MATTHIEU: needs implementing");
// scheduler_activate_send(s, ci->mpi.black_holes.send, cj_nodeID);
scheduler_activate_send(s, ci->mpi.black_holes.send, cj_nodeID);
cell_activate_drift_bpart(ci, s);
/* If the local cell is active, send its ti_end values. */
error("MATTHIEU: needs implementing");
// scheduler_activate_send(s, ci->mpi.black_holes.send_ti,
// cj_nodeID);
scheduler_activate_send(s, ci->mpi.black_holes.send_ti, cj_nodeID);
}
if (cj_active_black_holes) {
error("MATTHIEU: needs implementing");
// scheduler_activate(s, cj->mpi.black_holes.recv);
scheduler_activate(s, cj->mpi.black_holes.recv);
/* If the foreign cell is active, we want its ti_end values. */
error("MATTHIEU: needs implementing");
// scheduler_activate(s, cj->mpi.black_holes.recv_ti);
scheduler_activate(s, cj->mpi.black_holes.recv_ti);
/* Is the foreign cell active and will need stuff from us? */
scheduler_activate_send(s, ci->mpi.hydro.send_xv, cj_nodeID);
......
......@@ -64,7 +64,7 @@ extern int engine_rank;
clocks_get_timesincestart(), __FILE__, __FUNCTION__, __LINE__, \
##__VA_ARGS__); \
memdump(engine_rank); \
MPI_Abort(MPI_COMM_WORLD, -1); \
swift_abort(-1); \
})
#else
extern int engine_rank;
......
......@@ -3845,6 +3845,92 @@ void runner_do_recv_spart(struct runner *r, struct cell *c, int clear_sorts,
#endif
}
/**
* @brief Construct the cell properties from the received #bpart.
*
* Note that we do not need to clear the sorts since we do not sort
* the black holes.
*
* @param r The runner thread.
* @param c The cell.
* @param clear_sorts Should we clear the sort flag and hence trigger a sort ?
* @param timer Are we timing this ?
*/
void runner_do_recv_bpart(struct runner *r, struct cell *c, int clear_sorts,
int timer) {
#ifdef WITH_MPI
struct bpart *restrict bparts = c->black_holes.parts;
const size_t nr_bparts = c->black_holes.count;
const integertime_t ti_current = r->e->ti_current;
TIMER_TIC;
integertime_t ti_black_holes_end_min = max_nr_timesteps;
integertime_t ti_black_holes_end_max = 0;
timebin_t time_bin_min = num_time_bins;
timebin_t time_bin_max = 0;
float h_max = 0.f;
#ifdef SWIFT_DEBUG_CHECKS
if (c->nodeID == engine_rank) error("Updating a local cell!");
#endif
/* If this cell is a leaf, collect the particle data. */
if (!c->split) {
/* Collect everything... */
for (size_t k = 0; k < nr_bparts; k++) {
#ifdef DEBUG_INTERACTIONS_BLACK_HOLES
bparts[k].num_ngb_force = 0;
#endif
if (bparts[k].time_bin == time_bin_inhibited) continue;
time_bin_min = min(time_bin_min, bparts[k].time_bin);
time_bin_max = max(time_bin_max, bparts[k].time_bin);
h_max = max(h_max, bparts[k].h);
}
/* Convert into a time */
ti_black_holes_end_min = get_integer_time_end(ti_current, time_bin_min);
ti_black_holes_end_max = get_integer_time_end(ti_current, time_bin_max);
}
/* Otherwise, recurse and collect. */
else {
for (int k = 0; k < 8; k++) {
if (c->progeny[k] != NULL && c->progeny[k]->black_holes.count > 0) {
runner_do_recv_bpart(r, c->progeny[k], clear_sorts, 0);
ti_black_holes_end_min =
min(ti_black_holes_end_min, c->progeny[k]->black_holes.ti_end_min);
ti_black_holes_end_max =
max(ti_black_holes_end_max, c->progeny[k]->grav.ti_end_max);
h_max = max(h_max, c->progeny[k]->black_holes.h_max);
}
}
}
#ifdef SWIFT_DEBUG_CHECKS
if (ti_black_holes_end_min < ti_current)
error(
"Received a cell at an incorrect time c->ti_end_min=%lld, "
"e->ti_current=%lld.",
ti_black_holes_end_min, ti_current);
#endif
/* ... and store. */
// c->grav.ti_end_min = ti_gravity_end_min;
// c->grav.ti_end_max = ti_gravity_end_max;
c->black_holes.ti_old_part = ti_current;
c->black_holes.h_max = h_max;
if (timer) TIMER_TOC(timer_dorecv_bpart);
#else
error("SWIFT was not compiled with MPI support.");
#endif
}
/**
* @brief The #runner main thread routine.
*
......@@ -4113,6 +4199,8 @@ void *runner_main(void *data) {
runner_do_recv_gpart(r, ci, 1);
} else if (t->subtype == task_subtype_spart) {
runner_do_recv_spart(r, ci, 1, 1);
} else if (t->subtype == task_subtype_bpart) {
runner_do_recv_bpart(r, ci, 1, 1);
} else if (t->subtype == task_subtype_multipole) {
cell_unpack_multipoles(ci, (struct gravity_tensors *)t->buff);
free(t->buff);
......