diff --git a/src/cell.c b/src/cell.c
index 2e49e57d1f3e3c6566d37222adcd2273be0c33c5..23a444030dbbc71b34f060e9ca098b0e854b3652 100644
--- a/src/cell.c
+++ b/src/cell.c
@@ -1882,6 +1882,8 @@ void cell_clean_links(struct cell *c, void *data) {
c->grav.mm = NULL;
c->stars.density = NULL;
c->stars.feedback = NULL;
+ c->black_holes.density = NULL;
+ c->black_holes.feedback = NULL;
}
/**
@@ -2407,6 +2409,44 @@ void cell_activate_drift_spart(struct cell *c, struct scheduler *s) {
}
}
+/**
+ * @brief Activate the #bpart drifts on the given cell.
+ */
+void cell_activate_drift_bpart(struct cell *c, struct scheduler *s) {
+
+ /* If this cell is already marked for drift, quit early. */
+ if (cell_get_flag(c, cell_flag_do_bh_drift)) return;
+
+ /* Mark this cell for drifting. */
+ cell_set_flag(c, cell_flag_do_bh_drift);
+
+ /* Set the do_black_holes_sub_drifts all the way up and activate the super
+ drift if this has not yet been done. */
+ if (c == c->hydro.super) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->black_holes.drift == NULL)
+ error("Trying to activate un-existing c->black_holes.drift");
+#endif
+ scheduler_activate(s, c->black_holes.drift);
+ } else {
+ for (struct cell *parent = c->parent;
+ parent != NULL && !cell_get_flag(parent, cell_flag_do_bh_sub_drift);
+ parent = parent->parent) {
+ /* Mark this cell for drifting */
+ cell_set_flag(parent, cell_flag_do_bh_sub_drift);
+
+ if (parent == c->hydro.super) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (parent->black_holes.drift == NULL)
+ error("Trying to activate un-existing parent->black_holes.drift");
+#endif
+ scheduler_activate(s, parent->black_holes.drift);
+ break;
+ }
+ }
+ }
+}
+
/**
* @brief Activate the drifts on the given cell.
*/
@@ -2508,7 +2548,6 @@ void cell_activate_stars_sorts_up(struct cell *c, struct scheduler *s) {
#endif
scheduler_activate(s, c->stars.sorts);
if (c->nodeID == engine_rank) {
- // MATTHIEU: to do: do we actually need both drifts here?
cell_activate_drift_spart(c, s);
}
} else {
@@ -2758,6 +2797,100 @@ void cell_activate_subcell_stars_tasks(struct cell *ci, struct cell *cj,
} /* Otherwise, pair interation */
}
+/**
+ * @brief Traverse a sub-cell task and activate the black_holes drift tasks that
+ * are required by a black_holes task
+ *
+ * @param ci The first #cell we recurse in.
+ * @param cj The second #cell we recurse in.
+ * @param s The task #scheduler.
+ */
+void cell_activate_subcell_black_holes_tasks(struct cell *ci, struct cell *cj,
+ struct scheduler *s) {
+ const struct engine *e = s->space->e;
+
+ /* Store the current dx_max and h_max values. */
+ ci->black_holes.dx_max_part_old = ci->black_holes.dx_max_part;
+ ci->black_holes.h_max_old = ci->black_holes.h_max;
+ ci->hydro.dx_max_part_old = ci->hydro.dx_max_part;
+ ci->hydro.h_max_old = ci->hydro.h_max;
+
+ if (cj != NULL) {
+ cj->black_holes.dx_max_part_old = cj->black_holes.dx_max_part;
+ cj->black_holes.h_max_old = cj->black_holes.h_max;
+ cj->hydro.dx_max_part_old = cj->hydro.dx_max_part;
+ cj->hydro.h_max_old = cj->hydro.h_max;
+ }
+
+ /* Self interaction? */
+ if (cj == NULL) {
+ /* Do anything? */
+ if (!cell_is_active_black_holes(ci, e) || ci->hydro.count == 0 ||
+ ci->black_holes.count == 0)
+ return;
+
+ /* Recurse? */
+ if (cell_can_recurse_in_self_black_holes_task(ci)) {
+ /* Loop over all progenies and pairs of progenies */
+ for (int j = 0; j < 8; j++) {
+ if (ci->progeny[j] != NULL) {
+ cell_activate_subcell_black_holes_tasks(ci->progeny[j], NULL, s);
+ for (int k = j + 1; k < 8; k++)
+ if (ci->progeny[k] != NULL)
+ cell_activate_subcell_black_holes_tasks(ci->progeny[j],
+ ci->progeny[k], s);
+ }
+ }
+ } else {
+ /* We have reached the bottom of the tree: activate drift */
+ cell_activate_drift_bpart(ci, s);
+ cell_activate_drift_part(ci, s);
+ }
+ }
+
+ /* Otherwise, pair interation */
+ else {
+ /* Should we even bother? */
+ if (!cell_is_active_black_holes(ci, e) &&
+ !cell_is_active_black_holes(cj, e))
+ return;
+
+ /* Get the orientation of the pair. */
+ double shift[3];
+ int sid = space_getsid(s->space, &ci, &cj, shift);
+
+ /* recurse? */
+ if (cell_can_recurse_in_pair_black_holes_task(ci, cj) &&
+ cell_can_recurse_in_pair_black_holes_task(cj, ci)) {
+ struct cell_split_pair *csp = &cell_split_pairs[sid];
+ for (int k = 0; k < csp->count; k++) {
+ const int pid = csp->pairs[k].pid;
+ const int pjd = csp->pairs[k].pjd;
+ if (ci->progeny[pid] != NULL && cj->progeny[pjd] != NULL)
+ cell_activate_subcell_black_holes_tasks(ci->progeny[pid],
+ cj->progeny[pjd], s);
+ }
+ }
+
+ /* Otherwise, activate the sorts and drifts. */
+ else {
+ if (cell_is_active_black_holes(ci, e)) {
+
+ /* Activate the drifts if the cells are local. */
+ if (ci->nodeID == engine_rank) cell_activate_drift_bpart(ci, s);
+ if (cj->nodeID == engine_rank) cell_activate_drift_part(cj, s);
+ }
+
+ if (cell_is_active_black_holes(cj, e)) {
+
+ /* Activate the drifts if the cells are local. */
+ if (ci->nodeID == engine_rank) cell_activate_drift_part(ci, s);
+ if (cj->nodeID == engine_rank) cell_activate_drift_bpart(cj, s);
+ }
+ }
+ } /* Otherwise, pair interation */
+}
+
/**
* @brief Traverse a sub-cell task and activate the gravity drift tasks that
* are required by a self gravity task.
@@ -2917,9 +3050,13 @@ int cell_unskip_hydro_tasks(struct cell *c, struct scheduler *s) {
struct engine *e = s->space->e;
const int nodeID = e->nodeID;
const int with_limiter = (e->policy & engine_policy_limiter);
+
#ifdef WITH_MPI
+
const int with_star_formation = e->policy & engine_policy_star_formation;
+
const int with_feedback = e->policy & engine_policy_feedback;
+
#endif
int rebuild = 0;
@@ -2986,27 +3123,30 @@ int cell_unskip_hydro_tasks(struct cell *c, struct scheduler *s) {
if (ci_nodeID != nodeID) {
/* If the local cell is active, receive data from the foreign cell. */
if (cj_active) {
- scheduler_activate(s, ci->mpi.hydro.recv_xv);
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_xv);
if (ci_active) {
- scheduler_activate(s, ci->mpi.hydro.recv_rho);
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_rho);
#ifdef EXTRA_HYDRO_LOOP
- scheduler_activate(s, ci->mpi.hydro.recv_gradient);
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_gradient);
#endif
}
}
/* If the foreign cell is active, we want its ti_end values. */
if (ci_active || with_limiter)
- scheduler_activate(s, ci->mpi.hydro.recv_ti);
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_tend_part);
- if (with_limiter) scheduler_activate(s, ci->mpi.limiter.recv);
if (with_limiter)
- scheduler_activate_send(s, cj->mpi.limiter.send, ci->nodeID);
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_limiter);
+ if (with_limiter)
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_limiter,
+ ci->nodeID);
/* Is the foreign cell active and will need stuff from us? */
if (ci_active) {
- scheduler_activate_send(s, cj->mpi.hydro.send_xv, ci_nodeID);
+
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_xv, ci_nodeID);
/* Drift the cell which will be sent; note that not all sent
particles will be drifted, only those that are needed. */
@@ -3015,54 +3155,69 @@ int cell_unskip_hydro_tasks(struct cell *c, struct scheduler *s) {
/* If the local cell is also active, more stuff will be needed. */
if (cj_active) {
- scheduler_activate_send(s, cj->mpi.hydro.send_rho, ci_nodeID);
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_rho,
+ ci_nodeID);
#ifdef EXTRA_HYDRO_LOOP
- scheduler_activate_send(s, cj->mpi.hydro.send_gradient, ci_nodeID);
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_gradient,
+ ci_nodeID);
#endif
}
}
/* If the local cell is active, send its ti_end values. */
if (cj_active || with_limiter)
- scheduler_activate_send(s, cj->mpi.hydro.send_ti, ci_nodeID);
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_tend_part,
+ ci_nodeID);
/* Propagating new star counts? */
+
if (with_star_formation && with_feedback) {
+
if (ci_active && ci->hydro.count > 0) {
+
scheduler_activate_recv(s, ci->mpi.recv, task_subtype_sf_counts);
- scheduler_activate_recv(s, ci->mpi.recv, task_subtype_tend_spart);
+
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_tend_spart);
}
+
if (cj_active && cj->hydro.count > 0) {
- scheduler_activate_send(s, cj->mpi.send, task_subtype_sf_counts, ci_nodeID);
- scheduler_activate_send(s, cj->mpi.send, task_subtype_tend_spart, ci_nodeID);
+
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_sf_counts,
+ ci_nodeID);
+
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_tend_spart,
+ ci_nodeID);
}
}
} else if (cj_nodeID != nodeID) {
/* If the local cell is active, receive data from the foreign cell. */
if (ci_active) {
- scheduler_activate(s, cj->mpi.hydro.recv_xv);
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_xv);
if (cj_active) {
- scheduler_activate(s, cj->mpi.hydro.recv_rho);
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_rho);
#ifdef EXTRA_HYDRO_LOOP
- scheduler_activate(s, cj->mpi.hydro.recv_gradient);
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_gradient);
#endif
}
}
/* If the foreign cell is active, we want its ti_end values. */
if (cj_active || with_limiter)
- scheduler_activate(s, cj->mpi.hydro.recv_ti);
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_tend_part);
- if (with_limiter) scheduler_activate(s, cj->mpi.limiter.recv);
if (with_limiter)
- scheduler_activate_send(s, ci->mpi.limiter.send, cj->nodeID);
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_limiter);
+ if (with_limiter)
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_limiter,
+ cj->nodeID);
/* Is the foreign cell active and will need stuff from us? */
if (cj_active) {
- scheduler_activate_send(s, ci->mpi.hydro.send_xv, cj_nodeID);
+
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_xv, cj_nodeID);
/* Drift the cell which will be sent; note that not all sent
particles will be drifted, only those that are needed. */
@@ -3071,27 +3226,40 @@ int cell_unskip_hydro_tasks(struct cell *c, struct scheduler *s) {
/* If the local cell is also active, more stuff will be needed. */
if (ci_active) {
- scheduler_activate_send(s, ci->mpi.hydro.send_rho, cj_nodeID);
+
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_rho,
+ cj_nodeID);
#ifdef EXTRA_HYDRO_LOOP
- scheduler_activate_send(s, ci->mpi.hydro.send_gradient, cj_nodeID);
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_gradient,
+ cj_nodeID);
#endif
}
}
/* If the local cell is active, send its ti_end values. */
if (ci_active || with_limiter)
- scheduler_activate_send(s, ci->mpi.hydro.send_ti, cj_nodeID);
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_tend_part,
+ cj_nodeID);
/* Propagating new star counts? */
+
if (with_star_formation && with_feedback) {
+
if (cj_active && cj->hydro.count > 0) {
+
scheduler_activate_recv(s, cj->mpi.recv, task_subtype_sf_counts);
- scheduler_activate_recv(s, cj->mpi.recv, task_subtype_tend_spart);
+
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_tend_spart);
}
+
if (ci_active && ci->hydro.count > 0) {
- scheduler_activate_send(s, ci->mpi.send, task_subtype_sf_counts, cj_nodeID);
- scheduler_activate_send(s, ci->mpi.send, task_subtype_tend_spart, cj_nodeID);
+
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_sf_counts,
+ cj_nodeID);
+
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_tend_spart,
+ cj_nodeID);
}
}
}
@@ -3183,14 +3351,18 @@ int cell_unskip_gravity_tasks(struct cell *c, struct scheduler *s) {
/* Activate the send/recv tasks. */
if (ci_nodeID != nodeID) {
/* If the local cell is active, receive data from the foreign cell. */
- if (cj_active) scheduler_activate(s, ci->mpi.grav.recv);
+ if (cj_active)
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_gpart);
/* If the foreign cell is active, we want its ti_end values. */
- if (ci_active) scheduler_activate(s, ci->mpi.grav.recv_ti);
+ if (ci_active)
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_tend_gpart);
/* Is the foreign cell active and will need stuff from us? */
if (ci_active) {
- scheduler_activate_send(s, cj->mpi.grav.send, ci_nodeID);
+
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_gpart,
+ ci_nodeID);
/* Drift the cell which will be sent at the level at which it is
sent, i.e. drift the cell specified in the send task (l->t)
@@ -3200,18 +3372,23 @@ int cell_unskip_gravity_tasks(struct cell *c, struct scheduler *s) {
/* If the local cell is active, send its ti_end values. */
if (cj_active)
- scheduler_activate_send(s, cj->mpi.grav.send_ti, ci_nodeID);
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_tend_gpart,
+ ci_nodeID);
} else if (cj_nodeID != nodeID) {
/* If the local cell is active, receive data from the foreign cell. */
- if (ci_active) scheduler_activate(s, cj->mpi.grav.recv);
+ if (ci_active)
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_gpart);
/* If the foreign cell is active, we want its ti_end values. */
- if (cj_active) scheduler_activate(s, cj->mpi.grav.recv_ti);
+ if (cj_active)
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_tend_gpart);
/* Is the foreign cell active and will need stuff from us? */
if (cj_active) {
- scheduler_activate_send(s, ci->mpi.grav.send, cj_nodeID);
+
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_gpart,
+ cj_nodeID);
/* Drift the cell which will be sent at the level at which it is
sent, i.e. drift the cell specified in the send task (l->t)
@@ -3221,7 +3398,8 @@ int cell_unskip_gravity_tasks(struct cell *c, struct scheduler *s) {
/* If the local cell is active, send its ti_end values. */
if (ci_active)
- scheduler_activate_send(s, ci->mpi.grav.send_ti, cj_nodeID);
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_tend_gpart,
+ cj_nodeID);
}
#endif
}
@@ -3377,26 +3555,28 @@ int cell_unskip_stars_tasks(struct cell *c, struct scheduler *s) {
/* Activate the send/recv tasks. */
if (ci_nodeID != nodeID) {
if (cj_active) {
- scheduler_activate(s, ci->mpi.hydro.recv_xv);
- scheduler_activate(s, ci->mpi.hydro.recv_rho);
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_xv);
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_rho);
/* If the local cell is active, more stuff will be needed. */
- scheduler_activate_send(s, cj->mpi.stars.send, ci_nodeID);
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_spart,
+ ci_nodeID);
cell_activate_drift_spart(cj, s);
/* If the local cell is active, send its ti_end values. */
- scheduler_activate_send(s, cj->mpi.stars.send_ti, ci_nodeID);
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_tend_spart,
+ ci_nodeID);
}
if (ci_active) {
- scheduler_activate(s, ci->mpi.stars.recv);
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_spart);
/* If the foreign cell is active, we want its ti_end values. */
- scheduler_activate(s, ci->mpi.stars.recv_ti);
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_tend_spart);
/* Is the foreign cell active and will need stuff from us? */
- scheduler_activate_send(s, cj->mpi.hydro.send_xv, ci_nodeID);
- scheduler_activate_send(s, cj->mpi.hydro.send_rho, ci_nodeID);
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_xv, ci_nodeID);
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_rho, ci_nodeID);
/* Drift the cell which will be sent; note that not all sent
particles will be drifted, only those that are needed. */
@@ -3406,26 +3586,28 @@ int cell_unskip_stars_tasks(struct cell *c, struct scheduler *s) {
} else if (cj_nodeID != nodeID) {
/* If the local cell is active, receive data from the foreign cell. */
if (ci_active) {
- scheduler_activate(s, cj->mpi.hydro.recv_xv);
- scheduler_activate(s, cj->mpi.hydro.recv_rho);
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_xv);
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_rho);
/* If the local cell is active, more stuff will be needed. */
- scheduler_activate_send(s, ci->mpi.stars.send, cj_nodeID);
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_spart,
+ cj_nodeID);
cell_activate_drift_spart(ci, s);
/* If the local cell is active, send its ti_end values. */
- scheduler_activate_send(s, ci->mpi.stars.send_ti, cj_nodeID);
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_tend_spart,
+ cj_nodeID);
}
if (cj_active) {
- scheduler_activate(s, cj->mpi.stars.recv);
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_spart);
/* If the foreign cell is active, we want its ti_end values. */
- scheduler_activate(s, cj->mpi.stars.recv_ti);
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_tend_spart);
/* Is the foreign cell active and will need stuff from us? */
- scheduler_activate_send(s, ci->mpi.hydro.send_xv, cj_nodeID);
- scheduler_activate_send(s, ci->mpi.hydro.send_rho, cj_nodeID);
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_xv, cj_nodeID);
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_rho, cj_nodeID);
/* Drift the cell which will be sent; note that not all sent
particles will be drifted, only those that are needed. */
@@ -3491,6 +3673,205 @@ int cell_unskip_stars_tasks(struct cell *c, struct scheduler *s) {
return rebuild;
}
+/**
+ * @brief Un-skips all the black hole tasks associated with a given cell and
+ * checks if the space needs to be rebuilt.
+ *
+ * @param c the #cell.
+ * @param s the #scheduler.
+ *
+ * @return 1 If the space needs rebuilding. 0 otherwise.
+ */
+int cell_unskip_black_holes_tasks(struct cell *c, struct scheduler *s) {
+ struct engine *e = s->space->e;
+ const int nodeID = e->nodeID;
+ int rebuild = 0;
+
+ if (c->black_holes.drift != NULL && cell_is_active_black_holes(c, e)) {
+ cell_activate_drift_bpart(c, s);
+ }
+
+ /* Un-skip the density tasks involved with this cell. */
+ for (struct link *l = c->black_holes.density; l != NULL; l = l->next) {
+ struct task *t = l->t;
+ struct cell *ci = t->ci;
+ struct cell *cj = t->cj;
+ const int ci_active = cell_is_active_black_holes(ci, e);
+ const int cj_active = (cj != NULL) ? cell_is_active_black_holes(cj, e) : 0;
+#ifdef WITH_MPI
+ const int ci_nodeID = ci->nodeID;
+ const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
+#else
+ const int ci_nodeID = nodeID;
+ const int cj_nodeID = nodeID;
+#endif
+
+ /* Activate the drifts */
+ if (t->type == task_type_self && ci_active) {
+ cell_activate_drift_part(ci, s);
+ cell_activate_drift_bpart(ci, s);
+ }
+
+ /* Only activate tasks that involve a local active cell. */
+ if ((ci_active || cj_active) &&
+ (ci_nodeID == nodeID || cj_nodeID == nodeID)) {
+ scheduler_activate(s, t);
+
+ if (t->type == task_type_pair) {
+ /* Do ci */
+ if (ci_active) {
+
+ /* Activate the drift tasks. */
+ if (ci_nodeID == nodeID) cell_activate_drift_bpart(ci, s);
+ if (cj_nodeID == nodeID) cell_activate_drift_part(cj, s);
+ }
+
+ /* Do cj */
+ if (cj_active) {
+
+ /* Activate the drift tasks. */
+ if (cj_nodeID == nodeID) cell_activate_drift_bpart(cj, s);
+ if (ci_nodeID == nodeID) cell_activate_drift_part(ci, s);
+ }
+ }
+
+ else if (t->type == task_type_sub_pair || t->type == task_type_sub_self) {
+ cell_activate_subcell_black_holes_tasks(ci, cj, s);
+ }
+ }
+
+ /* Only interested in pair interactions as of here. */
+ if (t->type == task_type_pair || t->type == task_type_sub_pair) {
+ /* Check whether there was too much particle motion, i.e. the
+ cell neighbour conditions were violated. */
+ if (cell_need_rebuild_for_black_holes_pair(ci, cj)) rebuild = 1;
+ if (cell_need_rebuild_for_black_holes_pair(cj, ci)) rebuild = 1;
+
+#ifdef WITH_MPI
+ /* Activate the send/recv tasks. */
+ if (ci_nodeID != nodeID) {
+ if (cj_active) {
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_xv);
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_rho);
+
+ /* If the local cell is active, more stuff will be needed. */
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_bpart,
+ ci_nodeID);
+ cell_activate_drift_bpart(cj, s);
+
+ /* If the local cell is active, send its ti_end values. */
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_tend_bpart,
+ ci_nodeID);
+ }
+
+ if (ci_active) {
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_bpart);
+
+ /* If the foreign cell is active, we want its ti_end values. */
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_tend_bpart);
+
+ /* Is the foreign cell active and will need stuff from us? */
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_xv, ci_nodeID);
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_rho, ci_nodeID);
+
+ /* Drift the cell which will be sent; note that not all sent
+ particles will be drifted, only those that are needed. */
+ cell_activate_drift_part(cj, s);
+ }
+
+ } else if (cj_nodeID != nodeID) {
+ /* If the local cell is active, receive data from the foreign cell. */
+ if (ci_active) {
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_xv);
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_rho);
+
+ /* If the local cell is active, more stuff will be needed. */
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_bpart,
+ cj_nodeID);
+ cell_activate_drift_bpart(ci, s);
+
+ /* If the local cell is active, send its ti_end values. */
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_tend_bpart,
+ cj_nodeID);
+ }
+
+ if (cj_active) {
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_bpart);
+
+ /* If the foreign cell is active, we want its ti_end values. */
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_tend_bpart);
+
+ /* Is the foreign cell active and will need stuff from us? */
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_xv, cj_nodeID);
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_rho, cj_nodeID);
+
+ /* Drift the cell which will be sent; note that not all sent
+ particles will be drifted, only those that are needed. */
+ cell_activate_drift_part(ci, s);
+ }
+ }
+#endif
+ }
+ }
+
+ /* Un-skip the feedback tasks involved with this cell. */
+ for (struct link *l = c->black_holes.feedback; l != NULL; l = l->next) {
+ struct task *t = l->t;
+ struct cell *ci = t->ci;
+ struct cell *cj = t->cj;
+ const int ci_active = cell_is_active_black_holes(ci, e);
+ const int cj_active = (cj != NULL) ? cell_is_active_black_holes(cj, e) : 0;
+#ifdef WITH_MPI
+ const int ci_nodeID = ci->nodeID;
+ const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
+#else
+ const int ci_nodeID = nodeID;
+ const int cj_nodeID = nodeID;
+#endif
+
+ if (t->type == task_type_self && ci_active) {
+ scheduler_activate(s, t);
+ }
+
+ else if (t->type == task_type_sub_self && ci_active) {
+ scheduler_activate(s, t);
+ }
+
+ else if (t->type == task_type_pair || t->type == task_type_sub_pair) {
+ /* We only want to activate the task if the cell is active and is
+ going to update some gas on the *local* node */
+ if ((ci_nodeID == nodeID && cj_nodeID == nodeID) &&
+ (ci_active || cj_active)) {
+ scheduler_activate(s, t);
+
+ } else if ((ci_nodeID == nodeID && cj_nodeID != nodeID) && (cj_active)) {
+ scheduler_activate(s, t);
+
+ } else if ((ci_nodeID != nodeID && cj_nodeID == nodeID) && (ci_active)) {
+ scheduler_activate(s, t);
+ }
+ }
+
+ /* Nothing more to do here, all drifts and sorts activated above */
+ }
+
+ /* Unskip all the other task types. */
+ if (c->nodeID == nodeID && cell_is_active_black_holes(c, e)) {
+ if (c->black_holes.ghost != NULL)
+ scheduler_activate(s, c->black_holes.ghost);
+ if (c->black_holes.black_holes_in != NULL)
+ scheduler_activate(s, c->black_holes.black_holes_in);
+ if (c->black_holes.black_holes_out != NULL)
+ scheduler_activate(s, c->black_holes.black_holes_out);
+ if (c->kick1 != NULL) scheduler_activate(s, c->kick1);
+ if (c->kick2 != NULL) scheduler_activate(s, c->kick2);
+ if (c->timestep != NULL) scheduler_activate(s, c->timestep);
+ if (c->logger != NULL) scheduler_activate(s, c->logger);
+ }
+
+ return rebuild;
+}
+
/**
* @brief Set the super-cell pointers for all cells in a hierarchy.
*
@@ -3602,9 +3983,7 @@ void cell_set_super_mapper(void *map_data, int num_elements, void *extra_data) {
*/
int cell_has_tasks(struct cell *c) {
#ifdef WITH_MPI
- if (c->timestep != NULL || c->mpi.hydro.recv_ti != NULL ||
- c->mpi.grav.recv_ti != NULL || c->mpi.stars.recv_ti != NULL)
- return 1;
+ if (c->timestep != NULL || c->mpi.recv != NULL) return 1;
#else
if (c->timestep != NULL) return 1;
#endif
@@ -4335,7 +4714,8 @@ void cell_check_timesteps(struct cell *c) {
#ifdef SWIFT_DEBUG_CHECKS
if (c->hydro.ti_end_min == 0 && c->grav.ti_end_min == 0 &&
- c->stars.ti_end_min == 0 && c->nr_tasks > 0)
+ c->stars.ti_end_min == 0 && c->black_holes.ti_end_min == 0 &&
+ c->nr_tasks > 0)
error("Cell without assigned time-step");
if (c->split) {
diff --git a/src/engine_maketasks.c b/src/engine_maketasks.c
index dcc694b2f6cd2777e5c71469f784d12befe47ed6..8c928a763a0b09c34c4be0bacccd815b4df9065d 100644
--- a/src/engine_maketasks.c
+++ b/src/engine_maketasks.c
@@ -287,10 +287,11 @@ void engine_addtasks_send_stars(struct engine *e, struct cell *ci,
scheduler_addunlock(s, ci->top->hydro.star_formation, t_sf_counts);
}
- engine_addlink(e, &ci->mpi.send, t_feedback);
- engine_addlink(e, &ci->mpi.send, t_ti);
- if (with_star_formation) {
- engine_addlink(e, &ci->mpi.send, t_sf_counts);
+ engine_addlink(e, &ci->mpi.send, t_feedback);
+ engine_addlink(e, &ci->mpi.send, t_ti);
+ if (with_star_formation) {
+ engine_addlink(e, &ci->mpi.send, t_sf_counts);
+ }
}
}
@@ -298,7 +299,7 @@ void engine_addtasks_send_stars(struct engine *e, struct cell *ci,
if (ci->split)
for (int k = 0; k < 8; k++)
if (ci->progeny[k] != NULL)
- engine_addtasks_send_stars(e, ci->progeny[k], cj, t_feedback,
+ engine_addtasks_send_stars(e, ci->progeny[k], cj, t_feedback,
t_sf_counts, t_ti, with_star_formation);
#else
@@ -512,9 +513,9 @@ void engine_addtasks_recv_stars(struct engine *e, struct cell *c,
if (t_feedback != NULL) {
engine_addlink(e, &c->mpi.recv, t_feedback);
engine_addlink(e, &c->mpi.recv, t_ti);
- if (with_star_formation && c->hydro.count > 0) {
- engine_addlink(e, &c->mpi.recv, t_sf_counts);
- }
+ if (with_star_formation && c->hydro.count > 0) {
+ engine_addlink(e, &c->mpi.recv, t_sf_counts);
+ }
#ifdef SWIFT_DEBUG_CHECKS
if (c->nodeID == e->nodeID) error("Local cell!");
@@ -536,7 +537,7 @@ void engine_addtasks_recv_stars(struct engine *e, struct cell *c,
if (c->split)
for (int k = 0; k < 8; k++)
if (c->progeny[k] != NULL)
- engine_addtasks_recv_stars(e, c->progeny[k], t_feedback, t_sf_counts,
+ engine_addtasks_recv_stars(e, c->progeny[k], t_feedback, t_sf_counts,
t_ti, with_star_formation);
#else
@@ -2464,6 +2465,13 @@ void engine_addtasks_send_mapper(void *map_data, int num_elements,
/*t_sf_counts=*/NULL, /*t_ti=*/NULL,
with_star_formation);
+ /* Add the send tasks for the cells in the proxy that have a black holes
+ * connection. */
+ if ((e->policy & engine_policy_black_holes) &&
+ (type & proxy_cell_type_hydro))
+ engine_addtasks_send_black_holes(e, ci, cj, /*t_feedback=*/NULL,
+ /*t_ti=*/NULL);
+
/* Add the send tasks for the cells in the proxy that have a gravity
* connection. */
if ((e->policy & engine_policy_self_gravity) &&
@@ -2496,6 +2504,11 @@ void engine_addtasks_recv_mapper(void *map_data, int num_elements,
/*t_sf_counts=*/NULL, /*t_ti=*/NULL,
with_star_formation);
+ /* Add the recv tasks for the cells in the proxy that have a black holes
+ * connection. */
+ if ((e->policy & engine_policy_feedback) && (type & proxy_cell_type_hydro))
+ engine_addtasks_recv_black_holes(e, ci, NULL, NULL);
+
/* Add the recv tasks for the cells in the proxy that have a gravity
* connection. */
if ((e->policy & engine_policy_self_gravity) &&
diff --git a/src/space.c b/src/space.c
index e58b302d8171d6328a12762a6b7811c12e2ac8e9..9574bdbe57f0c07062dc892a7ae3630a77c975e0 100644
--- a/src/space.c
+++ b/src/space.c
@@ -4245,6 +4245,7 @@ void space_first_init_bparts_mapper(void *restrict map_data, int count,
struct bpart *restrict bp = (struct bpart *)map_data;
const struct space *restrict s = (struct space *)extra_data;
const struct engine *e = s->e;
+ const struct black_holes_props *props = e->black_holes_properties;
#ifdef SWIFT_DEBUG_CHECKS
const ptrdiff_t delta = bp - s->bparts;
@@ -4278,10 +4279,17 @@ void space_first_init_bparts_mapper(void *restrict map_data, int count,
#endif
}
+ /* Check that the smoothing lengths are non-zero */
+ for (int k = 0; k < count; k++) {
+ if (bp[k].h <= 0.)
+ error("Invalid value of smoothing length for bpart %lld h=%e", bp[k].id,
+ bp[k].h);
+ }
+
/* Initialise the rest */
for (int k = 0; k < count; k++) {
- black_holes_first_init_bpart(&bp[k]);
+ black_holes_first_init_bpart(&bp[k], props);
#ifdef SWIFT_DEBUG_CHECKS
if (bp[k].gpart && bp[k].gpart->id_or_neg_offset != -(k + delta))
diff --git a/src/task.c b/src/task.c
index 84a60036a8889d54bcc64c2e18fd435c2a04231a..1f45c2e17bc83cae3fd56fa3fe7b3f2cbcda0564 100644
--- a/src/task.c
+++ b/src/task.c
@@ -99,8 +99,8 @@ const char *subtaskID_names[task_subtype_count] = {
"limiter", "grav", "external_grav", "tend_part",
"tend_gpart", "tend_spart", "tend_bpart", "xv",
"rho", "gpart", "multipole", "spart",
- "stars_density", "stars_feedback", "sf_count", "bpart", "bh_density",
- "bh_feedback"};
+ "stars_density", "stars_feedback", "sf_count", "bpart",
+ "bh_density", "bh_feedback"};
#ifdef WITH_MPI
/* MPI communicators for the subtypes. */