diff --git a/src/black_holes/EAGLE/black_holes_iact.h b/src/black_holes/EAGLE/black_holes_iact.h
index bd3d8f0ce20a25f09c432c61a311593ba6656799..2546fd2b7b218b87defa0603da1407050a00e9cf 100644
--- a/src/black_holes/EAGLE/black_holes_iact.h
+++ b/src/black_holes/EAGLE/black_holes_iact.h
@@ -239,7 +239,8 @@ runner_iact_nonsym_bh_bh_swallow(const float r2, const float *dx,
(bj->merger_data.swallow_mass == bi->subgrid_mass &&
bj->merger_data.swallow_id < bi->id)) {
- message("BH %lld wants to swallow BH particle %lld", bi->id, bj->id);
+ message("BH %lld wants to swallow BH particle %lld on node %d",
+ bi->id, bj->id, engine_rank);
bj->merger_data.swallow_id = bi->id;
bj->merger_data.swallow_mass = bi->subgrid_mass;
diff --git a/src/cell.c b/src/cell.c
index b9998653ce96937cc628739b670b63b17e4a7532..2849f00ea9d8bc5cc4aee06930b00a1587fb3dda 100644
--- a/src/cell.c
+++ b/src/cell.c
@@ -619,7 +619,13 @@ void cell_pack_bpart_swallow(const struct cell *c,
const struct bpart *bparts = c->black_holes.parts;
for (size_t i = 0; i < count; ++i) {
- data[i] = bparts[i].merger_data;
+
+ message("Sending id=%lld swallowid=%lld",
+ bparts[i].id,
+ bparts[i].merger_data.swallow_id);
+
+
+ data[i] = bparts[i].merger_data;
}
}
@@ -630,7 +636,13 @@ void cell_unpack_bpart_swallow(struct cell *c,
struct bpart *bparts = c->black_holes.parts;
for (size_t i = 0; i < count; ++i) {
+
bparts[i].merger_data = data[i];
+
+ message("Receiving! id=%lld swallow_id=%lld",
+ bparts[i].id,
+ data[i].swallow_id);
+
}
}
@@ -4046,6 +4058,29 @@ int cell_unskip_black_holes_tasks(struct cell *c, struct scheduler *s) {
}
}
+ /* Un-skip the swallow tasks involved with this cell. */
+ for (struct link *l = c->black_holes.do_bh_swallow; l != NULL; l = l->next) {
+ struct task *t = l->t;
+ struct cell *ci = t->ci;
+ struct cell *cj = t->cj;
+ const int ci_active = cell_is_active_black_holes(ci, e);
+ const int cj_active = (cj != NULL) ? cell_is_active_black_holes(cj, e) : 0;
+#ifdef WITH_MPI
+ const int ci_nodeID = ci->nodeID;
+ const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
+#else
+ const int ci_nodeID = nodeID;
+ const int cj_nodeID = nodeID;
+#endif
+
+ /* Only activate tasks that involve a local active cell. */
+ if ((ci_active || cj_active) &&
+ (ci_nodeID == nodeID || cj_nodeID == nodeID)) {
+
+ scheduler_activate(s, t);
+ }
+ }
+
/* Un-skip the feedback tasks involved with this cell. */
for (struct link *l = c->black_holes.feedback; l != NULL; l = l->next) {
struct task *t = l->t;
diff --git a/src/engine.c b/src/engine.c
index eeeca1992b7fce06b55952d7b2602210e911d67e..05f8f7e6f3c20003838d0abb5713492d158a86cb 100644
--- a/src/engine.c
+++ b/src/engine.c
@@ -3336,6 +3336,7 @@ void engine_skip_force_and_kick(struct engine *e) {
t->subtype == task_subtype_do_bh_swallow ||
t->subtype == task_subtype_bpart_rho ||
t->subtype == task_subtype_part_swallow ||
+ t->subtype == task_subtype_bpart_merger ||
t->subtype == task_subtype_bpart_swallow ||
t->subtype == task_subtype_bpart_feedback ||
t->subtype == task_subtype_tend_part ||
diff --git a/src/engine_maketasks.c b/src/engine_maketasks.c
index 128dcb7e6cb6257086bf1b630a157c311ae73677..223379368d35877d69c5eae6c8e9b37aceaa167f 100644
--- a/src/engine_maketasks.c
+++ b/src/engine_maketasks.c
@@ -327,7 +327,7 @@ void engine_addtasks_send_stars(struct engine *e, struct cell *ci,
*/
void engine_addtasks_send_black_holes(struct engine *e, struct cell *ci,
struct cell *cj, struct task *t_rho,
- struct task *t_swallow,
+ struct task *t_bh_merger,
struct task *t_gas_swallow,
struct task *t_feedback,
struct task *t_ti) {
@@ -356,9 +356,9 @@ void engine_addtasks_send_black_holes(struct engine *e, struct cell *ci,
t_rho = scheduler_addtask(s, task_type_send, task_subtype_bpart_rho,
ci->mpi.tag, 0, ci, cj);
- /* t_swallow = */
- /* scheduler_addtask(s, task_type_send, task_subtype_bpart_swallow, */
- /* ci->mpi.tag, 0, ci, cj); */
+ t_bh_merger =
+ scheduler_addtask(s, task_type_send, task_subtype_bpart_merger,
+ ci->mpi.tag, 0, ci, cj);
t_gas_swallow = scheduler_addtask(
s, task_type_send, task_subtype_part_swallow, ci->mpi.tag, 0, ci, cj);
@@ -375,7 +375,7 @@ void engine_addtasks_send_black_holes(struct engine *e, struct cell *ci,
scheduler_addunlock(s, t_feedback,
ci->hydro.super->black_holes.black_holes_out);
- scheduler_addunlock(s, ci->hydro.super->black_holes.swallow_ghost[1],
+ scheduler_addunlock(s, ci->hydro.super->black_holes.swallow_ghost[2],
t_feedback);
/* Ghost before you send */
@@ -383,13 +383,11 @@ void engine_addtasks_send_black_holes(struct engine *e, struct cell *ci,
scheduler_addunlock(s, t_rho,
ci->hydro.super->black_holes.swallow_ghost[0]);
- /* Drift before you send */
- /* scheduler_addunlock(s, ci->hydro.super->black_holes.swallow_ghost[0],
- */
- /* t_swallow); */
- /* scheduler_addunlock(s, t_swallow, */
- /* ci->hydro.super->black_holes.swallow_ghost[1]); */
-
+ scheduler_addunlock(s, ci->hydro.super->black_holes.swallow_ghost[0],
+ t_bh_merger);
+ scheduler_addunlock(s, t_bh_merger,
+ ci->hydro.super->black_holes.swallow_ghost[2]);
+
scheduler_addunlock(s, ci->hydro.super->black_holes.swallow_ghost[0],
t_gas_swallow);
scheduler_addunlock(s, t_gas_swallow,
@@ -399,7 +397,7 @@ void engine_addtasks_send_black_holes(struct engine *e, struct cell *ci,
}
engine_addlink(e, &ci->mpi.send, t_rho);
- // engine_addlink(e, &ci->mpi.send, t_swallow);
+ engine_addlink(e, &ci->mpi.send, t_bh_merger);
engine_addlink(e, &ci->mpi.send, t_gas_swallow);
engine_addlink(e, &ci->mpi.send, t_feedback);
engine_addlink(e, &ci->mpi.send, t_ti);
@@ -410,7 +408,7 @@ void engine_addtasks_send_black_holes(struct engine *e, struct cell *ci,
for (int k = 0; k < 8; k++)
if (ci->progeny[k] != NULL)
engine_addtasks_send_black_holes(e, ci->progeny[k], cj, t_rho,
- t_swallow, t_gas_swallow, t_feedback,
+ t_bh_merger, t_gas_swallow, t_feedback,
t_ti);
#else
@@ -617,7 +615,7 @@ void engine_addtasks_recv_stars(struct engine *e, struct cell *c,
*/
void engine_addtasks_recv_black_holes(struct engine *e, struct cell *c,
struct task *t_rho,
- struct task *t_swallow,
+ struct task *t_bh_merger,
struct task *t_gas_swallow,
struct task *t_feedback,
struct task *t_ti) {
@@ -637,9 +635,9 @@ void engine_addtasks_recv_black_holes(struct engine *e, struct cell *c,
t_rho = scheduler_addtask(s, task_type_recv, task_subtype_bpart_rho,
c->mpi.tag, 0, c, NULL);
- /* t_swallow = scheduler_addtask(s, task_type_recv,
- * task_subtype_bpart_swallow, */
- /* c->mpi.tag, 0, c, NULL); */
+ t_bh_merger = scheduler_addtask(s, task_type_recv,
+ task_subtype_bpart_merger,
+ c->mpi.tag, 0, c, NULL);
t_gas_swallow = scheduler_addtask(
s, task_type_recv, task_subtype_part_swallow, c->mpi.tag, 0, c, NULL);
@@ -653,7 +651,7 @@ void engine_addtasks_recv_black_holes(struct engine *e, struct cell *c,
if (t_rho != NULL) {
engine_addlink(e, &c->mpi.recv, t_rho);
- // engine_addlink(e, &c->mpi.recv, t_swallow);
+ engine_addlink(e, &c->mpi.recv, t_bh_merger);
engine_addlink(e, &c->mpi.recv, t_gas_swallow);
engine_addlink(e, &c->mpi.recv, t_feedback);
engine_addlink(e, &c->mpi.recv, t_ti);
@@ -673,10 +671,15 @@ void engine_addtasks_recv_black_holes(struct engine *e, struct cell *c,
for (struct link *l = c->black_holes.swallow; l != NULL; l = l->next) {
scheduler_addunlock(s, t_rho, l->t);
scheduler_addunlock(s, l->t, t_gas_swallow);
+ scheduler_addunlock(s, l->t, t_bh_merger);
}
for (struct link *l = c->black_holes.do_gas_swallow; l != NULL;
l = l->next) {
scheduler_addunlock(s, t_gas_swallow, l->t);
+ }
+ for (struct link *l = c->black_holes.do_bh_swallow; l != NULL;
+ l = l->next) {
+ scheduler_addunlock(s, t_bh_merger, l->t);
scheduler_addunlock(s, l->t, t_feedback);
}
for (struct link *l = c->black_holes.feedback; l != NULL; l = l->next) {
@@ -689,7 +692,7 @@ void engine_addtasks_recv_black_holes(struct engine *e, struct cell *c,
if (c->split)
for (int k = 0; k < 8; k++)
if (c->progeny[k] != NULL)
- engine_addtasks_recv_black_holes(e, c->progeny[k], t_rho, t_swallow,
+ engine_addtasks_recv_black_holes(e, c->progeny[k], t_rho, t_bh_merger,
t_gas_swallow, t_feedback, t_ti);
#else
diff --git a/src/engine_marktasks.c b/src/engine_marktasks.c
index a776684e8651e4bbbdd5f5ea965631d131636769..7f6d1cb827b102c34a2d63f6f30779371bf58a21 100644
--- a/src/engine_marktasks.c
+++ b/src/engine_marktasks.c
@@ -417,6 +417,7 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
else if ((t_subtype == task_subtype_bh_density ||
t_subtype == task_subtype_bh_swallow ||
t_subtype == task_subtype_do_gas_swallow ||
+ t_subtype == task_subtype_do_bh_swallow ||
t_subtype == task_subtype_bh_feedback) &&
(ci_active_black_holes || cj_active_black_holes) &&
(ci_nodeID == nodeID || cj_nodeID == nodeID)) {
@@ -686,6 +687,7 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
* swallowing */
scheduler_activate_recv(s, ci->mpi.recv, task_subtype_rho);
scheduler_activate_recv(s, ci->mpi.recv, task_subtype_part_swallow);
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_bpart_merger);
/* Send the local BHs to tag the particles to swallow and to do
* feedback */
@@ -713,6 +715,8 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
scheduler_activate_send(s, cj->mpi.send, task_subtype_rho, ci_nodeID);
scheduler_activate_send(s, cj->mpi.send, task_subtype_part_swallow,
ci_nodeID);
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_bpart_merger,
+ ci_nodeID);
/* Drift the cell which will be sent; note that not all sent
particles will be drifted, only those that are needed. */
@@ -724,6 +728,7 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
* swallowing */
scheduler_activate_recv(s, cj->mpi.recv, task_subtype_rho);
scheduler_activate_recv(s, cj->mpi.recv, task_subtype_part_swallow);
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_bpart_merger);
/* Send the local BHs to tag the particles to swallow and to do
* feedback */
@@ -751,6 +756,8 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
scheduler_activate_send(s, ci->mpi.send, task_subtype_rho, cj_nodeID);
scheduler_activate_send(s, ci->mpi.send, task_subtype_part_swallow,
cj_nodeID);
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_bpart_merger,
+ cj_nodeID);
/* Drift the cell which will be sent; note that not all sent
particles will be drifted, only those that are needed. */
diff --git a/src/runner.c b/src/runner.c
index b22c9a644de1a569666d1a73b0970f43c44f114f..f25b5348354e142317d799ab01e6aed3fc18946a 100644
--- a/src/runner.c
+++ b/src/runner.c
@@ -3970,8 +3970,8 @@ void runner_do_bh_swallow(struct runner *r, struct cell *c, int timer) {
struct bpart *cell_bparts = c->black_holes.parts;
/* Early abort?
- * (We only want cells for which we drifted the gas as these are
- * the only ones that could have gas particles that have been flagged
+ * (We only want cells for which we drifted the BH as these are
+ * the only ones that could have BH particles that have been flagged
* for swallowing) */
if (c->black_holes.count == 0 ||
c->black_holes.ti_old_part != e->ti_current) {
@@ -3997,8 +3997,6 @@ void runner_do_bh_swallow(struct runner *r, struct cell *c, int timer) {
/* Get a handle on the part. */
struct bpart *const cell_bp = &cell_bparts[k];
- message("OO");
-
/* Ignore inhibited particles (they have already been removed!) */
if (bpart_is_inhibited(cell_bp, e)) continue;
@@ -4006,7 +4004,8 @@ void runner_do_bh_swallow(struct runner *r, struct cell *c, int timer) {
const long long swallow_id =
black_holes_get_bpart_swallow_id(&cell_bp->merger_data);
- message("%lld", swallow_id);
+ /* message("OO id=%lld swallow_id = %lld", cell_bp->id, */
+ /* swallow_id); */
/* Has this particle been flagged for swallowing? */
if (swallow_id >= 0) {
@@ -4431,6 +4430,12 @@ void runner_do_recv_bpart(struct runner *r, struct cell *c, int clear_sorts,
#ifdef DEBUG_INTERACTIONS_BLACK_HOLES
bparts[k].num_ngb_force = 0;
#endif
+
+ /* message("Receiving bparts id=%lld time_bin=%d", */
+ /* bparts[k].id, bparts[k].time_bin); */
+
+
+
if (bparts[k].time_bin == time_bin_inhibited) continue;
time_bin_min = min(time_bin_min, bparts[k].time_bin);
time_bin_max = max(time_bin_max, bparts[k].time_bin);
@@ -4600,12 +4605,15 @@ void *runner_main(void *data) {
runner_dopair_branch_bh_density(r, ci, cj);
else if (t->subtype == task_subtype_bh_swallow)
runner_dopair_branch_bh_swallow(r, ci, cj);
- else if (t->subtype == task_subtype_do_gas_swallow) {
+ else if (t->subtype == task_subtype_do_gas_swallow)
runner_do_gas_swallow_pair(r, ci, cj, 1);
- } else if (t->subtype == task_subtype_bh_feedback)
+ else if (t->subtype == task_subtype_do_bh_swallow)
+ runner_do_bh_swallow_pair(r, ci, cj, 1);
+ else if (t->subtype == task_subtype_bh_feedback)
runner_dopair_branch_bh_feedback(r, ci, cj);
else
- error("Unknown/invalid task subtype (%d).", t->subtype);
+ error("Unknown/invalid task subtype (%s/%s).",
+ taskID_names[t->type], subtaskID_names[t->subtype]);
break;
case task_type_sub_self:
@@ -4629,10 +4637,13 @@ void *runner_main(void *data) {
runner_dosub_self_bh_swallow(r, ci, 1);
else if (t->subtype == task_subtype_do_gas_swallow)
runner_do_gas_swallow_self(r, ci, 1);
+ else if (t->subtype == task_subtype_do_bh_swallow)
+ runner_do_bh_swallow_self(r, ci, 1);
else if (t->subtype == task_subtype_bh_feedback)
runner_dosub_self_bh_feedback(r, ci, 1);
else
- error("Unknown/invalid task subtype (%d).", t->subtype);
+ error("Unknown/invalid task subtype (%s/%s).",
+ taskID_names[t->type], subtaskID_names[t->subtype]);
break;
case task_type_sub_pair:
@@ -4654,12 +4665,15 @@ void *runner_main(void *data) {
runner_dosub_pair_bh_density(r, ci, cj, 1);
else if (t->subtype == task_subtype_bh_swallow)
runner_dosub_pair_bh_swallow(r, ci, cj, 1);
- else if (t->subtype == task_subtype_do_gas_swallow) {
+ else if (t->subtype == task_subtype_do_gas_swallow)
runner_do_gas_swallow_pair(r, ci, cj, 1);
- } else if (t->subtype == task_subtype_bh_feedback)
+ else if (t->subtype == task_subtype_do_bh_swallow)
+ runner_do_bh_swallow_pair(r, ci, cj, 1);
+ else if (t->subtype == task_subtype_bh_feedback)
runner_dosub_pair_bh_feedback(r, ci, cj, 1);
else
- error("Unknown/invalid task subtype (%d).", t->subtype);
+ error("Unknown/invalid task subtype (%s/%s).",
+ taskID_names[t->type], subtaskID_names[t->subtype]);
break;
case task_type_sort: