Commit fa13fb0e authored by Matthieu Schaller's avatar Matthieu Schaller
Browse files

Code now works on the simple BH merger test case over MPI as well.

parent 3ef53e6f
......@@ -239,7 +239,8 @@ runner_iact_nonsym_bh_bh_swallow(const float r2, const float *dx,
(bj->merger_data.swallow_mass == bi->subgrid_mass &&
bj->merger_data.swallow_id < bi->id)) {
message("BH %lld wants to swallow BH particle %lld", bi->id, bj->id);
message("BH %lld wants to swallow BH particle %lld on node %d",
bi->id, bj->id, engine_rank);
bj->merger_data.swallow_id = bi->id;
bj->merger_data.swallow_mass = bi->subgrid_mass;
......
......@@ -619,7 +619,13 @@ void cell_pack_bpart_swallow(const struct cell *c,
const struct bpart *bparts = c->black_holes.parts;
for (size_t i = 0; i < count; ++i) {
data[i] = bparts[i].merger_data;
message("Sending id=%lld swallowid=%lld",
bparts[i].id,
bparts[i].merger_data.swallow_id);
data[i] = bparts[i].merger_data;
}
}
......@@ -630,7 +636,13 @@ void cell_unpack_bpart_swallow(struct cell *c,
struct bpart *bparts = c->black_holes.parts;
for (size_t i = 0; i < count; ++i) {
bparts[i].merger_data = data[i];
message("Receiving! id=%lld swallow_id=%lld",
bparts[i].id,
data[i].swallow_id);
}
}
......@@ -4046,6 +4058,29 @@ int cell_unskip_black_holes_tasks(struct cell *c, struct scheduler *s) {
}
}
/* Un-skip the swallow tasks involved with this cell. */
for (struct link *l = c->black_holes.do_bh_swallow; l != NULL; l = l->next) {
struct task *t = l->t;
struct cell *ci = t->ci;
struct cell *cj = t->cj;
const int ci_active = cell_is_active_black_holes(ci, e);
const int cj_active = (cj != NULL) ? cell_is_active_black_holes(cj, e) : 0;
#ifdef WITH_MPI
const int ci_nodeID = ci->nodeID;
const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
#else
const int ci_nodeID = nodeID;
const int cj_nodeID = nodeID;
#endif
/* Only activate tasks that involve a local active cell. */
if ((ci_active || cj_active) &&
(ci_nodeID == nodeID || cj_nodeID == nodeID)) {
scheduler_activate(s, t);
}
}
/* Un-skip the feedback tasks involved with this cell. */
for (struct link *l = c->black_holes.feedback; l != NULL; l = l->next) {
struct task *t = l->t;
......
......@@ -3336,6 +3336,7 @@ void engine_skip_force_and_kick(struct engine *e) {
t->subtype == task_subtype_do_bh_swallow ||
t->subtype == task_subtype_bpart_rho ||
t->subtype == task_subtype_part_swallow ||
t->subtype == task_subtype_bpart_merger ||
t->subtype == task_subtype_bpart_swallow ||
t->subtype == task_subtype_bpart_feedback ||
t->subtype == task_subtype_tend_part ||
......
......@@ -327,7 +327,7 @@ void engine_addtasks_send_stars(struct engine *e, struct cell *ci,
*/
void engine_addtasks_send_black_holes(struct engine *e, struct cell *ci,
struct cell *cj, struct task *t_rho,
struct task *t_swallow,
struct task *t_bh_merger,
struct task *t_gas_swallow,
struct task *t_feedback,
struct task *t_ti) {
......@@ -356,9 +356,9 @@ void engine_addtasks_send_black_holes(struct engine *e, struct cell *ci,
t_rho = scheduler_addtask(s, task_type_send, task_subtype_bpart_rho,
ci->mpi.tag, 0, ci, cj);
/* t_swallow = */
/* scheduler_addtask(s, task_type_send, task_subtype_bpart_swallow, */
/* ci->mpi.tag, 0, ci, cj); */
t_bh_merger =
scheduler_addtask(s, task_type_send, task_subtype_bpart_merger,
ci->mpi.tag, 0, ci, cj);
t_gas_swallow = scheduler_addtask(
s, task_type_send, task_subtype_part_swallow, ci->mpi.tag, 0, ci, cj);
......@@ -375,7 +375,7 @@ void engine_addtasks_send_black_holes(struct engine *e, struct cell *ci,
scheduler_addunlock(s, t_feedback,
ci->hydro.super->black_holes.black_holes_out);
scheduler_addunlock(s, ci->hydro.super->black_holes.swallow_ghost[1],
scheduler_addunlock(s, ci->hydro.super->black_holes.swallow_ghost[2],
t_feedback);
/* Ghost before you send */
......@@ -383,13 +383,11 @@ void engine_addtasks_send_black_holes(struct engine *e, struct cell *ci,
scheduler_addunlock(s, t_rho,
ci->hydro.super->black_holes.swallow_ghost[0]);
/* Drift before you send */
/* scheduler_addunlock(s, ci->hydro.super->black_holes.swallow_ghost[0],
*/
/* t_swallow); */
/* scheduler_addunlock(s, t_swallow, */
/* ci->hydro.super->black_holes.swallow_ghost[1]); */
scheduler_addunlock(s, ci->hydro.super->black_holes.swallow_ghost[0],
t_bh_merger);
scheduler_addunlock(s, t_bh_merger,
ci->hydro.super->black_holes.swallow_ghost[2]);
scheduler_addunlock(s, ci->hydro.super->black_holes.swallow_ghost[0],
t_gas_swallow);
scheduler_addunlock(s, t_gas_swallow,
......@@ -399,7 +397,7 @@ void engine_addtasks_send_black_holes(struct engine *e, struct cell *ci,
}
engine_addlink(e, &ci->mpi.send, t_rho);
// engine_addlink(e, &ci->mpi.send, t_swallow);
engine_addlink(e, &ci->mpi.send, t_bh_merger);
engine_addlink(e, &ci->mpi.send, t_gas_swallow);
engine_addlink(e, &ci->mpi.send, t_feedback);
engine_addlink(e, &ci->mpi.send, t_ti);
......@@ -410,7 +408,7 @@ void engine_addtasks_send_black_holes(struct engine *e, struct cell *ci,
for (int k = 0; k < 8; k++)
if (ci->progeny[k] != NULL)
engine_addtasks_send_black_holes(e, ci->progeny[k], cj, t_rho,
t_swallow, t_gas_swallow, t_feedback,
t_bh_merger, t_gas_swallow, t_feedback,
t_ti);
#else
......@@ -617,7 +615,7 @@ void engine_addtasks_recv_stars(struct engine *e, struct cell *c,
*/
void engine_addtasks_recv_black_holes(struct engine *e, struct cell *c,
struct task *t_rho,
struct task *t_swallow,
struct task *t_bh_merger,
struct task *t_gas_swallow,
struct task *t_feedback,
struct task *t_ti) {
......@@ -637,9 +635,9 @@ void engine_addtasks_recv_black_holes(struct engine *e, struct cell *c,
t_rho = scheduler_addtask(s, task_type_recv, task_subtype_bpart_rho,
c->mpi.tag, 0, c, NULL);
/* t_swallow = scheduler_addtask(s, task_type_recv,
* task_subtype_bpart_swallow, */
/* c->mpi.tag, 0, c, NULL); */
t_bh_merger = scheduler_addtask(s, task_type_recv,
task_subtype_bpart_merger,
c->mpi.tag, 0, c, NULL);
t_gas_swallow = scheduler_addtask(
s, task_type_recv, task_subtype_part_swallow, c->mpi.tag, 0, c, NULL);
......@@ -653,7 +651,7 @@ void engine_addtasks_recv_black_holes(struct engine *e, struct cell *c,
if (t_rho != NULL) {
engine_addlink(e, &c->mpi.recv, t_rho);
// engine_addlink(e, &c->mpi.recv, t_swallow);
engine_addlink(e, &c->mpi.recv, t_bh_merger);
engine_addlink(e, &c->mpi.recv, t_gas_swallow);
engine_addlink(e, &c->mpi.recv, t_feedback);
engine_addlink(e, &c->mpi.recv, t_ti);
......@@ -673,10 +671,15 @@ void engine_addtasks_recv_black_holes(struct engine *e, struct cell *c,
for (struct link *l = c->black_holes.swallow; l != NULL; l = l->next) {
scheduler_addunlock(s, t_rho, l->t);
scheduler_addunlock(s, l->t, t_gas_swallow);
scheduler_addunlock(s, l->t, t_bh_merger);
}
for (struct link *l = c->black_holes.do_gas_swallow; l != NULL;
l = l->next) {
scheduler_addunlock(s, t_gas_swallow, l->t);
}
for (struct link *l = c->black_holes.do_bh_swallow; l != NULL;
l = l->next) {
scheduler_addunlock(s, t_bh_merger, l->t);
scheduler_addunlock(s, l->t, t_feedback);
}
for (struct link *l = c->black_holes.feedback; l != NULL; l = l->next) {
......@@ -689,7 +692,7 @@ void engine_addtasks_recv_black_holes(struct engine *e, struct cell *c,
if (c->split)
for (int k = 0; k < 8; k++)
if (c->progeny[k] != NULL)
engine_addtasks_recv_black_holes(e, c->progeny[k], t_rho, t_swallow,
engine_addtasks_recv_black_holes(e, c->progeny[k], t_rho, t_bh_merger,
t_gas_swallow, t_feedback, t_ti);
#else
......
......@@ -417,6 +417,7 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
else if ((t_subtype == task_subtype_bh_density ||
t_subtype == task_subtype_bh_swallow ||
t_subtype == task_subtype_do_gas_swallow ||
t_subtype == task_subtype_do_bh_swallow ||
t_subtype == task_subtype_bh_feedback) &&
(ci_active_black_holes || cj_active_black_holes) &&
(ci_nodeID == nodeID || cj_nodeID == nodeID)) {
......@@ -686,6 +687,7 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
* swallowing */
scheduler_activate_recv(s, ci->mpi.recv, task_subtype_rho);
scheduler_activate_recv(s, ci->mpi.recv, task_subtype_part_swallow);
scheduler_activate_recv(s, ci->mpi.recv, task_subtype_bpart_merger);
/* Send the local BHs to tag the particles to swallow and to do
* feedback */
......@@ -713,6 +715,8 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
scheduler_activate_send(s, cj->mpi.send, task_subtype_rho, ci_nodeID);
scheduler_activate_send(s, cj->mpi.send, task_subtype_part_swallow,
ci_nodeID);
scheduler_activate_send(s, cj->mpi.send, task_subtype_bpart_merger,
ci_nodeID);
/* Drift the cell which will be sent; note that not all sent
particles will be drifted, only those that are needed. */
......@@ -724,6 +728,7 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
* swallowing */
scheduler_activate_recv(s, cj->mpi.recv, task_subtype_rho);
scheduler_activate_recv(s, cj->mpi.recv, task_subtype_part_swallow);
scheduler_activate_recv(s, cj->mpi.recv, task_subtype_bpart_merger);
/* Send the local BHs to tag the particles to swallow and to do
* feedback */
......@@ -751,6 +756,8 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
scheduler_activate_send(s, ci->mpi.send, task_subtype_rho, cj_nodeID);
scheduler_activate_send(s, ci->mpi.send, task_subtype_part_swallow,
cj_nodeID);
scheduler_activate_send(s, ci->mpi.send, task_subtype_bpart_merger,
cj_nodeID);
/* Drift the cell which will be sent; note that not all sent
particles will be drifted, only those that are needed. */
......
......@@ -3970,8 +3970,8 @@ void runner_do_bh_swallow(struct runner *r, struct cell *c, int timer) {
struct bpart *cell_bparts = c->black_holes.parts;
/* Early abort?
* (We only want cells for which we drifted the gas as these are
* the only ones that could have gas particles that have been flagged
* (We only want cells for which we drifted the BH as these are
* the only ones that could have BH particles that have been flagged
* for swallowing) */
if (c->black_holes.count == 0 ||
c->black_holes.ti_old_part != e->ti_current) {
......@@ -3997,8 +3997,6 @@ void runner_do_bh_swallow(struct runner *r, struct cell *c, int timer) {
/* Get a handle on the part. */
struct bpart *const cell_bp = &cell_bparts[k];
message("OO");
/* Ignore inhibited particles (they have already been removed!) */
if (bpart_is_inhibited(cell_bp, e)) continue;
......@@ -4006,7 +4004,8 @@ void runner_do_bh_swallow(struct runner *r, struct cell *c, int timer) {
const long long swallow_id =
black_holes_get_bpart_swallow_id(&cell_bp->merger_data);
message("%lld", swallow_id);
/* message("OO id=%lld swallow_id = %lld", cell_bp->id, */
/* swallow_id); */
/* Has this particle been flagged for swallowing? */
if (swallow_id >= 0) {
......@@ -4431,6 +4430,12 @@ void runner_do_recv_bpart(struct runner *r, struct cell *c, int clear_sorts,
#ifdef DEBUG_INTERACTIONS_BLACK_HOLES
bparts[k].num_ngb_force = 0;
#endif
/* message("Receiving bparts id=%lld time_bin=%d", */
/* bparts[k].id, bparts[k].time_bin); */
if (bparts[k].time_bin == time_bin_inhibited) continue;
time_bin_min = min(time_bin_min, bparts[k].time_bin);
time_bin_max = max(time_bin_max, bparts[k].time_bin);
......@@ -4600,12 +4605,15 @@ void *runner_main(void *data) {
runner_dopair_branch_bh_density(r, ci, cj);
else if (t->subtype == task_subtype_bh_swallow)
runner_dopair_branch_bh_swallow(r, ci, cj);
else if (t->subtype == task_subtype_do_gas_swallow) {
else if (t->subtype == task_subtype_do_gas_swallow)
runner_do_gas_swallow_pair(r, ci, cj, 1);
} else if (t->subtype == task_subtype_bh_feedback)
else if (t->subtype == task_subtype_do_bh_swallow)
runner_do_bh_swallow_pair(r, ci, cj, 1);
else if (t->subtype == task_subtype_bh_feedback)
runner_dopair_branch_bh_feedback(r, ci, cj);
else
error("Unknown/invalid task subtype (%d).", t->subtype);
error("Unknown/invalid task subtype (%s/%s).",
taskID_names[t->type], subtaskID_names[t->subtype]);
break;
case task_type_sub_self:
......@@ -4629,10 +4637,13 @@ void *runner_main(void *data) {
runner_dosub_self_bh_swallow(r, ci, 1);
else if (t->subtype == task_subtype_do_gas_swallow)
runner_do_gas_swallow_self(r, ci, 1);
else if (t->subtype == task_subtype_do_bh_swallow)
runner_do_bh_swallow_self(r, ci, 1);
else if (t->subtype == task_subtype_bh_feedback)
runner_dosub_self_bh_feedback(r, ci, 1);
else
error("Unknown/invalid task subtype (%d).", t->subtype);
error("Unknown/invalid task subtype (%s/%s).",
taskID_names[t->type], subtaskID_names[t->subtype]);
break;
case task_type_sub_pair:
......@@ -4654,12 +4665,15 @@ void *runner_main(void *data) {
runner_dosub_pair_bh_density(r, ci, cj, 1);
else if (t->subtype == task_subtype_bh_swallow)
runner_dosub_pair_bh_swallow(r, ci, cj, 1);
else if (t->subtype == task_subtype_do_gas_swallow) {
else if (t->subtype == task_subtype_do_gas_swallow)
runner_do_gas_swallow_pair(r, ci, cj, 1);
} else if (t->subtype == task_subtype_bh_feedback)
else if (t->subtype == task_subtype_do_bh_swallow)
runner_do_bh_swallow_pair(r, ci, cj, 1);
else if (t->subtype == task_subtype_bh_feedback)
runner_dosub_pair_bh_feedback(r, ci, cj, 1);
else
error("Unknown/invalid task subtype (%d).", t->subtype);
error("Unknown/invalid task subtype (%s/%s).",
taskID_names[t->type], subtaskID_names[t->subtype]);
break;
case task_type_sort:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment