Skip to content
GitLab
Commit fa13fb0e authored by Matthieu Schaller's avatar Matthieu Schaller
Browse files

Code now works on the simple BH merger test case over MPI as well.

parent 3ef53e6f
Hide whitespace changes
Inline Side-by-side
src/black_holes/EAGLE/black_holes_iact.h
View file @ fa13fb0e
......@@ -239,7 +239,8 @@ runner_iact_nonsym_bh_bh_swallow(const float r2, const float *dx,
(bj->merger_data.swallow_mass == bi->subgrid_mass &&
bj->merger_data.swallow_id < bi->id)) {
message("BH %lld wants to swallow BH particle %lld", bi->id, bj->id);
message("BH %lld wants to swallow BH particle %lld on node %d",
bi->id, bj->id, engine_rank);
bj->merger_data.swallow_id = bi->id;
bj->merger_data.swallow_mass = bi->subgrid_mass;
......
src/cell.c
View file @ fa13fb0e
......@@ -619,7 +619,13 @@ void cell_pack_bpart_swallow(const struct cell *c,
const struct bpart *bparts = c->black_holes.parts;
for (size_t i = 0; i < count; ++i) {
data[i] = bparts[i].merger_data;
message("Sending id=%lld swallowid=%lld",
bparts[i].id,
bparts[i].merger_data.swallow_id);
data[i] = bparts[i].merger_data;
}
}
......@@ -630,7 +636,13 @@ void cell_unpack_bpart_swallow(struct cell *c,
struct bpart *bparts = c->black_holes.parts;
for (size_t i = 0; i < count; ++i) {
bparts[i].merger_data = data[i];
message("Receiving! id=%lld swallow_id=%lld",
bparts[i].id,
data[i].swallow_id);
}
}
......@@ -4046,6 +4058,29 @@ int cell_unskip_black_holes_tasks(struct cell *c, struct scheduler *s) {
}
}
/* Un-skip the swallow tasks involved with this cell. */
for (struct link *l = c->black_holes.do_bh_swallow; l != NULL; l = l->next) {
struct task *t = l->t;
struct cell *ci = t->ci;
struct cell *cj = t->cj;
const int ci_active = cell_is_active_black_holes(ci, e);
const int cj_active = (cj != NULL) ? cell_is_active_black_holes(cj, e) : 0;
#ifdef WITH_MPI
const int ci_nodeID = ci->nodeID;
const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
#else
const int ci_nodeID = nodeID;
const int cj_nodeID = nodeID;
#endif
/* Only activate tasks that involve a local active cell. */
if ((ci_active || cj_active) &&
(ci_nodeID == nodeID || cj_nodeID == nodeID)) {
scheduler_activate(s, t);
}
}
/* Un-skip the feedback tasks involved with this cell. */
for (struct link *l = c->black_holes.feedback; l != NULL; l = l->next) {
struct task *t = l->t;
......
src/engine.c
View file @ fa13fb0e
......@@ -3336,6 +3336,7 @@ void engine_skip_force_and_kick(struct engine *e) {
t->subtype == task_subtype_do_bh_swallow ||
t->subtype == task_subtype_bpart_rho ||
t->subtype == task_subtype_part_swallow ||
t->subtype == task_subtype_bpart_merger ||
t->subtype == task_subtype_bpart_swallow ||
t->subtype == task_subtype_bpart_feedback ||
t->subtype == task_subtype_tend_part ||
......
src/engine_maketasks.c
View file @ fa13fb0e
......@@ -327,7 +327,7 @@ void engine_addtasks_send_stars(struct engine *e, struct cell *ci,
*/
void engine_addtasks_send_black_holes(struct engine *e, struct cell *ci,
struct cell *cj, struct task *t_rho,
struct task *t_swallow,
struct task *t_bh_merger,
struct task *t_gas_swallow,
struct task *t_feedback,
struct task *t_ti) {
......@@ -356,9 +356,9 @@ void engine_addtasks_send_black_holes(struct engine *e, struct cell *ci,
t_rho = scheduler_addtask(s, task_type_send, task_subtype_bpart_rho,
ci->mpi.tag, 0, ci, cj);
/* t_swallow = */
/* scheduler_addtask(s, task_type_send, task_subtype_bpart_swallow, */
/* ci->mpi.tag, 0, ci, cj); */
t_bh_merger =
scheduler_addtask(s, task_type_send, task_subtype_bpart_merger,
ci->mpi.tag, 0, ci, cj);
t_gas_swallow = scheduler_addtask(
s, task_type_send, task_subtype_part_swallow, ci->mpi.tag, 0, ci, cj);
......@@ -375,7 +375,7 @@ void engine_addtasks_send_black_holes(struct engine *e, struct cell *ci,
scheduler_addunlock(s, t_feedback,
ci->hydro.super->black_holes.black_holes_out);
scheduler_addunlock(s, ci->hydro.super->black_holes.swallow_ghost[1],
scheduler_addunlock(s, ci->hydro.super->black_holes.swallow_ghost[2],
t_feedback);
/* Ghost before you send */
......@@ -383,13 +383,11 @@ void engine_addtasks_send_black_holes(struct engine *e, struct cell *ci,
scheduler_addunlock(s, t_rho,
ci->hydro.super->black_holes.swallow_ghost[0]);
/* Drift before you send */
/* scheduler_addunlock(s, ci->hydro.super->black_holes.swallow_ghost[0],
*/
/* t_swallow); */
/* scheduler_addunlock(s, t_swallow, */
/* ci->hydro.super->black_holes.swallow_ghost[1]); */
scheduler_addunlock(s, ci->hydro.super->black_holes.swallow_ghost[0],
t_bh_merger);
scheduler_addunlock(s, t_bh_merger,
ci->hydro.super->black_holes.swallow_ghost[2]);
scheduler_addunlock(s, ci->hydro.super->black_holes.swallow_ghost[0],
t_gas_swallow);
scheduler_addunlock(s, t_gas_swallow,
......@@ -399,7 +397,7 @@ void engine_addtasks_send_black_holes(struct engine *e, struct cell *ci,
}
engine_addlink(e, &ci->mpi.send, t_rho);
// engine_addlink(e, &ci->mpi.send, t_swallow);
engine_addlink(e, &ci->mpi.send, t_bh_merger);
engine_addlink(e, &ci->mpi.send, t_gas_swallow);
engine_addlink(e, &ci->mpi.send, t_feedback);
engine_addlink(e, &ci->mpi.send, t_ti);
......@@ -410,7 +408,7 @@ void engine_addtasks_send_black_holes(struct engine *e, struct cell *ci,
for (int k = 0; k < 8; k++)
if (ci->progeny[k] != NULL)
engine_addtasks_send_black_holes(e, ci->progeny[k], cj, t_rho,
t_swallow, t_gas_swallow, t_feedback,
t_bh_merger, t_gas_swallow, t_feedback,
t_ti);
#else
......@@ -617,7 +615,7 @@ void engine_addtasks_recv_stars(struct engine *e, struct cell *c,
*/
void engine_addtasks_recv_black_holes(struct engine *e, struct cell *c,
struct task *t_rho,
struct task *t_swallow,
struct task *t_bh_merger,
struct task *t_gas_swallow,
struct task *t_feedback,
struct task *t_ti) {
......@@ -637,9 +635,9 @@ void engine_addtasks_recv_black_holes(struct engine *e, struct cell *c,
t_rho = scheduler_addtask(s, task_type_recv, task_subtype_bpart_rho,
c->mpi.tag, 0, c, NULL);
/* t_swallow = scheduler_addtask(s, task_type_recv,
* task_subtype_bpart_swallow, */
/* c->mpi.tag, 0, c, NULL); */
t_bh_merger = scheduler_addtask(s, task_type_recv,
task_subtype_bpart_merger,
c->mpi.tag, 0, c, NULL);
t_gas_swallow = scheduler_addtask(
s, task_type_recv, task_subtype_part_swallow, c->mpi.tag, 0, c, NULL);
......@@ -653,7 +651,7 @@ void engine_addtasks_recv_black_holes(struct engine *e, struct cell *c,
if (t_rho != NULL) {
engine_addlink(e, &c->mpi.recv, t_rho);
// engine_addlink(e, &c->mpi.recv, t_swallow);
engine_addlink(e, &c->mpi.recv, t_bh_merger);
engine_addlink(e, &c->mpi.recv, t_gas_swallow);
engine_addlink(e, &c->mpi.recv, t_feedback);
engine_addlink(e, &c->mpi.recv, t_ti);
......@@ -673,10 +671,15 @@ void engine_addtasks_recv_black_holes(struct engine *e, struct cell *c,
for (struct link *l = c->black_holes.swallow; l != NULL; l = l->next) {
scheduler_addunlock(s, t_rho, l->t);
scheduler_addunlock(s, l->t, t_gas_swallow);
scheduler_addunlock(s, l->t, t_bh_merger);
}
for (struct link *l = c->black_holes.do_gas_swallow; l != NULL;
l = l->next) {
scheduler_addunlock(s, t_gas_swallow, l->t);
}
for (struct link *l = c->black_holes.do_bh_swallow; l != NULL;
l = l->next) {
scheduler_addunlock(s, t_bh_merger, l->t);
scheduler_addunlock(s, l->t, t_feedback);
}
for (struct link *l = c->black_holes.feedback; l != NULL; l = l->next) {
......@@ -689,7 +692,7 @@ void engine_addtasks_recv_black_holes(struct engine *e, struct cell *c,
if (c->split)
for (int k = 0; k < 8; k++)
if (c->progeny[k] != NULL)
engine_addtasks_recv_black_holes(e, c->progeny[k], t_rho, t_swallow,
engine_addtasks_recv_black_holes(e, c->progeny[k], t_rho, t_bh_merger,
t_gas_swallow, t_feedback, t_ti);
#else
......
src/engine_marktasks.c
View file @ fa13fb0e
......@@ -417,6 +417,7 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
else if ((t_subtype == task_subtype_bh_density ||
t_subtype == task_subtype_bh_swallow ||
t_subtype == task_subtype_do_gas_swallow ||
t_subtype == task_subtype_do_bh_swallow ||
t_subtype == task_subtype_bh_feedback) &&
(ci_active_black_holes || cj_active_black_holes) &&
(ci_nodeID == nodeID || cj_nodeID == nodeID)) {
......@@ -686,6 +687,7 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
* swallowing */
scheduler_activate_recv(s, ci->mpi.recv, task_subtype_rho);
scheduler_activate_recv(s, ci->mpi.recv, task_subtype_part_swallow);
scheduler_activate_recv(s, ci->mpi.recv, task_subtype_bpart_merger);
/* Send the local BHs to tag the particles to swallow and to do
* feedback */
......@@ -713,6 +715,8 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
scheduler_activate_send(s, cj->mpi.send, task_subtype_rho, ci_nodeID);
scheduler_activate_send(s, cj->mpi.send, task_subtype_part_swallow,
ci_nodeID);
scheduler_activate_send(s, cj->mpi.send, task_subtype_bpart_merger,
ci_nodeID);
/* Drift the cell which will be sent; note that not all sent
particles will be drifted, only those that are needed. */
......@@ -724,6 +728,7 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
* swallowing */
scheduler_activate_recv(s, cj->mpi.recv, task_subtype_rho);
scheduler_activate_recv(s, cj->mpi.recv, task_subtype_part_swallow);
scheduler_activate_recv(s, cj->mpi.recv, task_subtype_bpart_merger);
/* Send the local BHs to tag the particles to swallow and to do
* feedback */
......@@ -751,6 +756,8 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
scheduler_activate_send(s, ci->mpi.send, task_subtype_rho, cj_nodeID);
scheduler_activate_send(s, ci->mpi.send, task_subtype_part_swallow,
cj_nodeID);
scheduler_activate_send(s, ci->mpi.send, task_subtype_bpart_merger,
cj_nodeID);
/* Drift the cell which will be sent; note that not all sent
particles will be drifted, only those that are needed. */
......
src/runner.c
View file @ fa13fb0e
......@@ -3970,8 +3970,8 @@ void runner_do_bh_swallow(struct runner *r, struct cell *c, int timer) {
struct bpart *cell_bparts = c->black_holes.parts;
/* Early abort?
* (We only want cells for which we drifted the gas as these are
* the only ones that could have gas particles that have been flagged
* (We only want cells for which we drifted the BH as these are
* the only ones that could have BH particles that have been flagged
* for swallowing) */
if (c->black_holes.count == 0 ||
c->black_holes.ti_old_part != e->ti_current) {
......@@ -3997,8 +3997,6 @@ void runner_do_bh_swallow(struct runner *r, struct cell *c, int timer) {
/* Get a handle on the part. */
struct bpart *const cell_bp = &cell_bparts[k];
message("OO");
/* Ignore inhibited particles (they have already been removed!) */
if (bpart_is_inhibited(cell_bp, e)) continue;
......@@ -4006,7 +4004,8 @@ void runner_do_bh_swallow(struct runner *r, struct cell *c, int timer) {
const long long swallow_id =
black_holes_get_bpart_swallow_id(&cell_bp->merger_data);
message("%lld", swallow_id);
/* message("OO id=%lld swallow_id = %lld", cell_bp->id, */
/* swallow_id); */
/* Has this particle been flagged for swallowing? */
if (swallow_id >= 0) {
......@@ -4431,6 +4430,12 @@ void runner_do_recv_bpart(struct runner *r, struct cell *c, int clear_sorts,
#ifdef DEBUG_INTERACTIONS_BLACK_HOLES
bparts[k].num_ngb_force = 0;
#endif
/* message("Receiving bparts id=%lld time_bin=%d", */
/* bparts[k].id, bparts[k].time_bin); */
if (bparts[k].time_bin == time_bin_inhibited) continue;
time_bin_min = min(time_bin_min, bparts[k].time_bin);
time_bin_max = max(time_bin_max, bparts[k].time_bin);
......@@ -4600,12 +4605,15 @@ void *runner_main(void *data) {
runner_dopair_branch_bh_density(r, ci, cj);
else if (t->subtype == task_subtype_bh_swallow)
runner_dopair_branch_bh_swallow(r, ci, cj);
else if (t->subtype == task_subtype_do_gas_swallow) {
else if (t->subtype == task_subtype_do_gas_swallow)
runner_do_gas_swallow_pair(r, ci, cj, 1);
} else if (t->subtype == task_subtype_bh_feedback)
else if (t->subtype == task_subtype_do_bh_swallow)
runner_do_bh_swallow_pair(r, ci, cj, 1);
else if (t->subtype == task_subtype_bh_feedback)
runner_dopair_branch_bh_feedback(r, ci, cj);
else
error("Unknown/invalid task subtype (%d).", t->subtype);
error("Unknown/invalid task subtype (%s/%s).",
taskID_names[t->type], subtaskID_names[t->subtype]);
break;
case task_type_sub_self:
......@@ -4629,10 +4637,13 @@ void *runner_main(void *data) {
runner_dosub_self_bh_swallow(r, ci, 1);
else if (t->subtype == task_subtype_do_gas_swallow)
runner_do_gas_swallow_self(r, ci, 1);
else if (t->subtype == task_subtype_do_bh_swallow)
runner_do_bh_swallow_self(r, ci, 1);
else if (t->subtype == task_subtype_bh_feedback)
runner_dosub_self_bh_feedback(r, ci, 1);
else
error("Unknown/invalid task subtype (%d).", t->subtype);
error("Unknown/invalid task subtype (%s/%s).",
taskID_names[t->type], subtaskID_names[t->subtype]);
break;
case task_type_sub_pair:
......@@ -4654,12 +4665,15 @@ void *runner_main(void *data) {
runner_dosub_pair_bh_density(r, ci, cj, 1);
else if (t->subtype == task_subtype_bh_swallow)
runner_dosub_pair_bh_swallow(r, ci, cj, 1);
else if (t->subtype == task_subtype_do_gas_swallow) {
else if (t->subtype == task_subtype_do_gas_swallow)
runner_do_gas_swallow_pair(r, ci, cj, 1);
} else if (t->subtype == task_subtype_bh_feedback)
else if (t->subtype == task_subtype_do_bh_swallow)
runner_do_bh_swallow_pair(r, ci, cj, 1);
else if (t->subtype == task_subtype_bh_feedback)
runner_dosub_pair_bh_feedback(r, ci, cj, 1);
else
error("Unknown/invalid task subtype (%d).", t->subtype);
error("Unknown/invalid task subtype (%s/%s).",
taskID_names[t->type], subtaskID_names[t->subtype]);
break;
case task_type_sort:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment