Commit c88baee1 authored by Matthieu Schaller's avatar Matthieu Schaller
Browse files

Split engine and runner source files

parent fe5ed4e0
...@@ -44,7 +44,7 @@ include_HEADERS = space.h runner.h queue.h task.h lock.h cell.h part.h const.h \ ...@@ -44,7 +44,7 @@ include_HEADERS = space.h runner.h queue.h task.h lock.h cell.h part.h const.h \
common_io.h single_io.h multipole.h map.h tools.h partition.h partition_fixed_costs.h \ common_io.h single_io.h multipole.h map.h tools.h partition.h partition_fixed_costs.h \
clocks.h parser.h physical_constants.h physical_constants_cgs.h potential.h version.h \ clocks.h parser.h physical_constants.h physical_constants_cgs.h potential.h version.h \
hydro_properties.h riemann.h threadpool.h cooling_io.h cooling.h cooling_struct.h \ hydro_properties.h riemann.h threadpool.h cooling_io.h cooling.h cooling_struct.h \
statistics.h memswap.h cache.h runner_doiact_vec.h profiler.h entropy_floor.h \ statistics.h memswap.h cache.h runner_doiact_hydro_vec.h profiler.h entropy_floor.h \
dump.h logger.h active.h timeline.h xmf.h gravity_properties.h gravity_derivatives.h \ dump.h logger.h active.h timeline.h xmf.h gravity_properties.h gravity_derivatives.h \
gravity_softened_derivatives.h vector_power.h collectgroup.h hydro_space.h sort_part.h \ gravity_softened_derivatives.h vector_power.h collectgroup.h hydro_space.h sort_part.h \
chemistry.h chemistry_io.h chemistry_struct.h cosmology.h restart.h space_getsid.h utilities.h \ chemistry.h chemistry_io.h chemistry_struct.h cosmology.h restart.h space_getsid.h utilities.h \
...@@ -69,13 +69,18 @@ EAGLE_FEEDBACK_SOURCES += feedback/EAGLE/feedback.c ...@@ -69,13 +69,18 @@ EAGLE_FEEDBACK_SOURCES += feedback/EAGLE/feedback.c
endif endif
# Common source files # Common source files
AM_SOURCES = space.c runner.c queue.c task.c cell.c engine.c engine_maketasks.c \ AM_SOURCES = space.c runner_main.c runner_doiact_hydro.c runner_doiact_grav.c \
engine_marktasks.c engine_drift.c engine_unskip.c serial_io.c timers.c debug.c scheduler.c \ runner_doiact_stars.c runner_doiact_black_holes.c runner_ghost.c runner_recv.c \
runner_sort.c runner_drift.c runner_black_holes.c runner_time_integration.c \
runner_doiact_hydro_vec.c runner_others.c\
queue.c task.c cell.c engine.c engine_maketasks.c \
engine_marktasks.c engine_drift.c engine_unskip.c engine_collect_end_of_step.c \
engine_redistribute.c engine_fof.c serial_io.c timers.c debug.c scheduler.c \
proxy.c parallel_io.c units.c common_io.c single_io.c multipole.c version.c map.c \ proxy.c parallel_io.c units.c common_io.c single_io.c multipole.c version.c map.c \
kernel_hydro.c tools.c part.c partition.c clocks.c parser.c \ kernel_hydro.c tools.c part.c partition.c clocks.c parser.c \
physical_constants.c potential.c hydro_properties.c \ physical_constants.c potential.c hydro_properties.c \
threadpool.c cooling.c star_formation.c \ threadpool.c cooling.c star_formation.c \
statistics.c runner_doiact_vec.c profiler.c dump.c logger.c \ statistics.c profiler.c dump.c logger.c \
part_type.c xmf.c gravity_properties.c gravity.c \ part_type.c xmf.c gravity_properties.c gravity.c \
collectgroup.c hydro_space.c equation_of_state.c \ collectgroup.c hydro_space.c equation_of_state.c \
chemistry.c cosmology.c restart.c mesh_gravity.c velociraptor_interface.c \ chemistry.c cosmology.c restart.c mesh_gravity.c velociraptor_interface.c \
...@@ -85,8 +90,10 @@ AM_SOURCES = space.c runner.c queue.c task.c cell.c engine.c engine_maketasks.c ...@@ -85,8 +90,10 @@ AM_SOURCES = space.c runner.c queue.c task.c cell.c engine.c engine_maketasks.c
# Include files for distribution, not installation. # Include files for distribution, not installation.
nobase_noinst_HEADERS = align.h approx_math.h atomic.h barrier.h cycle.h error.h inline.h kernel_hydro.h kernel_gravity.h \ nobase_noinst_HEADERS = align.h approx_math.h atomic.h barrier.h cycle.h error.h inline.h kernel_hydro.h kernel_gravity.h \
gravity_iact.h kernel_long_gravity.h vector.h cache.h runner_doiact.h runner_doiact_vec.h runner_doiact_grav.h \ gravity_iact.h kernel_long_gravity.h vector.h cache.h \
runner_doiact_nosort.h runner_doiact_stars.h runner_doiact_black_holes.h units.h intrinsics.h minmax.h \ runner_doiact_nosort.h runner_doiact_hydro.h runner_doiact_stars.h runner_doiact_black_holes.h runner_doiact_grav.h \
runner_doiact_functions_hydro.h runner_doiact_functions_stars.h runner_doiact_functions_black_holes.h \
units.h intrinsics.h minmax.h \
kick.h timestep.h drift.h adiabatic_index.h io_properties.h dimension.h part_type.h periodic.h memswap.h \ kick.h timestep.h drift.h adiabatic_index.h io_properties.h dimension.h part_type.h periodic.h memswap.h \
dump.h logger.h sign.h logger_io.h timestep_limiter.h hashmap.h \ dump.h logger.h sign.h logger_io.h timestep_limiter.h hashmap.h \
gravity.h gravity_io.h gravity_cache.h \ gravity.h gravity_io.h gravity_cache.h \
......
This diff is collapsed.
...@@ -490,6 +490,7 @@ void engine_reconstruct_multipoles(struct engine *e); ...@@ -490,6 +490,7 @@ void engine_reconstruct_multipoles(struct engine *e);
void engine_allocate_foreign_particles(struct engine *e); void engine_allocate_foreign_particles(struct engine *e);
void engine_print_stats(struct engine *e); void engine_print_stats(struct engine *e);
void engine_check_for_dumps(struct engine *e); void engine_check_for_dumps(struct engine *e);
void engine_collect_end_of_step(struct engine *e, int apply);
void engine_dump_snapshot(struct engine *e); void engine_dump_snapshot(struct engine *e);
void engine_init_output_lists(struct engine *e, struct swift_params *params); void engine_init_output_lists(struct engine *e, struct swift_params *params);
void engine_init(struct engine *e, struct space *s, struct swift_params *params, void engine_init(struct engine *e, struct space *s, struct swift_params *params,
......
This diff is collapsed.
/*******************************************************************************
* This file is part of SWIFT.
* Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
* Matthieu Schaller (matthieu.schaller@durham.ac.uk)
* 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
******************************************************************************/
/* Config parameters. */
#include "../config.h"
/* This object's header. */
#include "engine.h"
/**
* @brief Activate all the #gpart communications in preparation
* fof a call to FOF.
*
* @param e The #engine to act on.
*/
void engine_activate_gpart_comms(struct engine *e) {
#ifdef WITH_MPI
const ticks tic = getticks();
struct scheduler *s = &e->sched;
const int nr_tasks = s->nr_tasks;
struct task *tasks = s->tasks;
for (int k = 0; k < nr_tasks; ++k) {
struct task *t = &tasks[k];
if ((t->type == task_type_send) && (t->subtype == task_subtype_gpart)) {
scheduler_activate(s, t);
} else if ((t->type == task_type_recv) &&
(t->subtype == task_subtype_gpart)) {
scheduler_activate(s, t);
} else {
t->skip = 1;
}
}
if (e->verbose)
message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
clocks_getunit());
#else
error("Calling an MPI function in non-MPI mode.");
#endif
}
/**
* @brief Activate all the FOF tasks.
*
* Marks all the other task types to be skipped.
*
* @param e The #engine to act on.
*/
void engine_activate_fof_tasks(struct engine *e) {
const ticks tic = getticks();
struct scheduler *s = &e->sched;
const int nr_tasks = s->nr_tasks;
struct task *tasks = s->tasks;
for (int k = 0; k < nr_tasks; k++) {
struct task *t = &tasks[k];
if (t->type == task_type_fof_self || t->type == task_type_fof_pair)
scheduler_activate(s, t);
else
t->skip = 1;
}
if (e->verbose)
message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
clocks_getunit());
}
/**
* @brief Run a FOF search.
*
* @param e the engine
* @param dump_results Are we writing group catalogues to output files?
* @param seed_black_holes Are we seeding black holes?
*/
void engine_fof(struct engine *e, const int dump_results,
const int seed_black_holes) {
#ifdef WITH_FOF
ticks tic = getticks();
/* Compute number of DM particles */
const long long total_nr_baryons =
e->total_nr_parts + e->total_nr_sparts + e->total_nr_bparts;
const long long total_nr_dmparts =
e->total_nr_gparts - e->total_nr_DM_background_gparts - total_nr_baryons;
/* Initialise FOF parameters and allocate FOF arrays. */
fof_allocate(e->s, total_nr_dmparts, e->fof_properties);
/* Make FOF tasks */
engine_make_fof_tasks(e);
/* and activate them. */
engine_activate_fof_tasks(e);
/* Perform local FOF tasks. */
engine_launch(e);
/* Perform FOF search over foreign particles and
* find groups which require black hole seeding. */
fof_search_tree(e->fof_properties, e->black_holes_properties,
e->physical_constants, e->cosmology, e->s, dump_results,
seed_black_holes);
/* Reset flag. */
e->run_fof = 0;
/* Flag that a FOF has taken place */
e->step_props |= engine_step_prop_fof;
/* ... and find the next FOF time */
if (seed_black_holes) engine_compute_next_fof_time(e);
if (engine_rank == 0)
message("Complete FOF search took: %.3f %s.",
clocks_from_ticks(getticks() - tic), clocks_getunit());
#else
error("SWIFT was not compiled with FOF enabled!");
#endif
}
This diff is collapsed.
This diff is collapsed.
...@@ -26,13 +26,21 @@ ...@@ -26,13 +26,21 @@
/* Config parameters. */ /* Config parameters. */
#include "../config.h" #include "../config.h"
/* Includes. */ /* Local headers. */
#include "cache.h" #include "cache.h"
#include "gravity_cache.h" #include "gravity_cache.h"
#include "task.h"
struct cell; struct cell;
struct engine; struct engine;
struct task;
/* Unique identifier of loop types */
#define TASK_LOOP_DENSITY 0
#define TASK_LOOP_GRADIENT 1
#define TASK_LOOP_FORCE 2
#define TASK_LOOP_LIMITER 3
#define TASK_LOOP_FEEDBACK 4
#define TASK_LOOP_SWALLOW 5
/** /**
* @brief A struct representing a runner's thread and its data. * @brief A struct representing a runner's thread and its data.
...@@ -75,6 +83,12 @@ struct runner { ...@@ -75,6 +83,12 @@ struct runner {
/* Function prototypes. */ /* Function prototypes. */
void runner_do_ghost(struct runner *r, struct cell *c, int timer); void runner_do_ghost(struct runner *r, struct cell *c, int timer);
void runner_do_extra_ghost(struct runner *r, struct cell *c, int timer); void runner_do_extra_ghost(struct runner *r, struct cell *c, int timer);
void runner_do_stars_ghost(struct runner *r, struct cell *c, int timer);
void runner_do_black_holes_density_ghost(struct runner *r, struct cell *c,
int timer);
void runner_do_black_holes_swallow_ghost(struct runner *r, struct cell *c,
int timer);
void runner_do_init_grav(struct runner *r, struct cell *c, int timer);
void runner_do_hydro_sort(struct runner *r, struct cell *c, int flag, void runner_do_hydro_sort(struct runner *r, struct cell *c, int flag,
int cleanup, int clock); int cleanup, int clock);
void runner_do_stars_sort(struct runner *r, struct cell *c, int flag, void runner_do_stars_sort(struct runner *r, struct cell *c, int flag,
...@@ -84,19 +98,38 @@ void runner_do_all_stars_sort(struct runner *r, struct cell *c); ...@@ -84,19 +98,38 @@ void runner_do_all_stars_sort(struct runner *r, struct cell *c);
void runner_do_drift_part(struct runner *r, struct cell *c, int timer); void runner_do_drift_part(struct runner *r, struct cell *c, int timer);
void runner_do_drift_gpart(struct runner *r, struct cell *c, int timer); void runner_do_drift_gpart(struct runner *r, struct cell *c, int timer);
void runner_do_drift_spart(struct runner *r, struct cell *c, int timer); void runner_do_drift_spart(struct runner *r, struct cell *c, int timer);
void runner_do_drift_bpart(struct runner *r, struct cell *c, int timer);
void runner_do_kick1(struct runner *r, struct cell *c, int timer); void runner_do_kick1(struct runner *r, struct cell *c, int timer);
void runner_do_kick2(struct runner *r, struct cell *c, int timer); void runner_do_kick2(struct runner *r, struct cell *c, int timer);
void runner_do_timestep(struct runner *r, struct cell *c, int timer);
void runner_do_end_hydro_force(struct runner *r, struct cell *c, int timer); void runner_do_end_hydro_force(struct runner *r, struct cell *c, int timer);
void runner_do_end_grav_force(struct runner *r, struct cell *c, int timer);
void runner_do_init(struct runner *r, struct cell *c, int timer); void runner_do_init(struct runner *r, struct cell *c, int timer);
void runner_do_cooling(struct runner *r, struct cell *c, int timer); void runner_do_cooling(struct runner *r, struct cell *c, int timer);
void runner_do_limiter(struct runner *r, struct cell *c, int force, int timer);
void runner_do_grav_mesh(struct runner *r, struct cell *c, int timer);
void runner_do_grav_external(struct runner *r, struct cell *c, int timer); void runner_do_grav_external(struct runner *r, struct cell *c, int timer);
void runner_do_grav_fft(struct runner *r, int timer); void runner_do_grav_fft(struct runner *r, int timer);
void runner_do_logger(struct runner *r, struct cell *c, int timer); void runner_do_logger(struct runner *r, struct cell *c, int timer);
void runner_do_fof_self(struct runner *r, struct cell *c, int timer); void runner_do_fof_self(struct runner *r, struct cell *c, int timer);
void runner_do_fof_pair(struct runner *r, struct cell *ci, struct cell *cj, void runner_do_fof_pair(struct runner *r, struct cell *ci, struct cell *cj,
int timer); int timer);
void runner_do_gas_swallow_self(struct runner *r, struct cell *c, int timer);
void runner_do_bh_swallow_self(struct runner *r, struct cell *c, int timer);
void runner_do_gas_swallow_pair(struct runner *r, struct cell *ci,
struct cell *cj, int timer);
void runner_do_bh_swallow_pair(struct runner *r, struct cell *ci,
struct cell *cj, int timer);
void runner_do_star_formation(struct runner *r, struct cell *c, int timer);
void runner_do_stars_resort(struct runner *r, struct cell *c, const int timer);
void runner_do_recv_gpart(struct runner *r, struct cell *c, int timer);
void runner_do_recv_part(struct runner *r, struct cell *c, int clear_sorts,
int timer);
void runner_do_recv_spart(struct runner *r, struct cell *c, int clear_sorts,
int timer);
void runner_do_recv_bpart(struct runner *r, struct cell *c, int clear_sorts,
int timer);
void *runner_main(void *data); void *runner_main(void *data);
void runner_do_drift_all_mapper(void *map_data, int num_elements,
void *extra_data);
#endif /* SWIFT_RUNNER_H */ #endif /* SWIFT_RUNNER_H */
/*******************************************************************************
* This file is part of SWIFT.
* Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
* Matthieu Schaller (matthieu.schaller@durham.ac.uk)
* 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
******************************************************************************/
/* Config parameters. */
#include "../config.h"
/* This object's header. */
#include "runner.h"
/* Local headers. */
#include "active.h"
#include "black_holes.h"
#include "cell.h"
#include "engine.h"
#include "timers.h"
/**
* @brief Process all the gas particles in a cell that have been flagged for
* swallowing by a black hole.
*
* This is done by recursing down to the leaf-level and skipping the sub-cells
* that have not been drifted as they would not have any particles with
* swallowing flag. We then loop over the particles with a flag and look into
* the space-wide list of black holes for the particle with the corresponding
* ID. If found, the BH swallows the gas particle and the gas particle is
* removed. If the cell is local, we may be looking for a foreign BH, in which
* case, we do not update the BH (that will be done on its node) but just remove
* the gas particle.
*
* @param r The thread #runner.
* @param c The #cell.
* @param timer Are we timing this?
*/
void runner_do_gas_swallow(struct runner *r, struct cell *c, int timer) {
struct engine *e = r->e;
struct space *s = e->s;
struct bpart *bparts = s->bparts;
const size_t nr_bpart = s->nr_bparts;
#ifdef WITH_MPI
struct bpart *bparts_foreign = s->bparts_foreign;
const size_t nr_bparts_foreign = s->nr_bparts_foreign;
#endif
struct part *parts = c->hydro.parts;
struct xpart *xparts = c->hydro.xparts;
/* Early abort?
* (We only want cells for which we drifted the gas as these are
* the only ones that could have gas particles that have been flagged
* for swallowing) */
if (c->hydro.count == 0 || c->hydro.ti_old_part != e->ti_current) {
return;
}
/* Loop over the progeny ? */
if (c->split) {
for (int k = 0; k < 8; k++) {
if (c->progeny[k] != NULL) {
struct cell *restrict cp = c->progeny[k];
runner_do_gas_swallow(r, cp, 0);
}
}
} else {
/* Loop over all the gas particles in the cell
* Note that the cell (and hence the parts) may be local or foreign. */
const size_t nr_parts = c->hydro.count;
for (size_t k = 0; k < nr_parts; k++) {
/* Get a handle on the part. */
struct part *const p = &parts[k];
struct xpart *const xp = &xparts[k];
/* Ignore inhibited particles (they have already been removed!) */
if (part_is_inhibited(p, e)) continue;
/* Get the ID of the black holes that will swallow this part */
const long long swallow_id =
black_holes_get_part_swallow_id(&p->black_holes_data);
/* Has this particle been flagged for swallowing? */
if (swallow_id >= 0) {
#ifdef SWIFT_DEBUG_CHECKS
if (p->ti_drift != e->ti_current)
error("Trying to swallow an un-drifted particle.");
#endif
/* ID of the BH swallowing this particle */
const long long BH_id = swallow_id;
/* Have we found this particle's BH already? */
int found = 0;
/* Let's look for the hungry black hole in the local list */
for (size_t i = 0; i < nr_bpart; ++i) {
/* Get a handle on the bpart. */
struct bpart *bp = &bparts[i];
if (bp->id == BH_id) {
/* Lock the space as we are going to work directly on the bpart list
*/
lock_lock(&s->lock);
/* Swallow the gas particle (i.e. update the BH properties) */
black_holes_swallow_part(bp, p, xp, e->cosmology);
/* Release the space as we are done updating the bpart */
if (lock_unlock(&s->lock) != 0)
error("Failed to unlock the space.");
message("BH %lld swallowing gas particle %lld", bp->id, p->id);
/* If the gas particle is local, remove it */
if (c->nodeID == e->nodeID) {
message("BH %lld removing gas particle %lld", bp->id, p->id);
lock_lock(&e->s->lock);
/* Re-check that the particle has not been removed
* by another thread before we do the deed. */
if (!part_is_inhibited(p, e)) {
/* Finally, remove the gas particle from the system
* Recall that the gpart associated with it is also removed
* at the same time. */
cell_remove_part(e, c, p, xp);
}
if (lock_unlock(&e->s->lock) != 0)
error("Failed to unlock the space!");
}
/* In any case, prevent the particle from being re-swallowed */
black_holes_mark_part_as_swallowed(&p->black_holes_data);
found = 1;
break;
}
} /* Loop over local BHs */
#ifdef WITH_MPI
/* We could also be in the case of a local gas particle being
* swallowed by a foreign BH. In this case, we won't update the
* BH but just remove the particle from the local list. */
if (c->nodeID == e->nodeID && !found) {
/* Let's look for the foreign hungry black hole */
for (size_t i = 0; i < nr_bparts_foreign; ++i) {
/* Get a handle on the bpart. */
struct bpart *bp = &bparts_foreign[i];
if (bp->id == BH_id) {
message("BH %lld removing gas particle %lld (foreign BH case)",
bp->id, p->id);
lock_lock(&e->s->lock);
/* Re-check that the particle has not been removed
* by another thread before we do the deed. */
if (!part_is_inhibited(p, e)) {
/* Finally, remove the gas particle from the system */
cell_remove_part(e, c, p, xp);
}
if (lock_unlock(&e->s->lock) != 0)
error("Failed to unlock the space!");
found = 1;
break;
}
} /* Loop over foreign BHs */
} /* Is the cell local? */
#endif
/* If we have a local particle, we must have found the BH in one
* of our list of black holes. */
if (c->nodeID == e->nodeID && !found) {
error("Gas particle %lld could not find BH %lld to be swallowed",
p->id, swallow_id);
}
} /* Part was flagged for swallowing */
} /* Loop over the parts */
} /* Cell is not split */
}
/**
* @brief Processing of gas particles to swallow - self task case.
*
* @param r The thread #runner.
* @param c The #cell.
* @param timer Are we timing this?
*/
void runner_do_gas_swallow_self(struct runner *r, struct cell *c, int timer) {
#ifdef SWIFT_DEBUG_CHECKS
if (c->nodeID != r->e->nodeID) error("Running self task on foreign node");
if (!cell_is_active_black_holes(c, r->e))
error("Running self task on inactive cell");
#endif
runner_do_gas_swallow(r, c, timer);
}
/**
* @brief Processing of gas particles to swallow - pair task case.
*
* @param r The thread #runner.
* @param ci First #cell.
* @param cj Second #cell.
* @param timer Are we timing this?
*/
void runner_do_gas_swallow_pair(struct runner *r, struct cell *ci,
struct cell *cj, int timer) {
const struct engine *e = r->e;
#ifdef SWIFT_DEBUG_CHECKS
if (ci->nodeID != e->nodeID && cj->nodeID != e->nodeID)
error("Running pair task on foreign node");
#endif
/* Run the swallowing loop only in the cell that is the neighbour of the
* active BH */
if (cell_is_active_black_holes(cj, e)) runner_do_gas_swallow(r, ci, timer);
if (cell_is_active_black_holes(ci, e)) runner_do_gas_swallow(r, cj, timer);
}
/**
* @brief Process all the BH particles in a cell that have been flagged for
* swallowing by a black hole.
*
* This is done by recursing down to the leaf-level and skipping the sub-cells
* that have not been drifted as they would not have any particles with
* swallowing flag. We then loop over the particles with a flag and look into
* the space-wide list of black holes for the particle with the corresponding
* ID. If found, the BH swallows the BH particle and the BH particle is
* removed. If the cell is local, we may be looking for a foreign BH, in which
* case, we do not update the BH (that will be done on its node) but just remove
* the BH particle.
*
* @param r The thread #runner.
* @param c The #cell.
* @param timer Are we timing this?
*/
void runner_do_bh_swallow(struct runner *r, struct cell *c, int timer) {
struct engine *e = r->e;
struct space *s = e->s;
struct bpart *bparts = s->bparts;
const size_t nr_bpart = s->nr_bparts;
#ifdef WITH_MPI
struct bpart *bparts_foreign = s->bparts_foreign;
const size_t nr_bparts_foreign = s->nr_bparts_foreign;
#endif
struct bpart *cell_bparts = c->black_holes.parts;