diff --git a/README b/README
index ae329e84310e0ecb99cf14f5fd1a7d10f3d99549..a35fa35f85509a79299cbc2cb0b11aa1f74f61e2 100644
--- a/README
+++ b/README
@@ -6,7 +6,7 @@
/____/ |__/|__/___/_/ /_/
SPH With Inter-dependent Fine-grained Tasking
- Version : 0.8.5
+ Version : 0.9.0
Website: www.swiftsim.com
Twitter: @SwiftSimulation
diff --git a/README.md b/README.md
index c44d576062a228061f1a8350f26fb6d6f06754fb..409d15d7e2525ab10e3f79b58bc1adf790c8ea29 100644
--- a/README.md
+++ b/README.md
@@ -93,7 +93,7 @@ Runtime parameters
/____/ |__/|__/___/_/ /_/
SPH With Inter-dependent Fine-grained Tasking
- Version : 0.8.5
+ Version : 0.9.0
Website: www.swiftsim.com
Twitter: @SwiftSimulation
diff --git a/configure.ac b/configure.ac
index 5e83a497add952997500694d1d477ee74a237038..3c5e43b579e8cd6065d962294f535467f8fd1441 100644
--- a/configure.ac
+++ b/configure.ac
@@ -16,7 +16,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Init the project.
-AC_INIT([SWIFT],[0.8.5],[https://gitlab.cosma.dur.ac.uk/swift/swiftsim])
+AC_INIT([SWIFT],[0.9.0],[https://gitlab.cosma.dur.ac.uk/swift/swiftsim])
swift_config_flags="$*"
# We want to stop when given unrecognised options. No subdirs so this is safe.
@@ -1619,6 +1619,9 @@ AC_ARG_WITH([gravity],
)
case "$with_gravity" in
+ with-potential)
+ AC_MSG_ERROR([The gravity 'with-potential' scheme does not exist anymore. Please use the basic scheme which now contains potentials.])
+ ;;
with-multi-softening)
AC_DEFINE([MULTI_SOFTENING_GRAVITY], [1], [Gravity scheme with per-particle type softening value and background particles])
;;
diff --git a/doc/RTD/source/conf.py b/doc/RTD/source/conf.py
index e4cdb061be7750b5a4bb8a5573664abfc66c4ba2..35d5b592800cc8556d074c6a5bbf68d9ec21e54c 100644
--- a/doc/RTD/source/conf.py
+++ b/doc/RTD/source/conf.py
@@ -23,9 +23,9 @@ copyright = '2014-2020, SWIFT Collaboration'
author = 'SWIFT Team'
# The short X.Y version
-version = '0.8'
+version = '0.9'
# The full version, including alpha/beta/rc tags
-release = '0.8.5'
+release = '0.9.0'
# -- General configuration ---------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index b03ee5ec42d3061c8594f27276e93248142697be..b75d8cf5bde1779539002c9f398309472c9857bc 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -41,25 +41,26 @@ noinst_LTLIBRARIES += libgrav_mpi.la
endif
# List required headers
-include_HEADERS = space.h runner.h queue.h task.h lock.h cell.h part.h const.h \
- engine.h swift.h serial_io.h timers.h debug.h scheduler.h proxy.h parallel_io.h \
- common_io.h single_io.h distributed_io.h map.h tools.h partition_fixed_costs.h \
- partition.h clocks.h parser.h physical_constants.h physical_constants_cgs.h potential.h version.h \
- hydro_properties.h riemann.h threadpool.h cooling_io.h cooling.h cooling_struct.h cooling_properties.h \
- statistics.h memswap.h cache.h runner_doiact_hydro_vec.h profiler.h entropy_floor.h \
- dump.h logger.h active.h timeline.h xmf.h gravity_properties.h gravity_derivatives.h \
- gravity_softened_derivatives.h vector_power.h collectgroup.h hydro_space.h sort_part.h \
- chemistry.h chemistry_io.h chemistry_struct.h cosmology.h restart.h space_getsid.h utilities.h \
- mesh_gravity.h cbrt.h exp10.h velociraptor_interface.h swift_velociraptor_part.h output_list.h \
- logger_io.h tracers_io.h tracers.h tracers_struct.h star_formation_io.h fof.h fof_struct.h fof_io.h \
- multipole.h multipole_accept.h multipole_struct.h binomial.h integer_power.h sincos.h \
- star_formation_struct.h star_formation.h star_formation_iact.h \
- star_formation_logger.h star_formation_logger_struct.h \
- pressure_floor.h pressure_floor_struct.h pressure_floor_iact.h \
- velociraptor_struct.h velociraptor_io.h random.h memuse.h mpiuse.h memuse_rnodes.h \
- black_holes.h black_holes_io.h black_holes_properties.h black_holes_struct.h \
- feedback.h feedback_struct.h feedback_properties.h \
- space_unique_id.h line_of_sight.h io_compression.h
+include_HEADERS = space.h runner.h queue.h task.h lock.h cell.h part.h const.h
+include_HEADERS += cell_hydro.h cell_stars.h cell_grav.h cell_sinks.h cell_black_holes.h
+include_HEADERS += engine.h swift.h serial_io.h timers.h debug.h scheduler.h proxy.h parallel_io.h
+include_HEADERS += common_io.h single_io.h distributed_io.h map.h tools.h partition_fixed_costs.h
+include_HEADERS += partition.h clocks.h parser.h physical_constants.h physical_constants_cgs.h potential.h version.h
+include_HEADERS += hydro_properties.h riemann.h threadpool.h cooling_io.h cooling.h cooling_struct.h cooling_properties.h
+include_HEADERS += statistics.h memswap.h cache.h runner_doiact_hydro_vec.h profiler.h entropy_floor.h
+include_HEADERS += dump.h logger.h active.h timeline.h xmf.h gravity_properties.h gravity_derivatives.h
+include_HEADERS += gravity_softened_derivatives.h vector_power.h collectgroup.h hydro_space.h sort_part.h
+include_HEADERS += chemistry.h chemistry_io.h chemistry_struct.h cosmology.h restart.h space_getsid.h utilities.h
+include_HEADERS += mesh_gravity.h cbrt.h exp10.h velociraptor_interface.h swift_velociraptor_part.h output_list.h
+include_HEADERS += logger_io.h tracers_io.h tracers.h tracers_struct.h star_formation_io.h fof.h fof_struct.h fof_io.h
+include_HEADERS += multipole.h multipole_accept.h multipole_struct.h binomial.h integer_power.h sincos.h
+include_HEADERS += star_formation_struct.h star_formation.h star_formation_iact.h
+include_HEADERS += star_formation_logger.h star_formation_logger_struct.h
+include_HEADERS += pressure_floor.h pressure_floor_struct.h pressure_floor_iact.h
+include_HEADERS += velociraptor_struct.h velociraptor_io.h random.h memuse.h mpiuse.h memuse_rnodes.h
+include_HEADERS += black_holes.h black_holes_io.h black_holes_properties.h black_holes_struct.h
+include_HEADERS += feedback.h feedback_struct.h feedback_properties.h
+include_HEADERS += space_unique_id.h line_of_sight.h io_compression.h
# source files for EAGLE cooling
QLA_COOLING_SOURCES =
@@ -99,28 +100,36 @@ GEAR_FEEDBACK_SOURCES += feedback/GEAR/stellar_evolution.c feedback/GEAR/feedbac
endif
# Common source files
-AM_SOURCES = space.c runner_main.c runner_doiact_hydro.c runner_doiact_limiter.c \
- runner_doiact_stars.c runner_doiact_black_holes.c runner_ghost.c runner_recv.c \
- runner_sort.c runner_drift.c runner_black_holes.c runner_time_integration.c \
- runner_doiact_hydro_vec.c runner_others.c runner_doiact_sinks.c \
- runner_doiact_rt.c\
- queue.c task.c cell.c engine.c engine_maketasks.c engine_split_particles.c \
- engine_marktasks.c engine_drift.c engine_unskip.c engine_collect_end_of_step.c \
- engine_redistribute.c engine_fof.c serial_io.c timers.c debug.c scheduler.c \
- proxy.c parallel_io.c units.c common_io.c single_io.c multipole.c version.c map.c \
- kernel_hydro.c tools.c part.c partition.c clocks.c parser.c distributed_io.c \
- physical_constants.c potential.c hydro_properties.c \
- threadpool.c cooling.c star_formation.c \
- statistics.c profiler.c dump.c logger.c \
- part_type.c xmf.c gravity_properties.c gravity.c \
- collectgroup.c hydro_space.c equation_of_state.c io_compression.c \
- chemistry.c cosmology.c restart.c mesh_gravity.c velociraptor_interface.c \
- output_list.c velociraptor_dummy.c logger_io.c memuse.c mpiuse.c memuse_rnodes.c fof.c \
- hashmap.c pressure_floor.c space_unique_id.c output_options.c line_of_sight.c \
- $(QLA_COOLING_SOURCES) \
- $(EAGLE_COOLING_SOURCES) $(EAGLE_FEEDBACK_SOURCES) \
- $(GRACKLE_COOLING_SOURCES) $(GEAR_FEEDBACK_SOURCES) \
- $(COLIBRE_COOLING_SOURCES)
+AM_SOURCES = space.c space_rebuild.c space_regrid.c space_unique_id.c
+AM_SOURCES += space_sort.c space_split.c space_extras.c space_first_init.c space_init.c
+AM_SOURCES += space_cell_index.c space_recycle.c
+AM_SOURCES += runner_main.c runner_doiact_hydro.c runner_doiact_limiter.c
+AM_SOURCES += runner_doiact_stars.c runner_doiact_black_holes.c runner_ghost.c runner_recv.c
+AM_SOURCES += runner_sort.c runner_drift.c runner_black_holes.c runner_time_integration.c
+AM_SOURCES += runner_doiact_hydro_vec.c runner_others.c runner_doiact_sinks.c
+AM_SOURCES += runner_doiact_rt.c
+AM_SOURCES += cell.c cell_convert_part.c cell_drift.c cell_lock.c cell_pack.c cell_split.c
+AM_SOURCES += cell_unskip.c
+AM_SOURCES += engine.c engine_maketasks.c engine_split_particles.c engine_strays.c
+AM_SOURCES += engine_marktasks.c engine_drift.c engine_unskip.c engine_collect_end_of_step.c
+AM_SOURCES += engine_redistribute.c engine_fof.c engine_proxy.c engine_io.c engine_config.c
+AM_SOURCES += queue.c task.c timers.c debug.c scheduler.c proxy.c version.c
+AM_SOURCES += common_io.c common_io_copy.c common_io_cells.c common_io_fields.c
+AM_SOURCES += single_io.c serial_io.c distributed_io.c parallel_io.c
+AM_SOURCES += output_options.c line_of_sight.c restart.c parser.c xmf.c
+AM_SOURCES += kernel_hydro.c tools.c map.c part.c partition.c clocks.c
+AM_SOURCES += physical_constants.c units.c potential.c hydro_properties.c
+AM_SOURCES += threadpool.c cooling.c star_formation.c
+AM_SOURCES += statistics.c profiler.c dump.c logger.c part_type.c
+AM_SOURCES += gravity_properties.c gravity.c multipole.c
+AM_SOURCES += collectgroup.c hydro_space.c equation_of_state.c io_compression.c
+AM_SOURCES += chemistry.c cosmology.c mesh_gravity.c velociraptor_interface.c
+AM_SOURCES += output_list.c velociraptor_dummy.c logger_io.c memuse.c mpiuse.c memuse_rnodes.c fof.c
+AM_SOURCES += hashmap.c pressure_floor.c
+AM_SOURCES += $(QLA_COOLING_SOURCES)
+AM_SOURCES += $(EAGLE_COOLING_SOURCES) $(EAGLE_FEEDBACK_SOURCES)
+AM_SOURCES += $(GRACKLE_COOLING_SOURCES) $(GEAR_FEEDBACK_SOURCES)
+AM_SOURCES += $(COLIBRE_COOLING_SOURCES)
# Include files for distribution, not installation.
nobase_noinst_HEADERS = align.h approx_math.h atomic.h barrier.h cycle.h error.h inline.h kernel_hydro.h kernel_gravity.h
diff --git a/src/cell.c b/src/cell.c
index 05220bd4149834d068f38626b3a9dcbe5dcc658f..8bd26b32343f2307673c5f0d9c3abfd68172c187 100644
--- a/src/cell.c
+++ b/src/cell.c
@@ -47,33 +47,11 @@
#include "cell.h"
/* Local headers. */
-#include "active.h"
-#include "atomic.h"
-#include "black_holes.h"
-#include "chemistry.h"
-#include "drift.h"
#include "engine.h"
-#include "entropy_floor.h"
#include "error.h"
-#include "feedback.h"
-#include "gravity.h"
-#include "hydro.h"
-#include "hydro_properties.h"
-#include "memswap.h"
-#include "minmax.h"
#include "multipole.h"
-#include "pressure_floor.h"
-#include "rt.h"
-#include "scheduler.h"
#include "space.h"
-#include "space_getsid.h"
-#include "star_formation.h"
-#include "stars.h"
-#include "timers.h"
#include "tools.h"
-#include "tracers.h"
-
-extern int engine_star_resort_task_depth;
/* Global variables. */
int cell_next_tag = 0;
@@ -171,14 +149,14 @@ struct cell_split_pair cell_split_pairs[13] = {
*
* @param c The #cell.
*/
-int cell_getsize(struct cell *c) {
+int cell_get_tree_size(struct cell *c) {
/* Number of cells in this subtree. */
int count = 1;
/* Sum up the progeny if split. */
if (c->split)
for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL) count += cell_getsize(c->progeny[k]);
+ if (c->progeny[k] != NULL) count += cell_get_tree_size(c->progeny[k]);
/* Return the final count. */
return count;
@@ -522,7062 +500,902 @@ int cell_count_gparts_for_tasks(const struct cell *c) {
}
/**
- * @brief Pack the data of the given cell and all it's sub-cells.
+ * @brief Sanitizes the smoothing length values of cells by setting large
+ * outliers to more sensible values.
*
- * @param c The #cell.
- * @param pc Pointer to an array of packed cells in which the
- * cells will be packed.
- * @param with_gravity Are we running with gravity and hence need
- * to exchange multipoles?
+ * Each cell with <1000 part will be processed. We limit h to be the size of
+ * the cell and replace 0s with a good estimate.
*
- * @return The number of packed cells.
+ * @param c The cell.
+ * @param treated Has the cell already been sanitized at this level ?
*/
-int cell_pack(struct cell *restrict c, struct pcell *restrict pc,
- const int with_gravity) {
-#ifdef WITH_MPI
+void cell_sanitize(struct cell *c, int treated) {
+ const int count = c->hydro.count;
+ const int scount = c->stars.count;
+ struct part *parts = c->hydro.parts;
+ struct spart *sparts = c->stars.parts;
+ float h_max = 0.f;
+ float stars_h_max = 0.f;
- /* Start by packing the data of the current cell. */
- pc->hydro.h_max = c->hydro.h_max;
- pc->stars.h_max = c->stars.h_max;
- pc->black_holes.h_max = c->black_holes.h_max;
- pc->sinks.r_cut_max = c->sinks.r_cut_max;
-
- pc->hydro.ti_end_min = c->hydro.ti_end_min;
- pc->hydro.ti_end_max = c->hydro.ti_end_max;
- pc->grav.ti_end_min = c->grav.ti_end_min;
- pc->grav.ti_end_max = c->grav.ti_end_max;
- pc->stars.ti_end_min = c->stars.ti_end_min;
- pc->stars.ti_end_max = c->stars.ti_end_max;
- pc->sinks.ti_end_min = c->sinks.ti_end_min;
- pc->sinks.ti_end_max = c->sinks.ti_end_max;
- pc->black_holes.ti_end_min = c->black_holes.ti_end_min;
- pc->black_holes.ti_end_max = c->black_holes.ti_end_max;
-
- pc->hydro.ti_old_part = c->hydro.ti_old_part;
- pc->grav.ti_old_part = c->grav.ti_old_part;
- pc->grav.ti_old_multipole = c->grav.ti_old_multipole;
- pc->stars.ti_old_part = c->stars.ti_old_part;
- pc->black_holes.ti_old_part = c->black_holes.ti_old_part;
- pc->sinks.ti_old_part = c->sinks.ti_old_part;
-
- pc->hydro.count = c->hydro.count;
- pc->grav.count = c->grav.count;
- pc->stars.count = c->stars.count;
- pc->sinks.count = c->sinks.count;
- pc->black_holes.count = c->black_holes.count;
- pc->maxdepth = c->maxdepth;
-
- /* Copy the Multipole related information */
- if (with_gravity) {
- const struct gravity_tensors *mp = c->grav.multipole;
-
- pc->grav.m_pole = mp->m_pole;
- pc->grav.CoM[0] = mp->CoM[0];
- pc->grav.CoM[1] = mp->CoM[1];
- pc->grav.CoM[2] = mp->CoM[2];
- pc->grav.CoM_rebuild[0] = mp->CoM_rebuild[0];
- pc->grav.CoM_rebuild[1] = mp->CoM_rebuild[1];
- pc->grav.CoM_rebuild[2] = mp->CoM_rebuild[2];
- pc->grav.r_max = mp->r_max;
- pc->grav.r_max_rebuild = mp->r_max_rebuild;
+ /* Treat cells will <1000 particles */
+ if (count < 1000 && !treated) {
+ /* Get an upper bound on h */
+ const float upper_h_max = c->dmin / (1.2f * kernel_gamma);
+
+ /* Apply it */
+ for (int i = 0; i < count; ++i) {
+ if (parts[i].h == 0.f || parts[i].h > upper_h_max)
+ parts[i].h = upper_h_max;
+ }
+ for (int i = 0; i < scount; ++i) {
+ if (sparts[i].h == 0.f || sparts[i].h > upper_h_max)
+ sparts[i].h = upper_h_max;
+ }
}
-#ifdef SWIFT_DEBUG_CHECKS
- pc->cellID = c->cellID;
-#endif
+ /* Recurse and gather the new h_max values */
+ if (c->split) {
+ for (int k = 0; k < 8; ++k) {
+ if (c->progeny[k] != NULL) {
+ /* Recurse */
+ cell_sanitize(c->progeny[k], (count < 1000));
- /* Fill in the progeny, depth-first recursion. */
- int count = 1;
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL) {
- pc->progeny[k] = count;
- count += cell_pack(c->progeny[k], &pc[count], with_gravity);
- } else {
- pc->progeny[k] = -1;
+ /* And collect */
+ h_max = max(h_max, c->progeny[k]->hydro.h_max);
+ stars_h_max = max(stars_h_max, c->progeny[k]->stars.h_max);
+ }
}
+ } else {
+ /* Get the new value of h_max */
+ for (int i = 0; i < count; ++i) h_max = max(h_max, parts[i].h);
+ for (int i = 0; i < scount; ++i)
+ stars_h_max = max(stars_h_max, sparts[i].h);
+ }
- /* Return the number of packed cells used. */
- c->mpi.pcell_size = count;
- return count;
+ /* Record the change */
+ c->hydro.h_max = h_max;
+ c->stars.h_max = stars_h_max;
+}
-#else
- error("SWIFT was not compiled with MPI support.");
- return 0;
-#endif
+/**
+ * @brief Cleans the links in a given cell.
+ *
+ * @param c Cell to act upon
+ * @param data Unused parameter
+ */
+void cell_clean_links(struct cell *c, void *data) {
+ c->hydro.density = NULL;
+ c->hydro.gradient = NULL;
+ c->hydro.force = NULL;
+ c->hydro.limiter = NULL;
+ c->hydro.rt_inject = NULL;
+ c->grav.grav = NULL;
+ c->grav.mm = NULL;
+ c->stars.density = NULL;
+ c->stars.feedback = NULL;
+ c->black_holes.density = NULL;
+ c->black_holes.swallow = NULL;
+ c->black_holes.do_gas_swallow = NULL;
+ c->black_holes.do_bh_swallow = NULL;
+ c->black_holes.feedback = NULL;
}
/**
- * @brief Pack the tag of the given cell and all it's sub-cells.
+ * @brief Checks that the #part in a cell are at the
+ * current point in time
*
- * @param c The #cell.
- * @param tags Pointer to an array of packed tags.
+ * Calls error() if the cell is not at the current time.
*
- * @return The number of packed tags.
+ * @param c Cell to act upon
+ * @param data The current time on the integer time-line
*/
-int cell_pack_tags(const struct cell *c, int *tags) {
-#ifdef WITH_MPI
+void cell_check_part_drift_point(struct cell *c, void *data) {
+#ifdef SWIFT_DEBUG_CHECKS
- /* Start by packing the data of the current cell. */
- tags[0] = c->mpi.tag;
+ const integertime_t ti_drift = *(integertime_t *)data;
- /* Fill in the progeny, depth-first recursion. */
- int count = 1;
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL)
- count += cell_pack_tags(c->progeny[k], &tags[count]);
+ /* Only check local cells */
+ if (c->nodeID != engine_rank) return;
-#ifdef SWIFT_DEBUG_CHECKS
- if (c->mpi.pcell_size != count) error("Inconsistent tag and pcell count!");
-#endif // SWIFT_DEBUG_CHECKS
+ /* Only check cells with content */
+ if (c->hydro.count == 0) return;
- /* Return the number of packed tags used. */
- return count;
+ if (c->hydro.ti_old_part != ti_drift)
+ error("Cell in an incorrect time-zone! c->hydro.ti_old=%lld ti_drift=%lld",
+ c->hydro.ti_old_part, ti_drift);
+ for (int i = 0; i < c->hydro.count; ++i)
+ if (c->hydro.parts[i].ti_drift != ti_drift &&
+ c->hydro.parts[i].time_bin != time_bin_inhibited)
+ error("part in an incorrect time-zone! p->ti_drift=%lld ti_drift=%lld",
+ c->hydro.parts[i].ti_drift, ti_drift);
#else
- error("SWIFT was not compiled with MPI support.");
- return 0;
+ error("Calling debugging code without debugging flag activated.");
#endif
}
-void cell_pack_part_swallow(const struct cell *c,
- struct black_holes_part_data *data) {
-
- const size_t count = c->hydro.count;
- const struct part *parts = c->hydro.parts;
-
- for (size_t i = 0; i < count; ++i) {
- data[i] = parts[i].black_holes_data;
- }
-}
-
-void cell_unpack_part_swallow(struct cell *c,
- const struct black_holes_part_data *data) {
-
- const size_t count = c->hydro.count;
- struct part *parts = c->hydro.parts;
-
- for (size_t i = 0; i < count; ++i) {
- parts[i].black_holes_data = data[i];
- }
-}
-
-void cell_pack_bpart_swallow(const struct cell *c,
- struct black_holes_bpart_data *data) {
-
- const size_t count = c->black_holes.count;
- const struct bpart *bparts = c->black_holes.parts;
-
- for (size_t i = 0; i < count; ++i) {
- data[i] = bparts[i].merger_data;
- }
-}
-
-void cell_unpack_bpart_swallow(struct cell *c,
- const struct black_holes_bpart_data *data) {
-
- const size_t count = c->black_holes.count;
- struct bpart *bparts = c->black_holes.parts;
-
- for (size_t i = 0; i < count; ++i) {
- bparts[i].merger_data = data[i];
- }
-}
-
/**
- * @brief Unpack the data of a given cell and its sub-cells.
+ * @brief Checks that the #gpart in a cell are at the
+ * current point in time
*
- * @param pc An array of packed #pcell.
- * @param c The #cell in which to unpack the #pcell.
- * @param s The #space in which the cells are created.
- * @param with_gravity Are we running with gravity and hence need
- * to exchange multipoles?
+ * Calls error() if the cell is not at the current time.
*
- * @return The number of cells created.
+ * @param c Cell to act upon
+ * @param data The current time on the integer time-line
*/
-int cell_unpack(struct pcell *restrict pc, struct cell *restrict c,
- struct space *restrict s, const int with_gravity) {
-#ifdef WITH_MPI
-
- /* Unpack the current pcell. */
- c->hydro.h_max = pc->hydro.h_max;
- c->stars.h_max = pc->stars.h_max;
- c->black_holes.h_max = pc->black_holes.h_max;
- c->sinks.r_cut_max = pc->sinks.r_cut_max;
-
- c->hydro.ti_end_min = pc->hydro.ti_end_min;
- c->hydro.ti_end_max = pc->hydro.ti_end_max;
- c->grav.ti_end_min = pc->grav.ti_end_min;
- c->grav.ti_end_max = pc->grav.ti_end_max;
- c->stars.ti_end_min = pc->stars.ti_end_min;
- c->stars.ti_end_max = pc->stars.ti_end_max;
- c->black_holes.ti_end_min = pc->black_holes.ti_end_min;
- c->black_holes.ti_end_max = pc->black_holes.ti_end_max;
- c->sinks.ti_end_min = pc->sinks.ti_end_min;
- c->sinks.ti_end_max = pc->sinks.ti_end_max;
-
- c->hydro.ti_old_part = pc->hydro.ti_old_part;
- c->grav.ti_old_part = pc->grav.ti_old_part;
- c->grav.ti_old_multipole = pc->grav.ti_old_multipole;
- c->stars.ti_old_part = pc->stars.ti_old_part;
- c->black_holes.ti_old_part = pc->black_holes.ti_old_part;
- c->sinks.ti_old_part = pc->sinks.ti_old_part;
-
- c->hydro.count = pc->hydro.count;
- c->grav.count = pc->grav.count;
- c->stars.count = pc->stars.count;
- c->sinks.count = pc->sinks.count;
- c->black_holes.count = pc->black_holes.count;
- c->maxdepth = pc->maxdepth;
-
+void cell_check_gpart_drift_point(struct cell *c, void *data) {
#ifdef SWIFT_DEBUG_CHECKS
- c->cellID = pc->cellID;
-#endif
- /* Copy the Multipole related information */
- if (with_gravity) {
- struct gravity_tensors *mp = c->grav.multipole;
-
- mp->m_pole = pc->grav.m_pole;
- mp->CoM[0] = pc->grav.CoM[0];
- mp->CoM[1] = pc->grav.CoM[1];
- mp->CoM[2] = pc->grav.CoM[2];
- mp->CoM_rebuild[0] = pc->grav.CoM_rebuild[0];
- mp->CoM_rebuild[1] = pc->grav.CoM_rebuild[1];
- mp->CoM_rebuild[2] = pc->grav.CoM_rebuild[2];
- mp->r_max = pc->grav.r_max;
- mp->r_max_rebuild = pc->grav.r_max_rebuild;
- }
+ const integertime_t ti_drift = *(integertime_t *)data;
- /* Number of new cells created. */
- int count = 1;
+ /* Only check local cells */
+ if (c->nodeID != engine_rank) return;
- /* Fill the progeny recursively, depth-first. */
- c->split = 0;
- for (int k = 0; k < 8; k++)
- if (pc->progeny[k] >= 0) {
- struct cell *temp;
- space_getcells(s, 1, &temp);
- temp->hydro.count = 0;
- temp->grav.count = 0;
- temp->stars.count = 0;
- temp->loc[0] = c->loc[0];
- temp->loc[1] = c->loc[1];
- temp->loc[2] = c->loc[2];
- temp->width[0] = c->width[0] / 2;
- temp->width[1] = c->width[1] / 2;
- temp->width[2] = c->width[2] / 2;
- temp->dmin = c->dmin / 2;
- if (k & 4) temp->loc[0] += temp->width[0];
- if (k & 2) temp->loc[1] += temp->width[1];
- if (k & 1) temp->loc[2] += temp->width[2];
- temp->depth = c->depth + 1;
- temp->split = 0;
- temp->hydro.dx_max_part = 0.f;
- temp->hydro.dx_max_sort = 0.f;
- temp->stars.dx_max_part = 0.f;
- temp->stars.dx_max_sort = 0.f;
- temp->black_holes.dx_max_part = 0.f;
- temp->nodeID = c->nodeID;
- temp->parent = c;
- c->progeny[k] = temp;
- c->split = 1;
- count += cell_unpack(&pc[pc->progeny[k]], temp, s, with_gravity);
- }
+ /* Only check cells with content */
+ if (c->grav.count == 0) return;
- /* Return the total number of unpacked cells. */
- c->mpi.pcell_size = count;
- return count;
+ if (c->grav.ti_old_part != ti_drift)
+ error(
+ "Cell in an incorrect time-zone! c->grav.ti_old_part=%lld "
+ "ti_drift=%lld",
+ c->grav.ti_old_part, ti_drift);
+ for (int i = 0; i < c->grav.count; ++i)
+ if (c->grav.parts[i].ti_drift != ti_drift &&
+ c->grav.parts[i].time_bin != time_bin_inhibited)
+ error("g-part in an incorrect time-zone! gp->ti_drift=%lld ti_drift=%lld",
+ c->grav.parts[i].ti_drift, ti_drift);
#else
- error("SWIFT was not compiled with MPI support.");
- return 0;
+ error("Calling debugging code without debugging flag activated.");
#endif
}
/**
- * @brief Unpack the tags of a given cell and its sub-cells.
+ * @brief Checks that the #sink in a cell are at the
+ * current point in time
*
- * @param tags An array of tags.
- * @param c The #cell in which to unpack the tags.
+ * Calls error() if the cell is not at the current time.
*
- * @return The number of tags created.
+ * @param c Cell to act upon
+ * @param data The current time on the integer time-line
*/
-int cell_unpack_tags(const int *tags, struct cell *restrict c) {
-#ifdef WITH_MPI
-
- /* Unpack the current pcell. */
- c->mpi.tag = tags[0];
+void cell_check_sink_drift_point(struct cell *c, void *data) {
+#ifdef SWIFT_DEBUG_CHECKS
- /* Number of new cells created. */
- int count = 1;
+ const integertime_t ti_drift = *(integertime_t *)data;
- /* Fill the progeny recursively, depth-first. */
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL) {
- count += cell_unpack_tags(&tags[count], c->progeny[k]);
- }
+ /* Only check local cells */
+ if (c->nodeID != engine_rank) return;
-#ifdef SWIFT_DEBUG_CHECKS
- if (c->mpi.pcell_size != count) error("Inconsistent tag and pcell count!");
-#endif // SWIFT_DEBUG_CHECKS
+ /* Only check cells with content */
+ if (c->sinks.count == 0) return;
- /* Return the total number of unpacked tags. */
- return count;
+ if (c->sinks.ti_old_part != ti_drift)
+ error(
+ "Cell in an incorrect time-zone! c->sinks.ti_old_part=%lld "
+ "ti_drift=%lld",
+ c->sinks.ti_old_part, ti_drift);
+ for (int i = 0; i < c->sinks.count; ++i)
+ if (c->sinks.parts[i].ti_drift != ti_drift &&
+ c->sinks.parts[i].time_bin != time_bin_inhibited)
+ error(
+ "sink-part in an incorrect time-zone! sink->ti_drift=%lld "
+ "ti_drift=%lld",
+ c->sinks.parts[i].ti_drift, ti_drift);
#else
- error("SWIFT was not compiled with MPI support.");
- return 0;
+ error("Calling debugging code without debugging flag activated.");
#endif
}
/**
- * @brief Pack the time information of the given cell and all it's sub-cells.
+ * @brief Checks that the #spart in a cell are at the
+ * current point in time
*
- * @param c The #cell.
- * @param pcells (output) The end-of-timestep information we pack into
+ * Calls error() if the cell is not at the current time.
*
- * @return The number of packed cells.
+ * @param c Cell to act upon
+ * @param data The current time on the integer time-line
*/
-int cell_pack_end_step_hydro(struct cell *restrict c,
- struct pcell_step_hydro *restrict pcells) {
-#ifdef WITH_MPI
+void cell_check_spart_drift_point(struct cell *c, void *data) {
+#ifdef SWIFT_DEBUG_CHECKS
- /* Pack this cell's data. */
- pcells[0].ti_end_min = c->hydro.ti_end_min;
- pcells[0].ti_end_max = c->hydro.ti_end_max;
- pcells[0].dx_max_part = c->hydro.dx_max_part;
+ const integertime_t ti_drift = *(integertime_t *)data;
- /* Fill in the progeny, depth-first recursion. */
- int count = 1;
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL) {
- count += cell_pack_end_step_hydro(c->progeny[k], &pcells[count]);
- }
+ /* Only check local cells */
+ if (c->nodeID != engine_rank) return;
- /* Return the number of packed values. */
- return count;
+ /* Only check cells with content */
+ if (c->stars.count == 0) return;
+
+ if (c->stars.ti_old_part != ti_drift)
+ error(
+ "Cell in an incorrect time-zone! c->stars.ti_old_part=%lld "
+ "ti_drift=%lld",
+ c->stars.ti_old_part, ti_drift);
+ for (int i = 0; i < c->stars.count; ++i)
+ if (c->stars.parts[i].ti_drift != ti_drift &&
+ c->stars.parts[i].time_bin != time_bin_inhibited)
+ error("g-part in an incorrect time-zone! gp->ti_drift=%lld ti_drift=%lld",
+ c->stars.parts[i].ti_drift, ti_drift);
#else
- error("SWIFT was not compiled with MPI support.");
- return 0;
+ error("Calling debugging code without debugging flag activated.");
#endif
}
/**
- * @brief Unpack the time information of a given cell and its sub-cells.
+ * @brief Checks that the multipole of a cell is at the current point in time
*
- * @param c The #cell
- * @param pcells The end-of-timestep information to unpack
+ * Calls error() if the cell is not at the current time.
*
- * @return The number of cells created.
+ * @param c Cell to act upon
+ * @param data The current time on the integer time-line
*/
-int cell_unpack_end_step_hydro(struct cell *restrict c,
- struct pcell_step_hydro *restrict pcells) {
-#ifdef WITH_MPI
+void cell_check_multipole_drift_point(struct cell *c, void *data) {
+#ifdef SWIFT_DEBUG_CHECKS
- /* Unpack this cell's data. */
- c->hydro.ti_end_min = pcells[0].ti_end_min;
- c->hydro.ti_end_max = pcells[0].ti_end_max;
- c->hydro.dx_max_part = pcells[0].dx_max_part;
+ const integertime_t ti_drift = *(integertime_t *)data;
- /* Fill in the progeny, depth-first recursion. */
- int count = 1;
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL) {
- count += cell_unpack_end_step_hydro(c->progeny[k], &pcells[count]);
- }
+ /* Only check local cells */
+ if (c->nodeID != engine_rank) return;
- /* Return the number of packed values. */
- return count;
+ /* Only check cells with content */
+ if (c->grav.count == 0) return;
+
+ if (c->grav.ti_old_multipole != ti_drift)
+ error(
+ "Cell multipole in an incorrect time-zone! "
+ "c->grav.ti_old_multipole=%lld "
+ "ti_drift=%lld (depth=%d, node=%d)",
+ c->grav.ti_old_multipole, ti_drift, c->depth, c->nodeID);
#else
- error("SWIFT was not compiled with MPI support.");
- return 0;
+ error("Calling debugging code without debugging flag activated.");
#endif
}
/**
- * @brief Pack the time information of the given cell and all it's sub-cells.
+ * @brief Resets all the individual cell task counters to 0.
*
- * @param c The #cell.
- * @param pcells (output) The end-of-timestep information we pack into
+ * Should only be used for debugging purposes.
*
- * @return The number of packed cells.
+ * @param c The #cell to reset.
*/
-int cell_pack_end_step_grav(struct cell *restrict c,
- struct pcell_step_grav *restrict pcells) {
-#ifdef WITH_MPI
-
- /* Pack this cell's data. */
- pcells[0].ti_end_min = c->grav.ti_end_min;
- pcells[0].ti_end_max = c->grav.ti_end_max;
-
- /* Fill in the progeny, depth-first recursion. */
- int count = 1;
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL) {
- count += cell_pack_end_step_grav(c->progeny[k], &pcells[count]);
- }
-
- /* Return the number of packed values. */
- return count;
-
+void cell_reset_task_counters(struct cell *c) {
+#ifdef SWIFT_DEBUG_CHECKS
+ for (int t = 0; t < task_type_count; ++t) c->tasks_executed[t] = 0;
+ for (int t = 0; t < task_subtype_count; ++t) c->subtasks_executed[t] = 0;
+ for (int k = 0; k < 8; ++k)
+ if (c->progeny[k] != NULL) cell_reset_task_counters(c->progeny[k]);
#else
- error("SWIFT was not compiled with MPI support.");
- return 0;
+ error("Calling debugging code without debugging flag activated.");
#endif
}
/**
- * @brief Unpack the time information of a given cell and its sub-cells.
- *
- * @param c The #cell
- * @param pcells The end-of-timestep information to unpack
+ * @brief Recursively construct all the multipoles in a cell hierarchy.
*
- * @return The number of cells created.
+ * @param c The #cell.
+ * @param ti_current The current integer time.
+ * @param grav_props The properties of the gravity scheme.
*/
-int cell_unpack_end_step_grav(struct cell *restrict c,
- struct pcell_step_grav *restrict pcells) {
-#ifdef WITH_MPI
+void cell_make_multipoles(struct cell *c, integertime_t ti_current,
+ const struct gravity_props *const grav_props) {
+
+ /* Reset everything */
+ gravity_reset(c->grav.multipole);
- /* Unpack this cell's data. */
- c->grav.ti_end_min = pcells[0].ti_end_min;
- c->grav.ti_end_max = pcells[0].ti_end_max;
+ if (c->split) {
- /* Fill in the progeny, depth-first recursion. */
- int count = 1;
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL) {
- count += cell_unpack_end_step_grav(c->progeny[k], &pcells[count]);
+ /* Start by recursing */
+ for (int k = 0; k < 8; ++k) {
+ if (c->progeny[k] != NULL)
+ cell_make_multipoles(c->progeny[k], ti_current, grav_props);
}
- /* Return the number of packed values. */
- return count;
+ /* Compute CoM of all progenies */
+ double CoM[3] = {0., 0., 0.};
+ double vel[3] = {0., 0., 0.};
+ float max_delta_vel[3] = {0.f, 0.f, 0.f};
+ float min_delta_vel[3] = {0.f, 0.f, 0.f};
+ double mass = 0.;
-#else
- error("SWIFT was not compiled with MPI support.");
- return 0;
-#endif
-}
+ for (int k = 0; k < 8; ++k) {
+ if (c->progeny[k] != NULL) {
+ const struct gravity_tensors *m = c->progeny[k]->grav.multipole;
-/**
- * @brief Pack the time information of the given cell and all it's sub-cells.
- *
- * @param c The #cell.
- * @param pcells (output) The end-of-timestep information we pack into
- *
- * @return The number of packed cells.
- */
-int cell_pack_end_step_stars(struct cell *restrict c,
- struct pcell_step_stars *restrict pcells) {
-#ifdef WITH_MPI
+ mass += m->m_pole.M_000;
- /* Pack this cell's data. */
- pcells[0].ti_end_min = c->stars.ti_end_min;
- pcells[0].ti_end_max = c->stars.ti_end_max;
- pcells[0].dx_max_part = c->stars.dx_max_part;
+ CoM[0] += m->CoM[0] * m->m_pole.M_000;
+ CoM[1] += m->CoM[1] * m->m_pole.M_000;
+ CoM[2] += m->CoM[2] * m->m_pole.M_000;
- /* Fill in the progeny, depth-first recursion. */
- int count = 1;
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL) {
- count += cell_pack_end_step_stars(c->progeny[k], &pcells[count]);
+ vel[0] += m->m_pole.vel[0] * m->m_pole.M_000;
+ vel[1] += m->m_pole.vel[1] * m->m_pole.M_000;
+ vel[2] += m->m_pole.vel[2] * m->m_pole.M_000;
+
+ max_delta_vel[0] = max(m->m_pole.max_delta_vel[0], max_delta_vel[0]);
+ max_delta_vel[1] = max(m->m_pole.max_delta_vel[1], max_delta_vel[1]);
+ max_delta_vel[2] = max(m->m_pole.max_delta_vel[2], max_delta_vel[2]);
+
+ min_delta_vel[0] = min(m->m_pole.min_delta_vel[0], min_delta_vel[0]);
+ min_delta_vel[1] = min(m->m_pole.min_delta_vel[1], min_delta_vel[1]);
+ min_delta_vel[2] = min(m->m_pole.min_delta_vel[2], min_delta_vel[2]);
+ }
}
- /* Return the number of packed values. */
- return count;
+ /* Final operation on the CoM and bulk velocity */
+ const double mass_inv = 1. / mass;
+ c->grav.multipole->CoM[0] = CoM[0] * mass_inv;
+ c->grav.multipole->CoM[1] = CoM[1] * mass_inv;
+ c->grav.multipole->CoM[2] = CoM[2] * mass_inv;
+ c->grav.multipole->m_pole.vel[0] = vel[0] * mass_inv;
+ c->grav.multipole->m_pole.vel[1] = vel[1] * mass_inv;
+ c->grav.multipole->m_pole.vel[2] = vel[2] * mass_inv;
-#else
- error("SWIFT was not compiled with MPI support.");
- return 0;
-#endif
-}
+ /* Min max velocity along each axis */
+ c->grav.multipole->m_pole.max_delta_vel[0] = max_delta_vel[0];
+ c->grav.multipole->m_pole.max_delta_vel[1] = max_delta_vel[1];
+ c->grav.multipole->m_pole.max_delta_vel[2] = max_delta_vel[2];
+ c->grav.multipole->m_pole.min_delta_vel[0] = min_delta_vel[0];
+ c->grav.multipole->m_pole.min_delta_vel[1] = min_delta_vel[1];
+ c->grav.multipole->m_pole.min_delta_vel[2] = min_delta_vel[2];
-/**
- * @brief Unpack the time information of a given cell and its sub-cells.
- *
- * @param c The #cell
- * @param pcells The end-of-timestep information to unpack
- *
- * @return The number of cells created.
- */
-int cell_unpack_end_step_stars(struct cell *restrict c,
- struct pcell_step_stars *restrict pcells) {
-#ifdef WITH_MPI
+ /* Now shift progeny multipoles and add them up */
+ struct multipole temp;
+ double r_max = 0.;
+ for (int k = 0; k < 8; ++k) {
+ if (c->progeny[k] != NULL) {
+ const struct cell *cp = c->progeny[k];
+ const struct multipole *m = &cp->grav.multipole->m_pole;
- /* Unpack this cell's data. */
- c->stars.ti_end_min = pcells[0].ti_end_min;
- c->stars.ti_end_max = pcells[0].ti_end_max;
- c->stars.dx_max_part = pcells[0].dx_max_part;
+ /* Contribution to multipole */
+ gravity_M2M(&temp, m, c->grav.multipole->CoM, cp->grav.multipole->CoM);
+ gravity_multipole_add(&c->grav.multipole->m_pole, &temp);
- /* Fill in the progeny, depth-first recursion. */
- int count = 1;
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL) {
- count += cell_unpack_end_step_stars(c->progeny[k], &pcells[count]);
+ /* Upper limit of max CoM<->gpart distance */
+ const double dx =
+ c->grav.multipole->CoM[0] - cp->grav.multipole->CoM[0];
+ const double dy =
+ c->grav.multipole->CoM[1] - cp->grav.multipole->CoM[1];
+ const double dz =
+ c->grav.multipole->CoM[2] - cp->grav.multipole->CoM[2];
+ const double r2 = dx * dx + dy * dy + dz * dz;
+ r_max = max(r_max, cp->grav.multipole->r_max + sqrt(r2));
+ }
}
+ /* Alternative upper limit of max CoM<->gpart distance */
+ const double dx = c->grav.multipole->CoM[0] > c->loc[0] + c->width[0] * 0.5
+ ? c->grav.multipole->CoM[0] - c->loc[0]
+ : c->loc[0] + c->width[0] - c->grav.multipole->CoM[0];
+ const double dy = c->grav.multipole->CoM[1] > c->loc[1] + c->width[1] * 0.5
+ ? c->grav.multipole->CoM[1] - c->loc[1]
+ : c->loc[1] + c->width[1] - c->grav.multipole->CoM[1];
+ const double dz = c->grav.multipole->CoM[2] > c->loc[2] + c->width[2] * 0.5
+ ? c->grav.multipole->CoM[2] - c->loc[2]
+ : c->loc[2] + c->width[2] - c->grav.multipole->CoM[2];
- /* Return the number of packed values. */
- return count;
+ /* Take minimum of both limits */
+ c->grav.multipole->r_max = min(r_max, sqrt(dx * dx + dy * dy + dz * dz));
-#else
- error("SWIFT was not compiled with MPI support.");
- return 0;
-#endif
-}
+ /* Compute the multipole power */
+ gravity_multipole_compute_power(&c->grav.multipole->m_pole);
-/**
- * @brief Pack the time information of the given cell and all it's sub-cells.
- *
- * @param c The #cell.
- * @param pcells (output) The end-of-timestep information we pack into
- *
- * @return The number of packed cells.
- */
-int cell_pack_end_step_black_holes(
- struct cell *restrict c, struct pcell_step_black_holes *restrict pcells) {
+ } else {
+ if (c->grav.count > 0) {
-#ifdef WITH_MPI
+ gravity_P2M(c->grav.multipole, c->grav.parts, c->grav.count, grav_props);
+
+ /* Compute the multipole power */
+ gravity_multipole_compute_power(&c->grav.multipole->m_pole);
- /* Pack this cell's data. */
- pcells[0].ti_end_min = c->black_holes.ti_end_min;
- pcells[0].ti_end_max = c->black_holes.ti_end_max;
- pcells[0].dx_max_part = c->black_holes.dx_max_part;
+ } else {
- /* Fill in the progeny, depth-first recursion. */
- int count = 1;
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL) {
- count += cell_pack_end_step_black_holes(c->progeny[k], &pcells[count]);
+ /* No gparts in that leaf cell */
+
+ /* Set the values to something sensible */
+ gravity_multipole_init(&c->grav.multipole->m_pole);
+ c->grav.multipole->CoM[0] = c->loc[0] + c->width[0] * 0.5;
+ c->grav.multipole->CoM[1] = c->loc[1] + c->width[1] * 0.5;
+ c->grav.multipole->CoM[2] = c->loc[2] + c->width[2] * 0.5;
+ c->grav.multipole->r_max = 0.;
}
+ }
- /* Return the number of packed values. */
- return count;
+ /* Also update the values at rebuild time */
+ c->grav.multipole->r_max_rebuild = c->grav.multipole->r_max;
+ c->grav.multipole->CoM_rebuild[0] = c->grav.multipole->CoM[0];
+ c->grav.multipole->CoM_rebuild[1] = c->grav.multipole->CoM[1];
+ c->grav.multipole->CoM_rebuild[2] = c->grav.multipole->CoM[2];
-#else
- error("SWIFT was not compiled with MPI support.");
- return 0;
-#endif
+ c->grav.ti_old_multipole = ti_current;
}
/**
- * @brief Unpack the time information of a given cell and its sub-cells.
+ * @brief Recursively verify that the multipoles are the sum of their progenies.
*
- * @param c The #cell
- * @param pcells The end-of-timestep information to unpack
+ * This function does not check whether the multipoles match the particle
+ * content as we may not have received the particles.
*
- * @return The number of cells created.
+ * @param c The #cell to recursively search and verify.
*/
-int cell_unpack_end_step_black_holes(
- struct cell *restrict c, struct pcell_step_black_holes *restrict pcells) {
+void cell_check_foreign_multipole(const struct cell *c) {
+#ifdef SWIFT_DEBUG_CHECKS
-#ifdef WITH_MPI
+ if (c->split) {
+ double M_000 = 0.;
+ long long num_gpart = 0;
- /* Unpack this cell's data. */
- c->black_holes.ti_end_min = pcells[0].ti_end_min;
- c->black_holes.ti_end_max = pcells[0].ti_end_max;
- c->black_holes.dx_max_part = pcells[0].dx_max_part;
+ for (int k = 0; k < 8; k++) {
+ const struct cell *cp = c->progeny[k];
- /* Fill in the progeny, depth-first recursion. */
- int count = 1;
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL) {
- count += cell_unpack_end_step_black_holes(c->progeny[k], &pcells[count]);
+ if (cp != NULL) {
+ /* Check the mass */
+ M_000 += cp->grav.multipole->m_pole.M_000;
+
+ /* Check the number of particles */
+ num_gpart += cp->grav.multipole->m_pole.num_gpart;
+
+ /* Now recurse */
+ cell_check_foreign_multipole(cp);
+ }
}
- /* Return the number of packed values. */
- return count;
+ if (num_gpart != c->grav.multipole->m_pole.num_gpart)
+ error("Sum of particles in progenies does not match");
+ }
#else
- error("SWIFT was not compiled with MPI support.");
- return 0;
+ error("Calling debugging code without debugging flag activated.");
#endif
}
/**
- * @brief Pack the multipole information of the given cell and all it's
- * sub-cells.
- *
- * @param c The #cell.
- * @param pcells (output) The multipole information we pack into
+ * @brief Computes the multi-pole brutally and compare to the
+ * recursively computed one.
*
- * @return The number of packed cells.
+ * @param c Cell to act upon
+ * @param grav_props The properties of the gravity scheme.
*/
-int cell_pack_multipoles(struct cell *restrict c,
- struct gravity_tensors *restrict pcells) {
-#ifdef WITH_MPI
+void cell_check_multipole(struct cell *c,
+ const struct gravity_props *const grav_props) {
- /* Pack this cell's data. */
- pcells[0] = *c->grav.multipole;
+#ifdef SWIFT_DEBUG_CHECKS
+ struct gravity_tensors ma;
+ const double tolerance = 1e-3; /* Relative */
- /* Fill in the progeny, depth-first recursion. */
- int count = 1;
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL) {
- count += cell_pack_multipoles(c->progeny[k], &pcells[count]);
- }
+ /* First recurse */
+ if (c->split)
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL)
+ cell_check_multipole(c->progeny[k], grav_props);
- /* Return the number of packed values. */
- return count;
+ if (c->grav.count > 0) {
+ /* Brute-force calculation */
+ gravity_P2M(&ma, c->grav.parts, c->grav.count, grav_props);
+ gravity_multipole_compute_power(&ma.m_pole);
+
+ /* Now compare the multipole expansion */
+ if (!gravity_multipole_equal(&ma, c->grav.multipole, tolerance)) {
+ message("Multipoles are not equal at depth=%d! tol=%f", c->depth,
+ tolerance);
+ message("Correct answer:");
+ gravity_multipole_print(&ma.m_pole);
+ message("Recursive multipole:");
+ gravity_multipole_print(&c->grav.multipole->m_pole);
+ error("Aborting");
+ }
+ /* Check that the upper limit of r_max is good enough */
+ if (!(1.1 * c->grav.multipole->r_max >= ma.r_max)) {
+ error("Upper-limit r_max=%e too small. Should be >=%e.",
+ c->grav.multipole->r_max, ma.r_max);
+ } else if (c->grav.multipole->r_max * c->grav.multipole->r_max >
+ 3. * c->width[0] * c->width[0]) {
+ error("r_max=%e larger than cell diagonal %e.", c->grav.multipole->r_max,
+ sqrt(3. * c->width[0] * c->width[0]));
+ }
+ }
#else
- error("SWIFT was not compiled with MPI support.");
- return 0;
+ error("Calling debugging code without debugging flag activated.");
#endif
}
/**
- * @brief Unpack the multipole information of a given cell and its sub-cells.
- *
- * @param c The #cell
- * @param pcells The multipole information to unpack
+ * @brief Frees up the memory allocated for this #cell.
*
- * @return The number of cells created.
+ * @param c The #cell.
*/
-int cell_unpack_multipoles(struct cell *restrict c,
- struct gravity_tensors *restrict pcells) {
-#ifdef WITH_MPI
+void cell_clean(struct cell *c) {
+ /* Hydro */
+ cell_free_hydro_sorts(c);
- /* Unpack this cell's data. */
- *c->grav.multipole = pcells[0];
+ /* Stars */
+ cell_free_stars_sorts(c);
- /* Fill in the progeny, depth-first recursion. */
- int count = 1;
+ /* Recurse */
for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL) {
- count += cell_unpack_multipoles(c->progeny[k], &pcells[count]);
- }
-
- /* Return the number of packed values. */
- return count;
-
-#else
- error("SWIFT was not compiled with MPI support.");
- return 0;
-#endif
+ if (c->progeny[k]) cell_clean(c->progeny[k]);
}
/**
- * @brief Pack the counts for star formation of the given cell and all it's
- * sub-cells.
- *
- * @param c The #cell.
- * @param pcells (output) The multipole information we pack into
- *
- * @return The number of packed cells.
- */
-int cell_pack_sf_counts(struct cell *restrict c,
- struct pcell_sf *restrict pcells) {
-
-#ifdef WITH_MPI
-
- /* Pack this cell's data. */
- pcells[0].stars.delta_from_rebuild = c->stars.parts - c->stars.parts_rebuild;
- pcells[0].stars.count = c->stars.count;
- pcells[0].stars.dx_max_part = c->stars.dx_max_part;
-
- /* Pack this cell's data. */
- pcells[0].grav.delta_from_rebuild = c->grav.parts - c->grav.parts_rebuild;
- pcells[0].grav.count = c->grav.count;
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Stars */
- if (c->stars.parts_rebuild == NULL)
- error("Star particles array at rebuild is NULL! c->depth=%d", c->depth);
-
- if (pcells[0].stars.delta_from_rebuild < 0)
- error("Stars part pointer moved in the wrong direction!");
-
- if (pcells[0].stars.delta_from_rebuild > 0 && c->depth == 0)
- error("Shifting the top-level pointer is not allowed!");
-
- /* Grav */
- if (c->grav.parts_rebuild == NULL)
- error("Grav. particles array at rebuild is NULL! c->depth=%d", c->depth);
-
- if (pcells[0].grav.delta_from_rebuild < 0)
- error("Grav part pointer moved in the wrong direction!");
-
- if (pcells[0].grav.delta_from_rebuild > 0 && c->depth == 0)
- error("Shifting the top-level pointer is not allowed!");
-#endif
-
- /* Fill in the progeny, depth-first recursion. */
- int count = 1;
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL) {
- count += cell_pack_sf_counts(c->progeny[k], &pcells[count]);
- }
-
- /* Return the number of packed values. */
- return count;
-
-#else
- error("SWIFT was not compiled with MPI support.");
- return 0;
-#endif
-}
-
-/**
- * @brief Unpack the counts for star formation of a given cell and its
- * sub-cells.
- *
- * @param c The #cell
- * @param pcells The multipole information to unpack
- *
- * @return The number of cells created.
- */
-int cell_unpack_sf_counts(struct cell *restrict c,
- struct pcell_sf *restrict pcells) {
-
-#ifdef WITH_MPI
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (c->stars.parts_rebuild == NULL)
- error("Star particles array at rebuild is NULL!");
- if (c->grav.parts_rebuild == NULL)
- error("Grav particles array at rebuild is NULL!");
-#endif
-
- /* Unpack this cell's data. */
- c->stars.count = pcells[0].stars.count;
- c->stars.parts = c->stars.parts_rebuild + pcells[0].stars.delta_from_rebuild;
- c->stars.dx_max_part = pcells[0].stars.dx_max_part;
-
- c->grav.count = pcells[0].grav.count;
- c->grav.parts = c->grav.parts_rebuild + pcells[0].grav.delta_from_rebuild;
-
- /* Fill in the progeny, depth-first recursion. */
- int count = 1;
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL) {
- count += cell_unpack_sf_counts(c->progeny[k], &pcells[count]);
- }
-
- /* Return the number of packed values. */
- return count;
-
-#else
- error("SWIFT was not compiled with MPI support.");
- return 0;
-#endif
-}
-
-/**
- * @brief Lock a cell for access to its array of #part and hold its parents.
- *
- * @param c The #cell.
- * @return 0 on success, 1 on failure
- */
-int cell_locktree(struct cell *c) {
- TIMER_TIC;
-
- /* First of all, try to lock this cell. */
- if (c->hydro.hold || lock_trylock(&c->hydro.lock) != 0) {
- TIMER_TOC(timer_locktree);
- return 1;
- }
-
- /* Did somebody hold this cell in the meantime? */
- if (c->hydro.hold) {
- /* Unlock this cell. */
- if (lock_unlock(&c->hydro.lock) != 0) error("Failed to unlock cell.");
-
- /* Admit defeat. */
- TIMER_TOC(timer_locktree);
- return 1;
- }
-
- /* Climb up the tree and lock/hold/unlock. */
- struct cell *finger;
- for (finger = c->parent; finger != NULL; finger = finger->parent) {
- /* Lock this cell. */
- if (lock_trylock(&finger->hydro.lock) != 0) break;
-
- /* Increment the hold. */
- atomic_inc(&finger->hydro.hold);
-
- /* Unlock the cell. */
- if (lock_unlock(&finger->hydro.lock) != 0) error("Failed to unlock cell.");
- }
-
- /* If we reached the top of the tree, we're done. */
- if (finger == NULL) {
- TIMER_TOC(timer_locktree);
- return 0;
- }
-
- /* Otherwise, we hit a snag. */
- else {
- /* Undo the holds up to finger. */
- for (struct cell *finger2 = c->parent; finger2 != finger;
- finger2 = finger2->parent)
- atomic_dec(&finger2->hydro.hold);
-
- /* Unlock this cell. */
- if (lock_unlock(&c->hydro.lock) != 0) error("Failed to unlock cell.");
-
- /* Admit defeat. */
- TIMER_TOC(timer_locktree);
- return 1;
- }
-}
-
-/**
- * @brief Lock a cell for access to its array of #gpart and hold its parents.
- *
- * @param c The #cell.
- * @return 0 on success, 1 on failure
- */
-int cell_glocktree(struct cell *c) {
- TIMER_TIC;
-
- /* First of all, try to lock this cell. */
- if (c->grav.phold || lock_trylock(&c->grav.plock) != 0) {
- TIMER_TOC(timer_locktree);
- return 1;
- }
-
- /* Did somebody hold this cell in the meantime? */
- if (c->grav.phold) {
- /* Unlock this cell. */
- if (lock_unlock(&c->grav.plock) != 0) error("Failed to unlock cell.");
-
- /* Admit defeat. */
- TIMER_TOC(timer_locktree);
- return 1;
- }
-
- /* Climb up the tree and lock/hold/unlock. */
- struct cell *finger;
- for (finger = c->parent; finger != NULL; finger = finger->parent) {
- /* Lock this cell. */
- if (lock_trylock(&finger->grav.plock) != 0) break;
-
- /* Increment the hold. */
- atomic_inc(&finger->grav.phold);
-
- /* Unlock the cell. */
- if (lock_unlock(&finger->grav.plock) != 0) error("Failed to unlock cell.");
- }
-
- /* If we reached the top of the tree, we're done. */
- if (finger == NULL) {
- TIMER_TOC(timer_locktree);
- return 0;
- }
-
- /* Otherwise, we hit a snag. */
- else {
- /* Undo the holds up to finger. */
- for (struct cell *finger2 = c->parent; finger2 != finger;
- finger2 = finger2->parent)
- atomic_dec(&finger2->grav.phold);
-
- /* Unlock this cell. */
- if (lock_unlock(&c->grav.plock) != 0) error("Failed to unlock cell.");
-
- /* Admit defeat. */
- TIMER_TOC(timer_locktree);
- return 1;
- }
-}
-
-/**
- * @brief Lock a cell for access to its #multipole and hold its parents.
- *
- * @param c The #cell.
- * @return 0 on success, 1 on failure
- */
-int cell_mlocktree(struct cell *c) {
- TIMER_TIC;
-
- /* First of all, try to lock this cell. */
- if (c->grav.mhold || lock_trylock(&c->grav.mlock) != 0) {
- TIMER_TOC(timer_locktree);
- return 1;
- }
-
- /* Did somebody hold this cell in the meantime? */
- if (c->grav.mhold) {
- /* Unlock this cell. */
- if (lock_unlock(&c->grav.mlock) != 0) error("Failed to unlock cell.");
-
- /* Admit defeat. */
- TIMER_TOC(timer_locktree);
- return 1;
- }
-
- /* Climb up the tree and lock/hold/unlock. */
- struct cell *finger;
- for (finger = c->parent; finger != NULL; finger = finger->parent) {
- /* Lock this cell. */
- if (lock_trylock(&finger->grav.mlock) != 0) break;
-
- /* Increment the hold. */
- atomic_inc(&finger->grav.mhold);
-
- /* Unlock the cell. */
- if (lock_unlock(&finger->grav.mlock) != 0) error("Failed to unlock cell.");
- }
-
- /* If we reached the top of the tree, we're done. */
- if (finger == NULL) {
- TIMER_TOC(timer_locktree);
- return 0;
- }
-
- /* Otherwise, we hit a snag. */
- else {
- /* Undo the holds up to finger. */
- for (struct cell *finger2 = c->parent; finger2 != finger;
- finger2 = finger2->parent)
- atomic_dec(&finger2->grav.mhold);
-
- /* Unlock this cell. */
- if (lock_unlock(&c->grav.mlock) != 0) error("Failed to unlock cell.");
-
- /* Admit defeat. */
- TIMER_TOC(timer_locktree);
- return 1;
- }
-}
-
-/**
- * @brief Lock a cell for access to its array of #spart and hold its parents.
- *
- * @param c The #cell.
- * @return 0 on success, 1 on failure
- */
-int cell_slocktree(struct cell *c) {
- TIMER_TIC;
-
- /* First of all, try to lock this cell. */
- if (c->stars.hold || lock_trylock(&c->stars.lock) != 0) {
- TIMER_TOC(timer_locktree);
- return 1;
- }
-
- /* Did somebody hold this cell in the meantime? */
- if (c->stars.hold) {
- /* Unlock this cell. */
- if (lock_unlock(&c->stars.lock) != 0) error("Failed to unlock cell.");
-
- /* Admit defeat. */
- TIMER_TOC(timer_locktree);
- return 1;
- }
-
- /* Climb up the tree and lock/hold/unlock. */
- struct cell *finger;
- for (finger = c->parent; finger != NULL; finger = finger->parent) {
- /* Lock this cell. */
- if (lock_trylock(&finger->stars.lock) != 0) break;
-
- /* Increment the hold. */
- atomic_inc(&finger->stars.hold);
-
- /* Unlock the cell. */
- if (lock_unlock(&finger->stars.lock) != 0) error("Failed to unlock cell.");
- }
-
- /* If we reached the top of the tree, we're done. */
- if (finger == NULL) {
- TIMER_TOC(timer_locktree);
- return 0;
- }
-
- /* Otherwise, we hit a snag. */
- else {
- /* Undo the holds up to finger. */
- for (struct cell *finger2 = c->parent; finger2 != finger;
- finger2 = finger2->parent)
- atomic_dec(&finger2->stars.hold);
-
- /* Unlock this cell. */
- if (lock_unlock(&c->stars.lock) != 0) error("Failed to unlock cell.");
-
- /* Admit defeat. */
- TIMER_TOC(timer_locktree);
- return 1;
- }
-}
-
-/**
- * @brief Lock a cell for access to its array of #sink and hold its parents.
- *
- * @param c The #cell.
- * @return 0 on success, 1 on failure
- */
-int cell_sink_locktree(struct cell *c) {
- TIMER_TIC;
-
- /* First of all, try to lock this cell. */
- if (c->sinks.hold || lock_trylock(&c->sinks.lock) != 0) {
- TIMER_TOC(timer_locktree);
- return 1;
- }
-
- /* Did somebody hold this cell in the meantime? */
- if (c->sinks.hold) {
- /* Unlock this cell. */
- if (lock_unlock(&c->sinks.lock) != 0) error("Failed to unlock cell.");
-
- /* Admit defeat. */
- TIMER_TOC(timer_locktree);
- return 1;
- }
-
- /* Climb up the tree and lock/hold/unlock. */
- struct cell *finger;
- for (finger = c->parent; finger != NULL; finger = finger->parent) {
- /* Lock this cell. */
- if (lock_trylock(&finger->sinks.lock) != 0) break;
-
- /* Increment the hold. */
- atomic_inc(&finger->sinks.hold);
-
- /* Unlock the cell. */
- if (lock_unlock(&finger->sinks.lock) != 0) error("Failed to unlock cell.");
- }
-
- /* If we reached the top of the tree, we're done. */
- if (finger == NULL) {
- TIMER_TOC(timer_locktree);
- return 0;
- }
-
- /* Otherwise, we hit a snag. */
- else {
- /* Undo the holds up to finger. */
- for (struct cell *finger2 = c->parent; finger2 != finger;
- finger2 = finger2->parent)
- atomic_dec(&finger2->sinks.hold);
-
- /* Unlock this cell. */
- if (lock_unlock(&c->sinks.lock) != 0) error("Failed to unlock cell.");
-
- /* Admit defeat. */
- TIMER_TOC(timer_locktree);
- return 1;
- }
-}
-
-/**
- * @brief Lock a cell for access to its array of #bpart and hold its parents.
- *
- * @param c The #cell.
- * @return 0 on success, 1 on failure
- */
-int cell_blocktree(struct cell *c) {
- TIMER_TIC;
-
- /* First of all, try to lock this cell. */
- if (c->black_holes.hold || lock_trylock(&c->black_holes.lock) != 0) {
- TIMER_TOC(timer_locktree);
- return 1;
- }
-
- /* Did somebody hold this cell in the meantime? */
- if (c->black_holes.hold) {
- /* Unlock this cell. */
- if (lock_unlock(&c->black_holes.lock) != 0) error("Failed to unlock cell.");
-
- /* Admit defeat. */
- TIMER_TOC(timer_locktree);
- return 1;
- }
-
- /* Climb up the tree and lock/hold/unlock. */
- struct cell *finger;
- for (finger = c->parent; finger != NULL; finger = finger->parent) {
- /* Lock this cell. */
- if (lock_trylock(&finger->black_holes.lock) != 0) break;
-
- /* Increment the hold. */
- atomic_inc(&finger->black_holes.hold);
-
- /* Unlock the cell. */
- if (lock_unlock(&finger->black_holes.lock) != 0)
- error("Failed to unlock cell.");
- }
-
- /* If we reached the top of the tree, we're done. */
- if (finger == NULL) {
- TIMER_TOC(timer_locktree);
- return 0;
- }
-
- /* Otherwise, we hit a snag. */
- else {
- /* Undo the holds up to finger. */
- for (struct cell *finger2 = c->parent; finger2 != finger;
- finger2 = finger2->parent)
- atomic_dec(&finger2->black_holes.hold);
-
- /* Unlock this cell. */
- if (lock_unlock(&c->black_holes.lock) != 0) error("Failed to unlock cell.");
-
- /* Admit defeat. */
- TIMER_TOC(timer_locktree);
- return 1;
- }
-}
-
-/**
- * @brief Unlock a cell's parents for access to #part array.
- *
- * @param c The #cell.
- */
-void cell_unlocktree(struct cell *c) {
- TIMER_TIC;
-
- /* First of all, try to unlock this cell. */
- if (lock_unlock(&c->hydro.lock) != 0) error("Failed to unlock cell.");
-
- /* Climb up the tree and unhold the parents. */
- for (struct cell *finger = c->parent; finger != NULL; finger = finger->parent)
- atomic_dec(&finger->hydro.hold);
-
- TIMER_TOC(timer_locktree);
-}
-
-/**
- * @brief Unlock a cell's parents for access to #gpart array.
- *
- * @param c The #cell.
- */
-void cell_gunlocktree(struct cell *c) {
- TIMER_TIC;
-
- /* First of all, try to unlock this cell. */
- if (lock_unlock(&c->grav.plock) != 0) error("Failed to unlock cell.");
-
- /* Climb up the tree and unhold the parents. */
- for (struct cell *finger = c->parent; finger != NULL; finger = finger->parent)
- atomic_dec(&finger->grav.phold);
-
- TIMER_TOC(timer_locktree);
-}
-
-/**
- * @brief Unlock a cell's parents for access to its #multipole.
- *
- * @param c The #cell.
- */
-void cell_munlocktree(struct cell *c) {
- TIMER_TIC;
-
- /* First of all, try to unlock this cell. */
- if (lock_unlock(&c->grav.mlock) != 0) error("Failed to unlock cell.");
-
- /* Climb up the tree and unhold the parents. */
- for (struct cell *finger = c->parent; finger != NULL; finger = finger->parent)
- atomic_dec(&finger->grav.mhold);
-
- TIMER_TOC(timer_locktree);
-}
-
-/**
- * @brief Unlock a cell's parents for access to #spart array.
- *
- * @param c The #cell.
- */
-void cell_sunlocktree(struct cell *c) {
- TIMER_TIC;
-
- /* First of all, try to unlock this cell. */
- if (lock_unlock(&c->stars.lock) != 0) error("Failed to unlock cell.");
-
- /* Climb up the tree and unhold the parents. */
- for (struct cell *finger = c->parent; finger != NULL; finger = finger->parent)
- atomic_dec(&finger->stars.hold);
-
- TIMER_TOC(timer_locktree);
-}
-
-/**
- * @brief Unlock a cell's parents for access to #sink array.
- *
- * @param c The #cell.
- */
-void cell_sink_unlocktree(struct cell *c) {
- TIMER_TIC;
-
- /* First of all, try to unlock this cell. */
- if (lock_unlock(&c->sinks.lock) != 0) error("Failed to unlock cell.");
-
- /* Climb up the tree and unhold the parents. */
- for (struct cell *finger = c->parent; finger != NULL; finger = finger->parent)
- atomic_dec(&finger->sinks.hold);
-
- TIMER_TOC(timer_locktree);
-}
-
-/**
- * @brief Unlock a cell's parents for access to #bpart array.
- *
- * @param c The #cell.
- */
-void cell_bunlocktree(struct cell *c) {
- TIMER_TIC;
-
- /* First of all, try to unlock this cell. */
- if (lock_unlock(&c->black_holes.lock) != 0) error("Failed to unlock cell.");
-
- /* Climb up the tree and unhold the parents. */
- for (struct cell *finger = c->parent; finger != NULL; finger = finger->parent)
- atomic_dec(&finger->black_holes.hold);
-
- TIMER_TOC(timer_locktree);
-}
-
-/**
- * @brief Sort the parts into eight bins along the given pivots.
- *
- * @param c The #cell array to be sorted.
- * @param parts_offset Offset of the cell parts array relative to the
- * space's parts array, i.e. c->hydro.parts - s->parts.
- * @param sparts_offset Offset of the cell sparts array relative to the
- * space's sparts array, i.e. c->stars.parts - s->stars.parts.
- * @param bparts_offset Offset of the cell bparts array relative to the
- * space's bparts array, i.e. c->black_holes.parts -
- * s->black_holes.parts.
- * @param sinks_offset Offset of the cell sink array relative to the
- * space's sink array, i.e. c->sinks.parts - s->sinks.parts.
- * @param buff A buffer with at least max(c->hydro.count, c->grav.count)
- * entries, used for sorting indices.
- * @param sbuff A buffer with at least max(c->stars.count, c->grav.count)
- * entries, used for sorting indices for the sparts.
- * @param bbuff A buffer with at least max(c->black_holes.count, c->grav.count)
- * entries, used for sorting indices for the sparts.
- * @param gbuff A buffer with at least max(c->hydro.count, c->grav.count)
- * entries, used for sorting indices for the gparts.
- * @param sinkbuff A buffer with at least max(c->sinks.count, c->grav.count)
- * entries, used for sorting indices for the sinks.
- */
-void cell_split(struct cell *c, const ptrdiff_t parts_offset,
- const ptrdiff_t sparts_offset, const ptrdiff_t bparts_offset,
- const ptrdiff_t sinks_offset, struct cell_buff *restrict buff,
- struct cell_buff *restrict sbuff,
- struct cell_buff *restrict bbuff,
- struct cell_buff *restrict gbuff,
- struct cell_buff *restrict sinkbuff) {
-
- const int count = c->hydro.count, gcount = c->grav.count,
- scount = c->stars.count, bcount = c->black_holes.count,
- sink_count = c->sinks.count;
- struct part *parts = c->hydro.parts;
- struct xpart *xparts = c->hydro.xparts;
- struct gpart *gparts = c->grav.parts;
- struct spart *sparts = c->stars.parts;
- struct bpart *bparts = c->black_holes.parts;
- struct sink *sinks = c->sinks.parts;
- const double pivot[3] = {c->loc[0] + c->width[0] / 2,
- c->loc[1] + c->width[1] / 2,
- c->loc[2] + c->width[2] / 2};
- int bucket_count[8] = {0, 0, 0, 0, 0, 0, 0, 0};
- int bucket_offset[9];
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Check that the buffs are OK. */
- for (int k = 0; k < count; k++) {
- if (buff[k].x[0] != parts[k].x[0] || buff[k].x[1] != parts[k].x[1] ||
- buff[k].x[2] != parts[k].x[2])
- error("Inconsistent buff contents.");
- }
- for (int k = 0; k < gcount; k++) {
- if (gbuff[k].x[0] != gparts[k].x[0] || gbuff[k].x[1] != gparts[k].x[1] ||
- gbuff[k].x[2] != gparts[k].x[2])
- error("Inconsistent gbuff contents.");
- }
- for (int k = 0; k < scount; k++) {
- if (sbuff[k].x[0] != sparts[k].x[0] || sbuff[k].x[1] != sparts[k].x[1] ||
- sbuff[k].x[2] != sparts[k].x[2])
- error("Inconsistent sbuff contents.");
- }
- for (int k = 0; k < bcount; k++) {
- if (bbuff[k].x[0] != bparts[k].x[0] || bbuff[k].x[1] != bparts[k].x[1] ||
- bbuff[k].x[2] != bparts[k].x[2])
- error("Inconsistent bbuff contents.");
- }
- for (int k = 0; k < sink_count; k++) {
- if (sinkbuff[k].x[0] != sinks[k].x[0] ||
- sinkbuff[k].x[1] != sinks[k].x[1] || sinkbuff[k].x[2] != sinks[k].x[2])
- error("Inconsistent sinkbuff contents.");
- }
-#endif /* SWIFT_DEBUG_CHECKS */
-
- /* Fill the buffer with the indices. */
- for (int k = 0; k < count; k++) {
- const int bid = (buff[k].x[0] >= pivot[0]) * 4 +
- (buff[k].x[1] >= pivot[1]) * 2 + (buff[k].x[2] >= pivot[2]);
- bucket_count[bid]++;
- buff[k].ind = bid;
- }
-
- /* Set the buffer offsets. */
- bucket_offset[0] = 0;
- for (int k = 1; k <= 8; k++) {
- bucket_offset[k] = bucket_offset[k - 1] + bucket_count[k - 1];
- bucket_count[k - 1] = 0;
- }
-
- /* Run through the buckets, and swap particles to their correct spot. */
- for (int bucket = 0; bucket < 8; bucket++) {
- for (int k = bucket_offset[bucket] + bucket_count[bucket];
- k < bucket_offset[bucket + 1]; k++) {
- int bid = buff[k].ind;
- if (bid != bucket) {
- struct part part = parts[k];
- struct xpart xpart = xparts[k];
- struct cell_buff temp_buff = buff[k];
- while (bid != bucket) {
- int j = bucket_offset[bid] + bucket_count[bid]++;
- while (buff[j].ind == bid) {
- j++;
- bucket_count[bid]++;
- }
- memswap(&parts[j], &part, sizeof(struct part));
- memswap(&xparts[j], &xpart, sizeof(struct xpart));
- memswap(&buff[j], &temp_buff, sizeof(struct cell_buff));
- if (parts[j].gpart)
- parts[j].gpart->id_or_neg_offset = -(j + parts_offset);
- bid = temp_buff.ind;
- }
- parts[k] = part;
- xparts[k] = xpart;
- buff[k] = temp_buff;
- if (parts[k].gpart)
- parts[k].gpart->id_or_neg_offset = -(k + parts_offset);
- }
- bucket_count[bid]++;
- }
- }
-
- /* Store the counts and offsets. */
- for (int k = 0; k < 8; k++) {
- c->progeny[k]->hydro.count = bucket_count[k];
- c->progeny[k]->hydro.count_total = c->progeny[k]->hydro.count;
- c->progeny[k]->hydro.parts = &c->hydro.parts[bucket_offset[k]];
- c->progeny[k]->hydro.xparts = &c->hydro.xparts[bucket_offset[k]];
- }
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Check that the buffs are OK. */
- for (int k = 1; k < count; k++) {
- if (buff[k].ind < buff[k - 1].ind) error("Buff not sorted.");
- if (buff[k].x[0] != parts[k].x[0] || buff[k].x[1] != parts[k].x[1] ||
- buff[k].x[2] != parts[k].x[2])
- error("Inconsistent buff contents (k=%i).", k);
- }
-
- /* Verify that _all_ the parts have been assigned to a cell. */
- for (int k = 1; k < 8; k++)
- if (&c->progeny[k - 1]->hydro.parts[c->progeny[k - 1]->hydro.count] !=
- c->progeny[k]->hydro.parts)
- error("Particle sorting failed (internal consistency).");
- if (c->progeny[0]->hydro.parts != c->hydro.parts)
- error("Particle sorting failed (left edge).");
- if (&c->progeny[7]->hydro.parts[c->progeny[7]->hydro.count] !=
- &c->hydro.parts[count])
- error("Particle sorting failed (right edge).");
-
- /* Verify a few sub-cells. */
- for (int k = 0; k < c->progeny[0]->hydro.count; k++)
- if (c->progeny[0]->hydro.parts[k].x[0] >= pivot[0] ||
- c->progeny[0]->hydro.parts[k].x[1] >= pivot[1] ||
- c->progeny[0]->hydro.parts[k].x[2] >= pivot[2])
- error("Sorting failed (progeny=0).");
- for (int k = 0; k < c->progeny[1]->hydro.count; k++)
- if (c->progeny[1]->hydro.parts[k].x[0] >= pivot[0] ||
- c->progeny[1]->hydro.parts[k].x[1] >= pivot[1] ||
- c->progeny[1]->hydro.parts[k].x[2] < pivot[2])
- error("Sorting failed (progeny=1).");
- for (int k = 0; k < c->progeny[2]->hydro.count; k++)
- if (c->progeny[2]->hydro.parts[k].x[0] >= pivot[0] ||
- c->progeny[2]->hydro.parts[k].x[1] < pivot[1] ||
- c->progeny[2]->hydro.parts[k].x[2] >= pivot[2])
- error("Sorting failed (progeny=2).");
- for (int k = 0; k < c->progeny[3]->hydro.count; k++)
- if (c->progeny[3]->hydro.parts[k].x[0] >= pivot[0] ||
- c->progeny[3]->hydro.parts[k].x[1] < pivot[1] ||
- c->progeny[3]->hydro.parts[k].x[2] < pivot[2])
- error("Sorting failed (progeny=3).");
- for (int k = 0; k < c->progeny[4]->hydro.count; k++)
- if (c->progeny[4]->hydro.parts[k].x[0] < pivot[0] ||
- c->progeny[4]->hydro.parts[k].x[1] >= pivot[1] ||
- c->progeny[4]->hydro.parts[k].x[2] >= pivot[2])
- error("Sorting failed (progeny=4).");
- for (int k = 0; k < c->progeny[5]->hydro.count; k++)
- if (c->progeny[5]->hydro.parts[k].x[0] < pivot[0] ||
- c->progeny[5]->hydro.parts[k].x[1] >= pivot[1] ||
- c->progeny[5]->hydro.parts[k].x[2] < pivot[2])
- error("Sorting failed (progeny=5).");
- for (int k = 0; k < c->progeny[6]->hydro.count; k++)
- if (c->progeny[6]->hydro.parts[k].x[0] < pivot[0] ||
- c->progeny[6]->hydro.parts[k].x[1] < pivot[1] ||
- c->progeny[6]->hydro.parts[k].x[2] >= pivot[2])
- error("Sorting failed (progeny=6).");
- for (int k = 0; k < c->progeny[7]->hydro.count; k++)
- if (c->progeny[7]->hydro.parts[k].x[0] < pivot[0] ||
- c->progeny[7]->hydro.parts[k].x[1] < pivot[1] ||
- c->progeny[7]->hydro.parts[k].x[2] < pivot[2])
- error("Sorting failed (progeny=7).");
-#endif
-
- /* Now do the same song and dance for the sparts. */
- for (int k = 0; k < 8; k++) bucket_count[k] = 0;
-
- /* Fill the buffer with the indices. */
- for (int k = 0; k < scount; k++) {
- const int bid = (sbuff[k].x[0] > pivot[0]) * 4 +
- (sbuff[k].x[1] > pivot[1]) * 2 + (sbuff[k].x[2] > pivot[2]);
- bucket_count[bid]++;
- sbuff[k].ind = bid;
- }
-
- /* Set the buffer offsets. */
- bucket_offset[0] = 0;
- for (int k = 1; k <= 8; k++) {
- bucket_offset[k] = bucket_offset[k - 1] + bucket_count[k - 1];
- bucket_count[k - 1] = 0;
- }
-
- /* Run through the buckets, and swap particles to their correct spot. */
- for (int bucket = 0; bucket < 8; bucket++) {
- for (int k = bucket_offset[bucket] + bucket_count[bucket];
- k < bucket_offset[bucket + 1]; k++) {
- int bid = sbuff[k].ind;
- if (bid != bucket) {
- struct spart spart = sparts[k];
- struct cell_buff temp_buff = sbuff[k];
- while (bid != bucket) {
- int j = bucket_offset[bid] + bucket_count[bid]++;
- while (sbuff[j].ind == bid) {
- j++;
- bucket_count[bid]++;
- }
- memswap(&sparts[j], &spart, sizeof(struct spart));
- memswap(&sbuff[j], &temp_buff, sizeof(struct cell_buff));
- if (sparts[j].gpart)
- sparts[j].gpart->id_or_neg_offset = -(j + sparts_offset);
- bid = temp_buff.ind;
- }
- sparts[k] = spart;
- sbuff[k] = temp_buff;
- if (sparts[k].gpart)
- sparts[k].gpart->id_or_neg_offset = -(k + sparts_offset);
- }
- bucket_count[bid]++;
- }
- }
-
- /* Store the counts and offsets. */
- for (int k = 0; k < 8; k++) {
- c->progeny[k]->stars.count = bucket_count[k];
- c->progeny[k]->stars.count_total = c->progeny[k]->stars.count;
- c->progeny[k]->stars.parts = &c->stars.parts[bucket_offset[k]];
- c->progeny[k]->stars.parts_rebuild = c->progeny[k]->stars.parts;
- }
-
- /* Now do the same song and dance for the bparts. */
- for (int k = 0; k < 8; k++) bucket_count[k] = 0;
-
- /* Fill the buffer with the indices. */
- for (int k = 0; k < bcount; k++) {
- const int bid = (bbuff[k].x[0] > pivot[0]) * 4 +
- (bbuff[k].x[1] > pivot[1]) * 2 + (bbuff[k].x[2] > pivot[2]);
- bucket_count[bid]++;
- bbuff[k].ind = bid;
- }
-
- /* Set the buffer offsets. */
- bucket_offset[0] = 0;
- for (int k = 1; k <= 8; k++) {
- bucket_offset[k] = bucket_offset[k - 1] + bucket_count[k - 1];
- bucket_count[k - 1] = 0;
- }
-
- /* Run through the buckets, and swap particles to their correct spot. */
- for (int bucket = 0; bucket < 8; bucket++) {
- for (int k = bucket_offset[bucket] + bucket_count[bucket];
- k < bucket_offset[bucket + 1]; k++) {
- int bid = bbuff[k].ind;
- if (bid != bucket) {
- struct bpart bpart = bparts[k];
- struct cell_buff temp_buff = bbuff[k];
- while (bid != bucket) {
- int j = bucket_offset[bid] + bucket_count[bid]++;
- while (bbuff[j].ind == bid) {
- j++;
- bucket_count[bid]++;
- }
- memswap(&bparts[j], &bpart, sizeof(struct bpart));
- memswap(&bbuff[j], &temp_buff, sizeof(struct cell_buff));
- if (bparts[j].gpart)
- bparts[j].gpart->id_or_neg_offset = -(j + bparts_offset);
- bid = temp_buff.ind;
- }
- bparts[k] = bpart;
- bbuff[k] = temp_buff;
- if (bparts[k].gpart)
- bparts[k].gpart->id_or_neg_offset = -(k + bparts_offset);
- }
- bucket_count[bid]++;
- }
- }
-
- /* Store the counts and offsets. */
- for (int k = 0; k < 8; k++) {
- c->progeny[k]->black_holes.count = bucket_count[k];
- c->progeny[k]->black_holes.count_total = c->progeny[k]->black_holes.count;
- c->progeny[k]->black_holes.parts = &c->black_holes.parts[bucket_offset[k]];
- }
-
- /* Now do the same song and dance for the sinks. */
- for (int k = 0; k < 8; k++) bucket_count[k] = 0;
-
- /* Fill the buffer with the indices. */
- for (int k = 0; k < sink_count; k++) {
- const int bid = (sinkbuff[k].x[0] > pivot[0]) * 4 +
- (sinkbuff[k].x[1] > pivot[1]) * 2 +
- (sinkbuff[k].x[2] > pivot[2]);
- bucket_count[bid]++;
- sinkbuff[k].ind = bid;
- }
-
- /* Set the buffer offsets. */
- bucket_offset[0] = 0;
- for (int k = 1; k <= 8; k++) {
- bucket_offset[k] = bucket_offset[k - 1] + bucket_count[k - 1];
- bucket_count[k - 1] = 0;
- }
-
- /* Run through the buckets, and swap particles to their correct spot. */
- for (int bucket = 0; bucket < 8; bucket++) {
- for (int k = bucket_offset[bucket] + bucket_count[bucket];
- k < bucket_offset[bucket + 1]; k++) {
- int bid = sinkbuff[k].ind;
- if (bid != bucket) {
- struct sink sink = sinks[k];
- struct cell_buff temp_buff = sinkbuff[k];
- while (bid != bucket) {
- int j = bucket_offset[bid] + bucket_count[bid]++;
- while (sinkbuff[j].ind == bid) {
- j++;
- bucket_count[bid]++;
- }
- memswap(&sinks[j], &sink, sizeof(struct sink));
- memswap(&sinkbuff[j], &temp_buff, sizeof(struct cell_buff));
- if (sinks[j].gpart)
- sinks[j].gpart->id_or_neg_offset = -(j + sinks_offset);
- bid = temp_buff.ind;
- }
- sinks[k] = sink;
- sinkbuff[k] = temp_buff;
- if (sinks[k].gpart)
- sinks[k].gpart->id_or_neg_offset = -(k + sinks_offset);
- }
- bucket_count[bid]++;
- }
- }
-
- /* Store the counts and offsets. */
- for (int k = 0; k < 8; k++) {
- c->progeny[k]->sinks.count = bucket_count[k];
- c->progeny[k]->sinks.count_total = c->progeny[k]->sinks.count;
- c->progeny[k]->sinks.parts = &c->sinks.parts[bucket_offset[k]];
- }
-
- /* Finally, do the same song and dance for the gparts. */
- for (int k = 0; k < 8; k++) bucket_count[k] = 0;
-
- /* Fill the buffer with the indices. */
- for (int k = 0; k < gcount; k++) {
- const int bid = (gbuff[k].x[0] > pivot[0]) * 4 +
- (gbuff[k].x[1] > pivot[1]) * 2 + (gbuff[k].x[2] > pivot[2]);
- bucket_count[bid]++;
- gbuff[k].ind = bid;
- }
-
- /* Set the buffer offsets. */
- bucket_offset[0] = 0;
- for (int k = 1; k <= 8; k++) {
- bucket_offset[k] = bucket_offset[k - 1] + bucket_count[k - 1];
- bucket_count[k - 1] = 0;
- }
-
- /* Run through the buckets, and swap particles to their correct spot. */
- for (int bucket = 0; bucket < 8; bucket++) {
- for (int k = bucket_offset[bucket] + bucket_count[bucket];
- k < bucket_offset[bucket + 1]; k++) {
- int bid = gbuff[k].ind;
- if (bid != bucket) {
- struct gpart gpart = gparts[k];
- struct cell_buff temp_buff = gbuff[k];
- while (bid != bucket) {
- int j = bucket_offset[bid] + bucket_count[bid]++;
- while (gbuff[j].ind == bid) {
- j++;
- bucket_count[bid]++;
- }
- memswap_unaligned(&gparts[j], &gpart, sizeof(struct gpart));
- memswap(&gbuff[j], &temp_buff, sizeof(struct cell_buff));
- if (gparts[j].type == swift_type_gas) {
- parts[-gparts[j].id_or_neg_offset - parts_offset].gpart =
- &gparts[j];
- } else if (gparts[j].type == swift_type_stars) {
- sparts[-gparts[j].id_or_neg_offset - sparts_offset].gpart =
- &gparts[j];
- } else if (gparts[j].type == swift_type_sink) {
- sinks[-gparts[j].id_or_neg_offset - sinks_offset].gpart =
- &gparts[j];
- } else if (gparts[j].type == swift_type_black_hole) {
- bparts[-gparts[j].id_or_neg_offset - bparts_offset].gpart =
- &gparts[j];
- }
- bid = temp_buff.ind;
- }
- gparts[k] = gpart;
- gbuff[k] = temp_buff;
- if (gparts[k].type == swift_type_gas) {
- parts[-gparts[k].id_or_neg_offset - parts_offset].gpart = &gparts[k];
- } else if (gparts[k].type == swift_type_stars) {
- sparts[-gparts[k].id_or_neg_offset - sparts_offset].gpart =
- &gparts[k];
- } else if (gparts[k].type == swift_type_sink) {
- sinks[-gparts[k].id_or_neg_offset - sinks_offset].gpart = &gparts[k];
- } else if (gparts[k].type == swift_type_black_hole) {
- bparts[-gparts[k].id_or_neg_offset - bparts_offset].gpart =
- &gparts[k];
- }
- }
- bucket_count[bid]++;
- }
- }
-
- /* Store the counts and offsets. */
- for (int k = 0; k < 8; k++) {
- c->progeny[k]->grav.count = bucket_count[k];
- c->progeny[k]->grav.count_total = c->progeny[k]->grav.count;
- c->progeny[k]->grav.parts = &c->grav.parts[bucket_offset[k]];
- c->progeny[k]->grav.parts_rebuild = c->progeny[k]->grav.parts;
- }
-}
-
-/**
- * @brief Sanitizes the smoothing length values of cells by setting large
- * outliers to more sensible values.
- *
- * Each cell with <1000 part will be processed. We limit h to be the size of
- * the cell and replace 0s with a good estimate.
- *
- * @param c The cell.
- * @param treated Has the cell already been sanitized at this level ?
- */
-void cell_sanitize(struct cell *c, int treated) {
- const int count = c->hydro.count;
- const int scount = c->stars.count;
- struct part *parts = c->hydro.parts;
- struct spart *sparts = c->stars.parts;
- float h_max = 0.f;
- float stars_h_max = 0.f;
-
- /* Treat cells will <1000 particles */
- if (count < 1000 && !treated) {
- /* Get an upper bound on h */
- const float upper_h_max = c->dmin / (1.2f * kernel_gamma);
-
- /* Apply it */
- for (int i = 0; i < count; ++i) {
- if (parts[i].h == 0.f || parts[i].h > upper_h_max)
- parts[i].h = upper_h_max;
- }
- for (int i = 0; i < scount; ++i) {
- if (sparts[i].h == 0.f || sparts[i].h > upper_h_max)
- sparts[i].h = upper_h_max;
- }
- }
-
- /* Recurse and gather the new h_max values */
- if (c->split) {
- for (int k = 0; k < 8; ++k) {
- if (c->progeny[k] != NULL) {
- /* Recurse */
- cell_sanitize(c->progeny[k], (count < 1000));
-
- /* And collect */
- h_max = max(h_max, c->progeny[k]->hydro.h_max);
- stars_h_max = max(stars_h_max, c->progeny[k]->stars.h_max);
- }
- }
- } else {
- /* Get the new value of h_max */
- for (int i = 0; i < count; ++i) h_max = max(h_max, parts[i].h);
- for (int i = 0; i < scount; ++i)
- stars_h_max = max(stars_h_max, sparts[i].h);
- }
-
- /* Record the change */
- c->hydro.h_max = h_max;
- c->stars.h_max = stars_h_max;
-}
-
-/**
- * @brief Cleans the links in a given cell.
- *
- * @param c Cell to act upon
- * @param data Unused parameter
- */
-void cell_clean_links(struct cell *c, void *data) {
- c->hydro.density = NULL;
- c->hydro.gradient = NULL;
- c->hydro.force = NULL;
- c->hydro.limiter = NULL;
- c->hydro.rt_inject = NULL;
- c->grav.grav = NULL;
- c->grav.mm = NULL;
- c->stars.density = NULL;
- c->stars.feedback = NULL;
- c->black_holes.density = NULL;
- c->black_holes.swallow = NULL;
- c->black_holes.do_gas_swallow = NULL;
- c->black_holes.do_bh_swallow = NULL;
- c->black_holes.feedback = NULL;
-}
-
-/**
- * @brief Checks that the #part in a cell are at the
- * current point in time
- *
- * Calls error() if the cell is not at the current time.
- *
- * @param c Cell to act upon
- * @param data The current time on the integer time-line
- */
-void cell_check_part_drift_point(struct cell *c, void *data) {
-#ifdef SWIFT_DEBUG_CHECKS
-
- const integertime_t ti_drift = *(integertime_t *)data;
-
- /* Only check local cells */
- if (c->nodeID != engine_rank) return;
-
- /* Only check cells with content */
- if (c->hydro.count == 0) return;
-
- if (c->hydro.ti_old_part != ti_drift)
- error("Cell in an incorrect time-zone! c->hydro.ti_old=%lld ti_drift=%lld",
- c->hydro.ti_old_part, ti_drift);
-
- for (int i = 0; i < c->hydro.count; ++i)
- if (c->hydro.parts[i].ti_drift != ti_drift &&
- c->hydro.parts[i].time_bin != time_bin_inhibited)
- error("part in an incorrect time-zone! p->ti_drift=%lld ti_drift=%lld",
- c->hydro.parts[i].ti_drift, ti_drift);
-#else
- error("Calling debugging code without debugging flag activated.");
-#endif
-}
-
-/**
- * @brief Checks that the #gpart in a cell are at the
- * current point in time
- *
- * Calls error() if the cell is not at the current time.
- *
- * @param c Cell to act upon
- * @param data The current time on the integer time-line
- */
-void cell_check_gpart_drift_point(struct cell *c, void *data) {
-#ifdef SWIFT_DEBUG_CHECKS
-
- const integertime_t ti_drift = *(integertime_t *)data;
-
- /* Only check local cells */
- if (c->nodeID != engine_rank) return;
-
- /* Only check cells with content */
- if (c->grav.count == 0) return;
-
- if (c->grav.ti_old_part != ti_drift)
- error(
- "Cell in an incorrect time-zone! c->grav.ti_old_part=%lld "
- "ti_drift=%lld",
- c->grav.ti_old_part, ti_drift);
-
- for (int i = 0; i < c->grav.count; ++i)
- if (c->grav.parts[i].ti_drift != ti_drift &&
- c->grav.parts[i].time_bin != time_bin_inhibited)
- error("g-part in an incorrect time-zone! gp->ti_drift=%lld ti_drift=%lld",
- c->grav.parts[i].ti_drift, ti_drift);
-#else
- error("Calling debugging code without debugging flag activated.");
-#endif
-}
-
-/**
- * @brief Checks that the #sink in a cell are at the
- * current point in time
- *
- * Calls error() if the cell is not at the current time.
- *
- * @param c Cell to act upon
- * @param data The current time on the integer time-line
- */
-void cell_check_sink_drift_point(struct cell *c, void *data) {
-#ifdef SWIFT_DEBUG_CHECKS
-
- const integertime_t ti_drift = *(integertime_t *)data;
-
- /* Only check local cells */
- if (c->nodeID != engine_rank) return;
-
- /* Only check cells with content */
- if (c->sinks.count == 0) return;
-
- if (c->sinks.ti_old_part != ti_drift)
- error(
- "Cell in an incorrect time-zone! c->sinks.ti_old_part=%lld "
- "ti_drift=%lld",
- c->sinks.ti_old_part, ti_drift);
-
- for (int i = 0; i < c->sinks.count; ++i)
- if (c->sinks.parts[i].ti_drift != ti_drift &&
- c->sinks.parts[i].time_bin != time_bin_inhibited)
- error(
- "sink-part in an incorrect time-zone! sink->ti_drift=%lld "
- "ti_drift=%lld",
- c->sinks.parts[i].ti_drift, ti_drift);
-#else
- error("Calling debugging code without debugging flag activated.");
-#endif
-}
-
-/**
- * @brief Checks that the #spart in a cell are at the
- * current point in time
- *
- * Calls error() if the cell is not at the current time.
- *
- * @param c Cell to act upon
- * @param data The current time on the integer time-line
- */
-void cell_check_spart_drift_point(struct cell *c, void *data) {
-#ifdef SWIFT_DEBUG_CHECKS
-
- const integertime_t ti_drift = *(integertime_t *)data;
-
- /* Only check local cells */
- if (c->nodeID != engine_rank) return;
-
- /* Only check cells with content */
- if (c->stars.count == 0) return;
-
- if (c->stars.ti_old_part != ti_drift)
- error(
- "Cell in an incorrect time-zone! c->stars.ti_old_part=%lld "
- "ti_drift=%lld",
- c->stars.ti_old_part, ti_drift);
-
- for (int i = 0; i < c->stars.count; ++i)
- if (c->stars.parts[i].ti_drift != ti_drift &&
- c->stars.parts[i].time_bin != time_bin_inhibited)
- error("g-part in an incorrect time-zone! gp->ti_drift=%lld ti_drift=%lld",
- c->stars.parts[i].ti_drift, ti_drift);
-#else
- error("Calling debugging code without debugging flag activated.");
-#endif
-}
-
-/**
- * @brief Checks that the multipole of a cell is at the current point in time
- *
- * Calls error() if the cell is not at the current time.
- *
- * @param c Cell to act upon
- * @param data The current time on the integer time-line
- */
-void cell_check_multipole_drift_point(struct cell *c, void *data) {
-#ifdef SWIFT_DEBUG_CHECKS
-
- const integertime_t ti_drift = *(integertime_t *)data;
-
- /* Only check local cells */
- if (c->nodeID != engine_rank) return;
-
- /* Only check cells with content */
- if (c->grav.count == 0) return;
-
- if (c->grav.ti_old_multipole != ti_drift)
- error(
- "Cell multipole in an incorrect time-zone! "
- "c->grav.ti_old_multipole=%lld "
- "ti_drift=%lld (depth=%d, node=%d)",
- c->grav.ti_old_multipole, ti_drift, c->depth, c->nodeID);
-
-#else
- error("Calling debugging code without debugging flag activated.");
-#endif
-}
-
-/**
- * @brief Resets all the individual cell task counters to 0.
- *
- * Should only be used for debugging purposes.
- *
- * @param c The #cell to reset.
- */
-void cell_reset_task_counters(struct cell *c) {
-#ifdef SWIFT_DEBUG_CHECKS
- for (int t = 0; t < task_type_count; ++t) c->tasks_executed[t] = 0;
- for (int t = 0; t < task_subtype_count; ++t) c->subtasks_executed[t] = 0;
- for (int k = 0; k < 8; ++k)
- if (c->progeny[k] != NULL) cell_reset_task_counters(c->progeny[k]);
-#else
- error("Calling debugging code without debugging flag activated.");
-#endif
-}
-
-/**
- * @brief Recursively construct all the multipoles in a cell hierarchy.
- *
- * @param c The #cell.
- * @param ti_current The current integer time.
- * @param grav_props The properties of the gravity scheme.
- */
-void cell_make_multipoles(struct cell *c, integertime_t ti_current,
- const struct gravity_props *const grav_props) {
-
- /* Reset everything */
- gravity_reset(c->grav.multipole);
-
- if (c->split) {
-
- /* Start by recursing */
- for (int k = 0; k < 8; ++k) {
- if (c->progeny[k] != NULL)
- cell_make_multipoles(c->progeny[k], ti_current, grav_props);
- }
-
- /* Compute CoM of all progenies */
- double CoM[3] = {0., 0., 0.};
- double vel[3] = {0., 0., 0.};
- float max_delta_vel[3] = {0.f, 0.f, 0.f};
- float min_delta_vel[3] = {0.f, 0.f, 0.f};
- double mass = 0.;
-
- for (int k = 0; k < 8; ++k) {
- if (c->progeny[k] != NULL) {
- const struct gravity_tensors *m = c->progeny[k]->grav.multipole;
-
- mass += m->m_pole.M_000;
-
- CoM[0] += m->CoM[0] * m->m_pole.M_000;
- CoM[1] += m->CoM[1] * m->m_pole.M_000;
- CoM[2] += m->CoM[2] * m->m_pole.M_000;
-
- vel[0] += m->m_pole.vel[0] * m->m_pole.M_000;
- vel[1] += m->m_pole.vel[1] * m->m_pole.M_000;
- vel[2] += m->m_pole.vel[2] * m->m_pole.M_000;
-
- max_delta_vel[0] = max(m->m_pole.max_delta_vel[0], max_delta_vel[0]);
- max_delta_vel[1] = max(m->m_pole.max_delta_vel[1], max_delta_vel[1]);
- max_delta_vel[2] = max(m->m_pole.max_delta_vel[2], max_delta_vel[2]);
-
- min_delta_vel[0] = min(m->m_pole.min_delta_vel[0], min_delta_vel[0]);
- min_delta_vel[1] = min(m->m_pole.min_delta_vel[1], min_delta_vel[1]);
- min_delta_vel[2] = min(m->m_pole.min_delta_vel[2], min_delta_vel[2]);
- }
- }
-
- /* Final operation on the CoM and bulk velocity */
- const double mass_inv = 1. / mass;
- c->grav.multipole->CoM[0] = CoM[0] * mass_inv;
- c->grav.multipole->CoM[1] = CoM[1] * mass_inv;
- c->grav.multipole->CoM[2] = CoM[2] * mass_inv;
- c->grav.multipole->m_pole.vel[0] = vel[0] * mass_inv;
- c->grav.multipole->m_pole.vel[1] = vel[1] * mass_inv;
- c->grav.multipole->m_pole.vel[2] = vel[2] * mass_inv;
-
- /* Min max velocity along each axis */
- c->grav.multipole->m_pole.max_delta_vel[0] = max_delta_vel[0];
- c->grav.multipole->m_pole.max_delta_vel[1] = max_delta_vel[1];
- c->grav.multipole->m_pole.max_delta_vel[2] = max_delta_vel[2];
- c->grav.multipole->m_pole.min_delta_vel[0] = min_delta_vel[0];
- c->grav.multipole->m_pole.min_delta_vel[1] = min_delta_vel[1];
- c->grav.multipole->m_pole.min_delta_vel[2] = min_delta_vel[2];
-
- /* Now shift progeny multipoles and add them up */
- struct multipole temp;
- double r_max = 0.;
- for (int k = 0; k < 8; ++k) {
- if (c->progeny[k] != NULL) {
- const struct cell *cp = c->progeny[k];
- const struct multipole *m = &cp->grav.multipole->m_pole;
-
- /* Contribution to multipole */
- gravity_M2M(&temp, m, c->grav.multipole->CoM, cp->grav.multipole->CoM);
- gravity_multipole_add(&c->grav.multipole->m_pole, &temp);
-
- /* Upper limit of max CoM<->gpart distance */
- const double dx =
- c->grav.multipole->CoM[0] - cp->grav.multipole->CoM[0];
- const double dy =
- c->grav.multipole->CoM[1] - cp->grav.multipole->CoM[1];
- const double dz =
- c->grav.multipole->CoM[2] - cp->grav.multipole->CoM[2];
- const double r2 = dx * dx + dy * dy + dz * dz;
- r_max = max(r_max, cp->grav.multipole->r_max + sqrt(r2));
- }
- }
- /* Alternative upper limit of max CoM<->gpart distance */
- const double dx = c->grav.multipole->CoM[0] > c->loc[0] + c->width[0] * 0.5
- ? c->grav.multipole->CoM[0] - c->loc[0]
- : c->loc[0] + c->width[0] - c->grav.multipole->CoM[0];
- const double dy = c->grav.multipole->CoM[1] > c->loc[1] + c->width[1] * 0.5
- ? c->grav.multipole->CoM[1] - c->loc[1]
- : c->loc[1] + c->width[1] - c->grav.multipole->CoM[1];
- const double dz = c->grav.multipole->CoM[2] > c->loc[2] + c->width[2] * 0.5
- ? c->grav.multipole->CoM[2] - c->loc[2]
- : c->loc[2] + c->width[2] - c->grav.multipole->CoM[2];
-
- /* Take minimum of both limits */
- c->grav.multipole->r_max = min(r_max, sqrt(dx * dx + dy * dy + dz * dz));
-
- /* Compute the multipole power */
- gravity_multipole_compute_power(&c->grav.multipole->m_pole);
-
- } else {
- if (c->grav.count > 0) {
-
- gravity_P2M(c->grav.multipole, c->grav.parts, c->grav.count, grav_props);
-
- /* Compute the multipole power */
- gravity_multipole_compute_power(&c->grav.multipole->m_pole);
-
- } else {
-
- /* No gparts in that leaf cell */
-
- /* Set the values to something sensible */
- gravity_multipole_init(&c->grav.multipole->m_pole);
- c->grav.multipole->CoM[0] = c->loc[0] + c->width[0] * 0.5;
- c->grav.multipole->CoM[1] = c->loc[1] + c->width[1] * 0.5;
- c->grav.multipole->CoM[2] = c->loc[2] + c->width[2] * 0.5;
- c->grav.multipole->r_max = 0.;
- }
- }
-
- /* Also update the values at rebuild time */
- c->grav.multipole->r_max_rebuild = c->grav.multipole->r_max;
- c->grav.multipole->CoM_rebuild[0] = c->grav.multipole->CoM[0];
- c->grav.multipole->CoM_rebuild[1] = c->grav.multipole->CoM[1];
- c->grav.multipole->CoM_rebuild[2] = c->grav.multipole->CoM[2];
-
- c->grav.ti_old_multipole = ti_current;
-}
-
-/**
- * @brief Recursively verify that the multipoles are the sum of their progenies.
- *
- * This function does not check whether the multipoles match the particle
- * content as we may not have received the particles.
- *
- * @param c The #cell to recursively search and verify.
- */
-void cell_check_foreign_multipole(const struct cell *c) {
-#ifdef SWIFT_DEBUG_CHECKS
-
- if (c->split) {
- double M_000 = 0.;
- long long num_gpart = 0;
-
- for (int k = 0; k < 8; k++) {
- const struct cell *cp = c->progeny[k];
-
- if (cp != NULL) {
- /* Check the mass */
- M_000 += cp->grav.multipole->m_pole.M_000;
-
- /* Check the number of particles */
- num_gpart += cp->grav.multipole->m_pole.num_gpart;
-
- /* Now recurse */
- cell_check_foreign_multipole(cp);
- }
- }
-
- if (num_gpart != c->grav.multipole->m_pole.num_gpart)
- error("Sum of particles in progenies does not match");
- }
-
-#else
- error("Calling debugging code without debugging flag activated.");
-#endif
-}
-
-/**
- * @brief Computes the multi-pole brutally and compare to the
- * recursively computed one.
- *
- * @param c Cell to act upon
- * @param grav_props The properties of the gravity scheme.
- */
-void cell_check_multipole(struct cell *c,
- const struct gravity_props *const grav_props) {
-
-#ifdef SWIFT_DEBUG_CHECKS
- struct gravity_tensors ma;
- const double tolerance = 1e-3; /* Relative */
-
- /* First recurse */
- if (c->split)
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL)
- cell_check_multipole(c->progeny[k], grav_props);
-
- if (c->grav.count > 0) {
- /* Brute-force calculation */
- gravity_P2M(&ma, c->grav.parts, c->grav.count, grav_props);
- gravity_multipole_compute_power(&ma.m_pole);
-
- /* Now compare the multipole expansion */
- if (!gravity_multipole_equal(&ma, c->grav.multipole, tolerance)) {
- message("Multipoles are not equal at depth=%d! tol=%f", c->depth,
- tolerance);
- message("Correct answer:");
- gravity_multipole_print(&ma.m_pole);
- message("Recursive multipole:");
- gravity_multipole_print(&c->grav.multipole->m_pole);
- error("Aborting");
- }
-
- /* Check that the upper limit of r_max is good enough */
- if (!(1.1 * c->grav.multipole->r_max >= ma.r_max)) {
- error("Upper-limit r_max=%e too small. Should be >=%e.",
- c->grav.multipole->r_max, ma.r_max);
- } else if (c->grav.multipole->r_max * c->grav.multipole->r_max >
- 3. * c->width[0] * c->width[0]) {
- error("r_max=%e larger than cell diagonal %e.", c->grav.multipole->r_max,
- sqrt(3. * c->width[0] * c->width[0]));
- }
- }
-#else
- error("Calling debugging code without debugging flag activated.");
-#endif
-}
-
-/**
- * @brief Frees up the memory allocated for this #cell.
- *
- * @param c The #cell.
- */
-void cell_clean(struct cell *c) {
- /* Hydro */
- cell_free_hydro_sorts(c);
-
- /* Stars */
- cell_free_stars_sorts(c);
-
- /* Recurse */
- for (int k = 0; k < 8; k++)
- if (c->progeny[k]) cell_clean(c->progeny[k]);
-}
-
-/**
- * @brief Clear the drift flags on the given cell.
- */
-void cell_clear_drift_flags(struct cell *c, void *data) {
- cell_clear_flag(c, cell_flag_do_hydro_drift | cell_flag_do_hydro_sub_drift |
- cell_flag_do_grav_drift | cell_flag_do_grav_sub_drift |
- cell_flag_do_bh_drift | cell_flag_do_bh_sub_drift |
- cell_flag_do_stars_drift |
- cell_flag_do_stars_sub_drift |
- cell_flag_do_sink_drift | cell_flag_do_sink_sub_drift);
-}
-
-/**
- * @brief Clear the limiter flags on the given cell.
- */
-void cell_clear_limiter_flags(struct cell *c, void *data) {
- cell_clear_flag(c,
- cell_flag_do_hydro_limiter | cell_flag_do_hydro_sub_limiter);
-}
-
-/**
- * @brief Recursively clear the stars_resort flag in a cell hierarchy.
- *
- * @param c The #cell to act on.
- */
-void cell_set_star_resort_flag(struct cell *c) {
-
- cell_set_flag(c, cell_flag_do_stars_resort);
-
- /* Abort if we reched the level where the resorting task lives */
- if (c->depth == engine_star_resort_task_depth || c->hydro.super == c) return;
-
- if (c->split) {
- for (int k = 0; k < 8; ++k)
- if (c->progeny[k] != NULL) cell_set_star_resort_flag(c->progeny[k]);
- }
-}
-
-/**
- * @brief Recurses in a cell hierarchy down to the level where the
- * star resort tasks are and activates them.
- *
- * The function will fail if called *below* the super-level
- *
- * @param c The #cell to recurse into.
- * @param s The #scheduler.
- */
-void cell_activate_star_resort_tasks(struct cell *c, struct scheduler *s) {
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (c->hydro.super != NULL && c->hydro.super != c)
- error("Function called below the super level!");
-#endif
-
- /* The resort tasks are at either the chosen depth or the super level,
- * whichever comes first. */
- if ((c->depth == engine_star_resort_task_depth || c->hydro.super == c) &&
- c->hydro.count > 0) {
- scheduler_activate(s, c->hydro.stars_resort);
- } else {
- for (int k = 0; k < 8; ++k) {
- if (c->progeny[k] != NULL) {
- cell_activate_star_resort_tasks(c->progeny[k], s);
- }
- }
- }
-}
-
-/**
- * @brief Activate the star formation task as well as the resorting of stars
- *
- * Must be called at the top-level in the tree (where the SF task is...)
- *
- * @param c The (top-level) #cell.
- * @param s The #scheduler.
- * @param with_feedback Are we running with feedback?
- */
-void cell_activate_star_formation_tasks(struct cell *c, struct scheduler *s,
- const int with_feedback) {
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (c->depth != 0) error("Function should be called at the top-level only");
-#endif
-
- /* Have we already unskipped that task? */
- if (c->hydro.star_formation->skip == 0) return;
-
- /* Activate the star formation task */
- scheduler_activate(s, c->hydro.star_formation);
-
- /* Activate the star resort tasks at whatever level they are */
- if (with_feedback) {
- cell_activate_star_resort_tasks(c, s);
- }
-}
-
-/**
- * @brief Activate the sink formation task.
- *
- * Must be called at the top-level in the tree (where the SF task is...)
- *
- * @param c The (top-level) #cell.
- * @param s The #scheduler.
- */
-void cell_activate_sink_formation_tasks(struct cell *c, struct scheduler *s) {
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (c->depth != 0) error("Function should be called at the top-level only");
-#endif
-
- /* Have we already unskipped that task? */
- if (c->hydro.sink_formation->skip == 0) return;
-
- /* Activate the star formation task */
- scheduler_activate(s, c->hydro.sink_formation);
-}
-
-/**
- * @brief Recursively activate the hydro ghosts (and implicit links) in a cell
- * hierarchy.
- *
- * @param c The #cell.
- * @param s The #scheduler.
- * @param e The #engine.
- */
-void cell_recursively_activate_hydro_ghosts(struct cell *c, struct scheduler *s,
- const struct engine *e) {
- /* Early abort? */
- if ((c->hydro.count == 0) || !cell_is_active_hydro(c, e)) return;
-
- /* Is the ghost at this level? */
- if (c->hydro.ghost != NULL) {
- scheduler_activate(s, c->hydro.ghost);
- } else {
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (!c->split)
- error("Reached the leaf level without finding a hydro ghost!");
-#endif
-
- /* Keep recursing */
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL)
- cell_recursively_activate_hydro_ghosts(c->progeny[k], s, e);
- }
-}
-
-/**
- * @brief Activate the hydro ghosts (and implicit links) in a cell hierarchy.
- *
- * @param c The #cell.
- * @param s The #scheduler.
- * @param e The #engine.
- */
-void cell_activate_hydro_ghosts(struct cell *c, struct scheduler *s,
- const struct engine *e) {
- scheduler_activate(s, c->hydro.ghost_in);
- scheduler_activate(s, c->hydro.ghost_out);
- cell_recursively_activate_hydro_ghosts(c, s, e);
-}
-
-/**
- * @brief Recursively activate the cooling (and implicit links) in a cell
- * hierarchy.
- *
- * @param c The #cell.
- * @param s The #scheduler.
- * @param e The #engine.
- */
-void cell_recursively_activate_cooling(struct cell *c, struct scheduler *s,
- const struct engine *e) {
- /* Early abort? */
- if ((c->hydro.count == 0) || !cell_is_active_hydro(c, e)) return;
-
- /* Is the ghost at this level? */
- if (c->hydro.cooling != NULL) {
- scheduler_activate(s, c->hydro.cooling);
- } else {
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (!c->split)
- error("Reached the leaf level without finding a cooling task!");
-#endif
-
- /* Keep recursing */
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL)
- cell_recursively_activate_cooling(c->progeny[k], s, e);
- }
-}
-
-/**
- * @brief Activate the cooling tasks (and implicit links) in a cell hierarchy.
- *
- * @param c The #cell.
- * @param s The #scheduler.
- * @param e The #engine.
- */
-void cell_activate_cooling(struct cell *c, struct scheduler *s,
- const struct engine *e) {
- scheduler_activate(s, c->hydro.cooling_in);
- scheduler_activate(s, c->hydro.cooling_out);
- cell_recursively_activate_cooling(c, s, e);
-}
-
-/**
- * @brief Recurse down in a cell hierarchy until the hydro.super level is
- * reached and activate the spart drift at that level.
- *
- * @param c The #cell to recurse into.
- * @param s The #scheduler.
- */
-void cell_activate_super_spart_drifts(struct cell *c, struct scheduler *s) {
-
- /* Early abort? */
- if (c->hydro.count == 0) return;
-
- if (c == c->hydro.super) {
- cell_activate_drift_spart(c, s);
- } else {
- if (c->split) {
- for (int k = 0; k < 8; ++k) {
- if (c->progeny[k] != NULL) {
- cell_activate_super_spart_drifts(c->progeny[k], s);
- }
- }
- } else {
-#ifdef SWIFT_DEBUG_CHECKS
- error("Reached a leaf cell without finding a hydro.super!!");
-#endif
- }
- }
-}
-
-/**
- * @brief Recurse down in a cell hierarchy until the hydro.super level is
- * reached and activate the sink drift at that level.
- *
- * @param c The #cell to recurse into.
- * @param s The #scheduler.
- */
-void cell_activate_super_sink_drifts(struct cell *c, struct scheduler *s) {
-
- /* Early abort? */
- if (c->hydro.count == 0) return;
-
- if (c == c->hydro.super) {
- cell_activate_drift_sink(c, s);
- } else {
- if (c->split) {
- for (int k = 0; k < 8; ++k) {
- if (c->progeny[k] != NULL) {
- cell_activate_super_sink_drifts(c->progeny[k], s);
- }
- }
- } else {
-#ifdef SWIFT_DEBUG_CHECKS
- error("Reached a leaf cell without finding a hydro.super!!");
-#endif
- }
- }
-}
-
-/**
- * @brief Activate the #part drifts on the given cell.
- */
-void cell_activate_drift_part(struct cell *c, struct scheduler *s) {
- /* If this cell is already marked for drift, quit early. */
- if (cell_get_flag(c, cell_flag_do_hydro_drift)) return;
-
- /* Mark this cell for drifting. */
- cell_set_flag(c, cell_flag_do_hydro_drift);
-
- /* Set the do_sub_drifts all the way up and activate the super drift
- if this has not yet been done. */
- if (c == c->hydro.super) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (c->hydro.drift == NULL)
- error("Trying to activate un-existing c->hydro.drift");
-#endif
- scheduler_activate(s, c->hydro.drift);
- } else {
- for (struct cell *parent = c->parent;
- parent != NULL && !cell_get_flag(parent, cell_flag_do_hydro_sub_drift);
- parent = parent->parent) {
- /* Mark this cell for drifting */
- cell_set_flag(parent, cell_flag_do_hydro_sub_drift);
-
- if (parent == c->hydro.super) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (parent->hydro.drift == NULL)
- error("Trying to activate un-existing parent->hydro.drift");
-#endif
- scheduler_activate(s, parent->hydro.drift);
- break;
- }
- }
- }
-}
-
-void cell_activate_sync_part(struct cell *c, struct scheduler *s) {
- /* If this cell is already marked for drift, quit early. */
- if (cell_get_flag(c, cell_flag_do_hydro_sync)) return;
-
- /* Mark this cell for synchronization. */
- cell_set_flag(c, cell_flag_do_hydro_sync);
-
- /* Set the do_sub_sync all the way up and activate the super sync
- if this has not yet been done. */
- if (c == c->super) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (c->timestep_sync == NULL)
- error("Trying to activate un-existing c->timestep_sync");
-#endif
- scheduler_activate(s, c->timestep_sync);
- scheduler_activate(s, c->kick1);
- } else {
- for (struct cell *parent = c->parent;
- parent != NULL && !cell_get_flag(parent, cell_flag_do_hydro_sub_sync);
- parent = parent->parent) {
- /* Mark this cell for drifting */
- cell_set_flag(parent, cell_flag_do_hydro_sub_sync);
-
- if (parent == c->super) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (parent->timestep_sync == NULL)
- error("Trying to activate un-existing parent->timestep_sync");
-#endif
- scheduler_activate(s, parent->timestep_sync);
- scheduler_activate(s, parent->kick1);
- break;
- }
- }
- }
-}
-
-/**
- * @brief Activate the #gpart drifts on the given cell.
- */
-void cell_activate_drift_gpart(struct cell *c, struct scheduler *s) {
- /* If this cell is already marked for drift, quit early. */
- if (cell_get_flag(c, cell_flag_do_grav_drift)) return;
-
- /* Mark this cell for drifting. */
- cell_set_flag(c, cell_flag_do_grav_drift);
-
- if (c->grav.drift_out != NULL) scheduler_activate(s, c->grav.drift_out);
-
- /* Set the do_grav_sub_drifts all the way up and activate the super drift
- if this has not yet been done. */
- if (c == c->grav.super) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (c->grav.drift == NULL)
- error("Trying to activate un-existing c->grav.drift");
-#endif
- scheduler_activate(s, c->grav.drift);
- } else {
- for (struct cell *parent = c->parent;
- parent != NULL && !cell_get_flag(parent, cell_flag_do_grav_sub_drift);
- parent = parent->parent) {
- cell_set_flag(parent, cell_flag_do_grav_sub_drift);
-
- if (parent->grav.drift_out) {
- scheduler_activate(s, parent->grav.drift_out);
- }
-
- if (parent == c->grav.super) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (parent->grav.drift == NULL)
- error("Trying to activate un-existing parent->grav.drift");
-#endif
- scheduler_activate(s, parent->grav.drift);
- break;
- }
- }
- }
-}
-
-/**
- * @brief Activate the #spart drifts on the given cell.
- */
-void cell_activate_drift_spart(struct cell *c, struct scheduler *s) {
- /* If this cell is already marked for drift, quit early. */
- if (cell_get_flag(c, cell_flag_do_stars_drift)) return;
-
- /* Mark this cell for drifting. */
- cell_set_flag(c, cell_flag_do_stars_drift);
-
- /* Set the do_stars_sub_drifts all the way up and activate the super drift
- if this has not yet been done. */
- if (c == c->hydro.super) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (c->stars.drift == NULL)
- error("Trying to activate un-existing c->stars.drift");
-#endif
- scheduler_activate(s, c->stars.drift);
- } else {
- for (struct cell *parent = c->parent;
- parent != NULL && !cell_get_flag(parent, cell_flag_do_stars_sub_drift);
- parent = parent->parent) {
- /* Mark this cell for drifting */
- cell_set_flag(parent, cell_flag_do_stars_sub_drift);
-
- if (parent == c->hydro.super) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (parent->stars.drift == NULL)
- error("Trying to activate un-existing parent->stars.drift");
-#endif
- scheduler_activate(s, parent->stars.drift);
- break;
- }
- }
- }
-}
-
-/**
- * @brief Activate the #bpart drifts on the given cell.
- */
-void cell_activate_drift_bpart(struct cell *c, struct scheduler *s) {
-
- /* If this cell is already marked for drift, quit early. */
- if (cell_get_flag(c, cell_flag_do_bh_drift)) return;
-
- /* Mark this cell for drifting. */
- cell_set_flag(c, cell_flag_do_bh_drift);
-
- /* Set the do_black_holes_sub_drifts all the way up and activate the super
- drift if this has not yet been done. */
- if (c == c->hydro.super) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (c->black_holes.drift == NULL)
- error("Trying to activate un-existing c->black_holes.drift");
-#endif
- scheduler_activate(s, c->black_holes.drift);
- } else {
- for (struct cell *parent = c->parent;
- parent != NULL && !cell_get_flag(parent, cell_flag_do_bh_sub_drift);
- parent = parent->parent) {
- /* Mark this cell for drifting */
- cell_set_flag(parent, cell_flag_do_bh_sub_drift);
-
- if (parent == c->hydro.super) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (parent->black_holes.drift == NULL)
- error("Trying to activate un-existing parent->black_holes.drift");
-#endif
- scheduler_activate(s, parent->black_holes.drift);
- break;
- }
- }
- }
-}
-
-/**
- * @brief Activate the #sink drifts on the given cell.
- */
-void cell_activate_drift_sink(struct cell *c, struct scheduler *s) {
-
- /* If this cell is already marked for drift, quit early. */
- if (cell_get_flag(c, cell_flag_do_sink_drift)) return;
-
- /* Mark this cell for drifting. */
- cell_set_flag(c, cell_flag_do_sink_drift);
-
- /* Set the do_sink_sub_drifts all the way up and activate the super
- drift if this has not yet been done. */
- if (c == c->hydro.super) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (c->sinks.drift == NULL)
- error("Trying to activate un-existing c->sinks.drift");
-#endif
- scheduler_activate(s, c->sinks.drift);
- } else {
- for (struct cell *parent = c->parent;
- parent != NULL && !cell_get_flag(parent, cell_flag_do_sink_sub_drift);
- parent = parent->parent) {
- /* Mark this cell for drifting */
- cell_set_flag(parent, cell_flag_do_sink_sub_drift);
-
- if (parent == c->hydro.super) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (parent->sinks.drift == NULL)
- error("Trying to activate un-existing parent->sinks.drift");
-#endif
- scheduler_activate(s, parent->sinks.drift);
- break;
- }
- }
- }
-}
-
-/**
- * @brief Activate the drifts on the given cell.
- */
-void cell_activate_limiter(struct cell *c, struct scheduler *s) {
- /* If this cell is already marked for limiting, quit early. */
- if (cell_get_flag(c, cell_flag_do_hydro_limiter)) return;
-
- /* Mark this cell for limiting. */
- cell_set_flag(c, cell_flag_do_hydro_limiter);
-
- /* Set the do_sub_limiter all the way up and activate the super limiter
- if this has not yet been done. */
- if (c == c->super) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (c->timestep_limiter == NULL)
- error("Trying to activate un-existing c->timestep_limiter");
-#endif
- scheduler_activate(s, c->timestep_limiter);
- scheduler_activate(s, c->kick1);
- } else {
- for (struct cell *parent = c->parent;
- parent != NULL &&
- !cell_get_flag(parent, cell_flag_do_hydro_sub_limiter);
- parent = parent->parent) {
- /* Mark this cell for limiting */
- cell_set_flag(parent, cell_flag_do_hydro_sub_limiter);
-
- if (parent == c->super) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (parent->timestep_limiter == NULL)
- error("Trying to activate un-existing parent->timestep_limiter");
-#endif
- scheduler_activate(s, parent->timestep_limiter);
- scheduler_activate(s, parent->kick1);
- break;
- }
- }
- }
-}
-
-/**
- * @brief Activate the sorts up a cell hierarchy.
- */
-void cell_activate_hydro_sorts_up(struct cell *c, struct scheduler *s) {
- if (c == c->hydro.super) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (c->hydro.sorts == NULL)
- error("Trying to activate un-existing c->hydro.sorts");
-#endif
- scheduler_activate(s, c->hydro.sorts);
- if (c->nodeID == engine_rank) cell_activate_drift_part(c, s);
- } else {
- for (struct cell *parent = c->parent;
- parent != NULL && !cell_get_flag(parent, cell_flag_do_hydro_sub_sort);
- parent = parent->parent) {
- cell_set_flag(parent, cell_flag_do_hydro_sub_sort);
- if (parent == c->hydro.super) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (parent->hydro.sorts == NULL)
- error("Trying to activate un-existing parents->hydro.sorts");
-#endif
- scheduler_activate(s, parent->hydro.sorts);
- if (parent->nodeID == engine_rank) cell_activate_drift_part(parent, s);
- break;
- }
- }
- }
-}
-
-/**
- * @brief Activate the sorts on a given cell, if needed.
- */
-void cell_activate_hydro_sorts(struct cell *c, int sid, struct scheduler *s) {
- /* Do we need to re-sort? */
- if (c->hydro.dx_max_sort > space_maxreldx * c->dmin) {
- /* Climb up the tree to active the sorts in that direction */
- for (struct cell *finger = c; finger != NULL; finger = finger->parent) {
- if (finger->hydro.requires_sorts) {
- atomic_or(&finger->hydro.do_sort, finger->hydro.requires_sorts);
- cell_activate_hydro_sorts_up(finger, s);
- }
- finger->hydro.sorted = 0;
- }
- }
-
- /* Has this cell been sorted at all for the given sid? */
- if (!(c->hydro.sorted & (1 << sid)) || c->nodeID != engine_rank) {
- atomic_or(&c->hydro.do_sort, (1 << sid));
- cell_activate_hydro_sorts_up(c, s);
- }
-}
-
-/**
- * @brief Activate the sorts up a cell hierarchy.
- */
-void cell_activate_stars_sorts_up(struct cell *c, struct scheduler *s) {
-
- if (c == c->hydro.super) {
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (c->stars.sorts == NULL)
- error("Trying to activate un-existing c->stars.sorts");
-#endif
- scheduler_activate(s, c->stars.sorts);
- if (c->nodeID == engine_rank) {
- cell_activate_drift_spart(c, s);
- }
- } else {
-
- /* Climb up the tree and set the flags */
- for (struct cell *parent = c->parent;
- parent != NULL && !cell_get_flag(parent, cell_flag_do_stars_sub_sort);
- parent = parent->parent) {
-
- cell_set_flag(parent, cell_flag_do_stars_sub_sort);
-
- /* Reached the super-level? Activate the task and abort */
- if (parent == c->hydro.super) {
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (parent->stars.sorts == NULL)
- error("Trying to activate un-existing parents->stars.sorts");
-#endif
- scheduler_activate(s, parent->stars.sorts);
- if (parent->nodeID == engine_rank) cell_activate_drift_spart(parent, s);
- break;
- }
- }
- }
-}
-
-/**
- * @brief Activate the sorts on a given cell, if needed.
- */
-void cell_activate_stars_sorts(struct cell *c, int sid, struct scheduler *s) {
-
- /* Do we need to re-sort? */
- if (c->stars.dx_max_sort > space_maxreldx * c->dmin) {
-
- /* Climb up the tree to active the sorts in that direction */
- for (struct cell *finger = c; finger != NULL; finger = finger->parent) {
- if (finger->stars.requires_sorts) {
- atomic_or(&finger->stars.do_sort, finger->stars.requires_sorts);
- cell_activate_stars_sorts_up(finger, s);
- }
- finger->stars.sorted = 0;
- }
- }
-
- /* Has this cell been sorted at all for the given sid? */
- if (!(c->stars.sorted & (1 << sid)) || c->nodeID != engine_rank) {
- atomic_or(&c->stars.do_sort, (1 << sid));
- cell_activate_stars_sorts_up(c, s);
- }
-}
-
-/**
- * @brief Traverse a sub-cell task and activate the hydro drift tasks that are
- * required by a hydro task
- *
- * @param ci The first #cell we recurse in.
- * @param cj The second #cell we recurse in.
- * @param s The task #scheduler.
- * @param with_timestep_limiter Are we running with time-step limiting on?
- */
-void cell_activate_subcell_hydro_tasks(struct cell *ci, struct cell *cj,
- struct scheduler *s,
- const int with_timestep_limiter) {
- const struct engine *e = s->space->e;
-
- /* Store the current dx_max and h_max values. */
- ci->hydro.dx_max_part_old = ci->hydro.dx_max_part;
- ci->hydro.h_max_old = ci->hydro.h_max;
-
- if (cj != NULL) {
- cj->hydro.dx_max_part_old = cj->hydro.dx_max_part;
- cj->hydro.h_max_old = cj->hydro.h_max;
- }
-
- /* Self interaction? */
- if (cj == NULL) {
- /* Do anything? */
- if (ci->hydro.count == 0 || !cell_is_active_hydro(ci, e)) return;
-
- /* Recurse? */
- if (cell_can_recurse_in_self_hydro_task(ci)) {
- /* Loop over all progenies and pairs of progenies */
- for (int j = 0; j < 8; j++) {
- if (ci->progeny[j] != NULL) {
- cell_activate_subcell_hydro_tasks(ci->progeny[j], NULL, s,
- with_timestep_limiter);
- for (int k = j + 1; k < 8; k++)
- if (ci->progeny[k] != NULL)
- cell_activate_subcell_hydro_tasks(ci->progeny[j], ci->progeny[k],
- s, with_timestep_limiter);
- }
- }
- } else {
- /* We have reached the bottom of the tree: activate drift */
- cell_activate_drift_part(ci, s);
- if (with_timestep_limiter) cell_activate_limiter(ci, s);
- }
- }
-
- /* Otherwise, pair interation */
- else {
- /* Should we even bother? */
- if (!cell_is_active_hydro(ci, e) && !cell_is_active_hydro(cj, e)) return;
- if (ci->hydro.count == 0 || cj->hydro.count == 0) return;
-
- /* Get the orientation of the pair. */
- double shift[3];
- const int sid = space_getsid(s->space, &ci, &cj, shift);
-
- /* recurse? */
- if (cell_can_recurse_in_pair_hydro_task(ci) &&
- cell_can_recurse_in_pair_hydro_task(cj)) {
- const struct cell_split_pair *csp = &cell_split_pairs[sid];
- for (int k = 0; k < csp->count; k++) {
- const int pid = csp->pairs[k].pid;
- const int pjd = csp->pairs[k].pjd;
- if (ci->progeny[pid] != NULL && cj->progeny[pjd] != NULL)
- cell_activate_subcell_hydro_tasks(ci->progeny[pid], cj->progeny[pjd],
- s, with_timestep_limiter);
- }
- }
-
- /* Otherwise, activate the sorts and drifts. */
- else if (cell_is_active_hydro(ci, e) || cell_is_active_hydro(cj, e)) {
- /* We are going to interact this pair, so store some values. */
- atomic_or(&ci->hydro.requires_sorts, 1 << sid);
- atomic_or(&cj->hydro.requires_sorts, 1 << sid);
- ci->hydro.dx_max_sort_old = ci->hydro.dx_max_sort;
- cj->hydro.dx_max_sort_old = cj->hydro.dx_max_sort;
-
- /* Activate the drifts if the cells are local. */
- if (ci->nodeID == engine_rank) cell_activate_drift_part(ci, s);
- if (cj->nodeID == engine_rank) cell_activate_drift_part(cj, s);
-
- /* Also activate the time-step limiter */
- if (ci->nodeID == engine_rank && with_timestep_limiter)
- cell_activate_limiter(ci, s);
- if (cj->nodeID == engine_rank && with_timestep_limiter)
- cell_activate_limiter(cj, s);
-
- /* Do we need to sort the cells? */
- cell_activate_hydro_sorts(ci, sid, s);
- cell_activate_hydro_sorts(cj, sid, s);
- }
- } /* Otherwise, pair interation */
-}
-
-/**
- * @brief Traverse a sub-cell task and activate the stars drift tasks that are
- * required by a stars task
- *
- * @param ci The first #cell we recurse in.
- * @param cj The second #cell we recurse in.
- * @param s The task #scheduler.
- * @param with_star_formation Are we running with star formation switched on?
- * @param with_timestep_sync Are we running with time-step synchronization on?
- */
-void cell_activate_subcell_stars_tasks(struct cell *ci, struct cell *cj,
- struct scheduler *s,
- const int with_star_formation,
- const int with_timestep_sync) {
- const struct engine *e = s->space->e;
-
- /* Store the current dx_max and h_max values. */
- ci->stars.dx_max_part_old = ci->stars.dx_max_part;
- ci->stars.h_max_old = ci->stars.h_max;
- ci->hydro.dx_max_part_old = ci->hydro.dx_max_part;
- ci->hydro.h_max_old = ci->hydro.h_max;
-
- if (cj != NULL) {
- cj->stars.dx_max_part_old = cj->stars.dx_max_part;
- cj->stars.h_max_old = cj->stars.h_max;
- cj->hydro.dx_max_part_old = cj->hydro.dx_max_part;
- cj->hydro.h_max_old = cj->hydro.h_max;
- }
-
- /* Self interaction? */
- if (cj == NULL) {
-
- const int ci_active = cell_is_active_stars(ci, e) ||
- (with_star_formation && cell_is_active_hydro(ci, e));
-
- /* Do anything? */
- if (!ci_active || ci->hydro.count == 0 ||
- (!with_star_formation && ci->stars.count == 0))
- return;
-
- /* Recurse? */
- if (cell_can_recurse_in_self_stars_task(ci)) {
- /* Loop over all progenies and pairs of progenies */
- for (int j = 0; j < 8; j++) {
- if (ci->progeny[j] != NULL) {
- cell_activate_subcell_stars_tasks(
- ci->progeny[j], NULL, s, with_star_formation, with_timestep_sync);
- for (int k = j + 1; k < 8; k++)
- if (ci->progeny[k] != NULL)
- cell_activate_subcell_stars_tasks(ci->progeny[j], ci->progeny[k],
- s, with_star_formation,
- with_timestep_sync);
- }
- }
- } else {
- /* We have reached the bottom of the tree: activate drift */
- cell_activate_drift_spart(ci, s);
- cell_activate_drift_part(ci, s);
- if (with_timestep_sync) cell_activate_sync_part(ci, s);
- }
- }
-
- /* Otherwise, pair interation */
- else {
-
- /* Get the orientation of the pair. */
- double shift[3];
- const int sid = space_getsid(s->space, &ci, &cj, shift);
-
- const int ci_active = cell_is_active_stars(ci, e) ||
- (with_star_formation && cell_is_active_hydro(ci, e));
- const int cj_active = cell_is_active_stars(cj, e) ||
- (with_star_formation && cell_is_active_hydro(cj, e));
-
- /* Should we even bother? */
- if (!ci_active && !cj_active) return;
-
- /* recurse? */
- if (cell_can_recurse_in_pair_stars_task(ci, cj) &&
- cell_can_recurse_in_pair_stars_task(cj, ci)) {
-
- const struct cell_split_pair *csp = &cell_split_pairs[sid];
- for (int k = 0; k < csp->count; k++) {
- const int pid = csp->pairs[k].pid;
- const int pjd = csp->pairs[k].pjd;
- if (ci->progeny[pid] != NULL && cj->progeny[pjd] != NULL)
- cell_activate_subcell_stars_tasks(ci->progeny[pid], cj->progeny[pjd],
- s, with_star_formation,
- with_timestep_sync);
- }
- }
-
- /* Otherwise, activate the sorts and drifts. */
- else {
-
- if (ci_active) {
-
- /* We are going to interact this pair, so store some values. */
- atomic_or(&cj->hydro.requires_sorts, 1 << sid);
- atomic_or(&ci->stars.requires_sorts, 1 << sid);
-
- cj->hydro.dx_max_sort_old = cj->hydro.dx_max_sort;
- ci->stars.dx_max_sort_old = ci->stars.dx_max_sort;
-
- /* Activate the drifts if the cells are local. */
- if (ci->nodeID == engine_rank) cell_activate_drift_spart(ci, s);
- if (cj->nodeID == engine_rank) cell_activate_drift_part(cj, s);
- if (cj->nodeID == engine_rank && with_timestep_sync)
- cell_activate_sync_part(cj, s);
-
- /* Do we need to sort the cells? */
- cell_activate_hydro_sorts(cj, sid, s);
- cell_activate_stars_sorts(ci, sid, s);
- }
-
- if (cj_active) {
-
- /* We are going to interact this pair, so store some values. */
- atomic_or(&cj->stars.requires_sorts, 1 << sid);
- atomic_or(&ci->hydro.requires_sorts, 1 << sid);
-
- ci->hydro.dx_max_sort_old = ci->hydro.dx_max_sort;
- cj->stars.dx_max_sort_old = cj->stars.dx_max_sort;
-
- /* Activate the drifts if the cells are local. */
- if (ci->nodeID == engine_rank) cell_activate_drift_part(ci, s);
- if (cj->nodeID == engine_rank) cell_activate_drift_spart(cj, s);
- if (ci->nodeID == engine_rank && with_timestep_sync)
- cell_activate_sync_part(ci, s);
-
- /* Do we need to sort the cells? */
- cell_activate_hydro_sorts(ci, sid, s);
- cell_activate_stars_sorts(cj, sid, s);
- }
- }
- } /* Otherwise, pair interation */
-}
-
-/**
- * @brief Traverse a sub-cell task and activate the black_holes drift tasks that
- * are required by a black_holes task
- *
- * @param ci The first #cell we recurse in.
- * @param cj The second #cell we recurse in.
- * @param s The task #scheduler.
- * @param with_timestep_sync Are we running with time-step synchronization on?
- */
-void cell_activate_subcell_black_holes_tasks(struct cell *ci, struct cell *cj,
- struct scheduler *s,
- const int with_timestep_sync) {
- const struct engine *e = s->space->e;
-
- /* Store the current dx_max and h_max values. */
- ci->black_holes.dx_max_part_old = ci->black_holes.dx_max_part;
- ci->black_holes.h_max_old = ci->black_holes.h_max;
- ci->hydro.dx_max_part_old = ci->hydro.dx_max_part;
- ci->hydro.h_max_old = ci->hydro.h_max;
-
- if (cj != NULL) {
- cj->black_holes.dx_max_part_old = cj->black_holes.dx_max_part;
- cj->black_holes.h_max_old = cj->black_holes.h_max;
- cj->hydro.dx_max_part_old = cj->hydro.dx_max_part;
- cj->hydro.h_max_old = cj->hydro.h_max;
- }
-
- /* Self interaction? */
- if (cj == NULL) {
- /* Do anything? */
- if (!cell_is_active_black_holes(ci, e) || ci->hydro.count == 0 ||
- ci->black_holes.count == 0)
- return;
-
- /* Recurse? */
- if (cell_can_recurse_in_self_black_holes_task(ci)) {
- /* Loop over all progenies and pairs of progenies */
- for (int j = 0; j < 8; j++) {
- if (ci->progeny[j] != NULL) {
- cell_activate_subcell_black_holes_tasks(ci->progeny[j], NULL, s,
- with_timestep_sync);
- for (int k = j + 1; k < 8; k++)
- if (ci->progeny[k] != NULL)
- cell_activate_subcell_black_holes_tasks(
- ci->progeny[j], ci->progeny[k], s, with_timestep_sync);
- }
- }
- } else {
- /* We have reached the bottom of the tree: activate drift */
- cell_activate_drift_bpart(ci, s);
- cell_activate_drift_part(ci, s);
- }
- }
-
- /* Otherwise, pair interation */
- else {
- /* Should we even bother? */
- if (!cell_is_active_black_holes(ci, e) &&
- !cell_is_active_black_holes(cj, e))
- return;
-
- /* Get the orientation of the pair. */
- double shift[3];
- const int sid = space_getsid(s->space, &ci, &cj, shift);
-
- /* recurse? */
- if (cell_can_recurse_in_pair_black_holes_task(ci, cj) &&
- cell_can_recurse_in_pair_black_holes_task(cj, ci)) {
- const struct cell_split_pair *csp = &cell_split_pairs[sid];
- for (int k = 0; k < csp->count; k++) {
- const int pid = csp->pairs[k].pid;
- const int pjd = csp->pairs[k].pjd;
- if (ci->progeny[pid] != NULL && cj->progeny[pjd] != NULL)
- cell_activate_subcell_black_holes_tasks(
- ci->progeny[pid], cj->progeny[pjd], s, with_timestep_sync);
- }
- }
-
- /* Otherwise, activate the drifts. */
- else if (cell_is_active_black_holes(ci, e) ||
- cell_is_active_black_holes(cj, e)) {
-
- /* Activate the drifts if the cells are local. */
- if (ci->nodeID == engine_rank) cell_activate_drift_bpart(ci, s);
- if (cj->nodeID == engine_rank) cell_activate_drift_part(cj, s);
- if (cj->nodeID == engine_rank && with_timestep_sync)
- cell_activate_sync_part(cj, s);
-
- /* Activate the drifts if the cells are local. */
- if (ci->nodeID == engine_rank) cell_activate_drift_part(ci, s);
- if (cj->nodeID == engine_rank) cell_activate_drift_bpart(cj, s);
- if (ci->nodeID == engine_rank && with_timestep_sync)
- cell_activate_sync_part(ci, s);
- }
- } /* Otherwise, pair interation */
-}
-
-/**
- * @brief Traverse a sub-cell task and activate the sinks drift tasks that
- * are required by a sinks task
- *
- * @param ci The first #cell we recurse in.
- * @param cj The second #cell we recurse in.
- * @param s The task #scheduler.
- * @param with_timestep_sync Are we running with time-step synchronization on?
- */
-void cell_activate_subcell_sinks_tasks(struct cell *ci, struct cell *cj,
- struct scheduler *s,
- const int with_timestep_sync) {
- const struct engine *e = s->space->e;
-
- /* Store the current dx_max and h_max values. */
- ci->sinks.dx_max_part_old = ci->sinks.dx_max_part;
- ci->sinks.r_cut_max_old = ci->sinks.r_cut_max;
- ci->hydro.dx_max_part_old = ci->hydro.dx_max_part;
- ci->hydro.h_max_old = ci->hydro.h_max;
-
- if (cj != NULL) {
- cj->sinks.dx_max_part_old = cj->sinks.dx_max_part;
- cj->sinks.r_cut_max_old = cj->sinks.r_cut_max;
- cj->hydro.dx_max_part_old = cj->hydro.dx_max_part;
- cj->hydro.h_max_old = cj->hydro.h_max;
- }
-
- /* Self interaction? */
- if (cj == NULL) {
-
- const int ci_active =
- cell_is_active_sinks(ci, e) || cell_is_active_hydro(ci, e);
-
- /* Do anything? */
- if (!ci_active || ci->hydro.count == 0 || ci->sinks.count == 0) return;
-
- /* Recurse? */
- if (cell_can_recurse_in_self_sinks_task(ci)) {
- /* Loop over all progenies and pairs of progenies */
- for (int j = 0; j < 8; j++) {
- if (ci->progeny[j] != NULL) {
- cell_activate_subcell_sinks_tasks(ci->progeny[j], NULL, s,
- with_timestep_sync);
- for (int k = j + 1; k < 8; k++)
- if (ci->progeny[k] != NULL)
- cell_activate_subcell_sinks_tasks(ci->progeny[j], ci->progeny[k],
- s, with_timestep_sync);
- }
- }
- } else {
- /* We have reached the bottom of the tree: activate drift */
- cell_activate_drift_sink(ci, s);
- cell_activate_drift_part(ci, s);
- if (with_timestep_sync) cell_activate_sync_part(ci, s);
- }
- }
-
- /* Otherwise, pair interation */
- else {
- /* Get the orientation of the pair. */
- double shift[3];
- const int sid = space_getsid(s->space, &ci, &cj, shift);
-
- const int ci_active =
- cell_is_active_sinks(ci, e) || cell_is_active_hydro(ci, e);
- const int cj_active =
- cell_is_active_sinks(cj, e) || cell_is_active_hydro(cj, e);
-
- /* Should we even bother? */
- if (!ci_active && !cj_active) return;
-
- /* recurse? */
- if (cell_can_recurse_in_pair_sinks_task(ci, cj) &&
- cell_can_recurse_in_pair_sinks_task(cj, ci)) {
-
- const struct cell_split_pair *csp = &cell_split_pairs[sid];
- for (int k = 0; k < csp->count; k++) {
- const int pid = csp->pairs[k].pid;
- const int pjd = csp->pairs[k].pjd;
- if (ci->progeny[pid] != NULL && cj->progeny[pjd] != NULL)
- cell_activate_subcell_sinks_tasks(ci->progeny[pid], cj->progeny[pjd],
- s, with_timestep_sync);
- }
- }
-
- /* Otherwise, activate the sorts and drifts. */
- else {
-
- if (ci_active) {
-
- /* We are going to interact this pair, so store some values. */
- atomic_or(&cj->hydro.requires_sorts, 1 << sid);
- cj->hydro.dx_max_sort_old = cj->hydro.dx_max_sort;
-
- /* Activate the drifts if the cells are local. */
- if (ci->nodeID == engine_rank) cell_activate_drift_sink(ci, s);
- if (cj->nodeID == engine_rank) cell_activate_drift_part(cj, s);
- if (cj->nodeID == engine_rank && with_timestep_sync)
- cell_activate_sync_part(cj, s);
-
- /* Do we need to sort the cells? */
- cell_activate_hydro_sorts(cj, sid, s);
- }
-
- if (cj_active) {
-
- /* We are going to interact this pair, so store some values. */
- atomic_or(&ci->hydro.requires_sorts, 1 << sid);
- ci->hydro.dx_max_sort_old = ci->hydro.dx_max_sort;
-
- /* Activate the drifts if the cells are local. */
- if (ci->nodeID == engine_rank) cell_activate_drift_part(ci, s);
- if (cj->nodeID == engine_rank) cell_activate_drift_sink(cj, s);
- if (ci->nodeID == engine_rank && with_timestep_sync)
- cell_activate_sync_part(ci, s);
-
- /* Do we need to sort the cells? */
- cell_activate_hydro_sorts(ci, sid, s);
- }
- }
- } /* Otherwise, pair interation */
-}
-
-/**
- * @brief Traverse a sub-cell task and activate the gravity drift tasks that
- * are required by a self gravity task.
- *
- * @param ci The first #cell we recurse in.
- * @param cj The second #cell we recurse in.
- * @param s The task #scheduler.
- */
-void cell_activate_subcell_grav_tasks(struct cell *ci, struct cell *cj,
- struct scheduler *s) {
- /* Some constants */
- const struct space *sp = s->space;
- const struct engine *e = sp->e;
-
- /* Self interaction? */
- if (cj == NULL) {
- /* Do anything? */
- if (ci->grav.count == 0 || !cell_is_active_gravity(ci, e)) return;
-
- /* Recurse? */
- if (ci->split) {
- /* Loop over all progenies and pairs of progenies */
- for (int j = 0; j < 8; j++) {
- if (ci->progeny[j] != NULL) {
- cell_activate_subcell_grav_tasks(ci->progeny[j], NULL, s);
- for (int k = j + 1; k < 8; k++)
- if (ci->progeny[k] != NULL)
- cell_activate_subcell_grav_tasks(ci->progeny[j], ci->progeny[k],
- s);
- }
- }
- } else {
- /* We have reached the bottom of the tree: activate gpart drift */
- cell_activate_drift_gpart(ci, s);
- }
- }
-
- /* Pair interaction */
- else {
- /* Anything to do here? */
- if (!cell_is_active_gravity(ci, e) && !cell_is_active_gravity(cj, e))
- return;
- if (ci->grav.count == 0 || cj->grav.count == 0) return;
-
- /* Atomically drift the multipole in ci */
- lock_lock(&ci->grav.mlock);
- if (ci->grav.ti_old_multipole < e->ti_current) cell_drift_multipole(ci, e);
- if (lock_unlock(&ci->grav.mlock) != 0) error("Impossible to unlock m-pole");
-
- /* Atomically drift the multipole in cj */
- lock_lock(&cj->grav.mlock);
- if (cj->grav.ti_old_multipole < e->ti_current) cell_drift_multipole(cj, e);
- if (lock_unlock(&cj->grav.mlock) != 0) error("Impossible to unlock m-pole");
-
- /* Can we use multipoles ? */
- if (cell_can_use_pair_mm(ci, cj, e, sp, /*use_rebuild_data=*/0,
- /*is_tree_walk=*/1)) {
-
- /* Ok, no need to drift anything */
- return;
- }
- /* Otherwise, activate the gpart drifts if we are at the bottom. */
- else if (!ci->split && !cj->split) {
- /* Activate the drifts if the cells are local. */
- if (cell_is_active_gravity(ci, e) || cell_is_active_gravity(cj, e)) {
- if (ci->nodeID == engine_rank) cell_activate_drift_gpart(ci, s);
- if (cj->nodeID == engine_rank) cell_activate_drift_gpart(cj, s);
- }
- }
- /* Ok, we can still recurse */
- else {
- /* Recover the multipole information */
- const struct gravity_tensors *const multi_i = ci->grav.multipole;
- const struct gravity_tensors *const multi_j = cj->grav.multipole;
- const double ri_max = multi_i->r_max;
- const double rj_max = multi_j->r_max;
-
- if (ri_max > rj_max) {
- if (ci->split) {
- /* Loop over ci's children */
- for (int k = 0; k < 8; k++) {
- if (ci->progeny[k] != NULL)
- cell_activate_subcell_grav_tasks(ci->progeny[k], cj, s);
- }
-
- } else if (cj->split) {
- /* Loop over cj's children */
- for (int k = 0; k < 8; k++) {
- if (cj->progeny[k] != NULL)
- cell_activate_subcell_grav_tasks(ci, cj->progeny[k], s);
- }
-
- } else {
- error("Fundamental error in the logic");
- }
- } else if (rj_max >= ri_max) {
- if (cj->split) {
- /* Loop over cj's children */
- for (int k = 0; k < 8; k++) {
- if (cj->progeny[k] != NULL)
- cell_activate_subcell_grav_tasks(ci, cj->progeny[k], s);
- }
-
- } else if (ci->split) {
- /* Loop over ci's children */
- for (int k = 0; k < 8; k++) {
- if (ci->progeny[k] != NULL)
- cell_activate_subcell_grav_tasks(ci->progeny[k], cj, s);
- }
-
- } else {
- error("Fundamental error in the logic");
- }
- }
- }
- }
-}
-
-/**
- * @brief Traverse a sub-cell task and activate the gravity drift tasks that
- * are required by an external gravity task.
- *
- * @param ci The #cell we recurse in.
- * @param s The task #scheduler.
- */
-void cell_activate_subcell_external_grav_tasks(struct cell *ci,
- struct scheduler *s) {
- /* Some constants */
- const struct space *sp = s->space;
- const struct engine *e = sp->e;
-
- /* Do anything? */
- if (!cell_is_active_gravity(ci, e)) return;
-
- /* Recurse? */
- if (ci->split) {
- /* Loop over all progenies (no need for pairs for self-gravity) */
- for (int j = 0; j < 8; j++) {
- if (ci->progeny[j] != NULL) {
- cell_activate_subcell_external_grav_tasks(ci->progeny[j], s);
- }
- }
- } else {
- /* We have reached the bottom of the tree: activate gpart drift */
- cell_activate_drift_gpart(ci, s);
- }
-}
-
-/**
- * @brief Traverse a sub-cell task and activate the radiative transfer tasks
- *
- * @param ci The first #cell we recurse in.
- * @param cj The second #cell we recurse in.
- * @param s The task #scheduler.
- */
-void cell_activate_subcell_rt_tasks(struct cell *ci, struct cell *cj,
- struct scheduler *s) {
- const struct engine *e = s->space->e;
-
- /* Self interaction? */
- if (cj == NULL) {
- /* Do anything? */
- if (ci->hydro.count == 0 || !cell_is_active_hydro(ci, e)) return;
-
- /* Recurse? */
- if (cell_can_recurse_in_self_hydro_task(ci)) {
- /* Loop over all progenies and pairs of progenies */
- for (int j = 0; j < 8; j++) {
- if (ci->progeny[j] != NULL) {
- cell_activate_subcell_rt_tasks(ci->progeny[j], NULL, s);
- for (int k = j + 1; k < 8; k++)
- if (ci->progeny[k] != NULL)
- cell_activate_subcell_rt_tasks(ci->progeny[j], ci->progeny[k], s);
- }
- }
- } else {
- /* We have reached the bottom of the tree: activate tasks */
- for (struct link *l = ci->hydro.rt_inject; l != NULL; l = l->next) {
- struct task *t = l->t;
- const int ci_active = cell_is_active_hydro(ci, e);
-#ifdef WITH_MPI
- const int ci_nodeID = ci->nodeID;
-#else
- const int ci_nodeID = e->nodeID;
-#endif
- /* Only activate tasks that involve a local active cell. */
- if (ci_active && ci_nodeID == e->nodeID) {
- scheduler_activate(s, t);
- }
- }
- }
- }
-
- /* Otherwise, pair interaction */
- else {
- /* Should we even bother? */
- if (!cell_is_active_hydro(ci, e) && !cell_is_active_hydro(cj, e)) return;
- if (ci->hydro.count == 0 || cj->hydro.count == 0) return;
-
- /* Get the orientation of the pair. */
- double shift[3];
- const int sid = space_getsid(s->space, &ci, &cj, shift);
-
- /* recurse? */
- if (cell_can_recurse_in_pair_hydro_task(ci) &&
- cell_can_recurse_in_pair_hydro_task(cj)) {
- const struct cell_split_pair *csp = &cell_split_pairs[sid];
- for (int k = 0; k < csp->count; k++) {
- const int pid = csp->pairs[k].pid;
- const int pjd = csp->pairs[k].pjd;
- if (ci->progeny[pid] != NULL && cj->progeny[pjd] != NULL)
- cell_activate_subcell_rt_tasks(ci->progeny[pid], cj->progeny[pjd], s);
- }
- }
-
- /* Otherwise, activate the RT tasks. */
- else if (cell_is_active_hydro(ci, e) || cell_is_active_hydro(cj, e)) {
-
- /* Activate the drifts if the cells are local. */
- for (struct link *l = ci->hydro.rt_inject; l != NULL; l = l->next) {
- struct task *t = l->t;
- const int ci_active = cell_is_active_hydro(ci, e);
- const int cj_active = (cj != NULL) ? cell_is_active_hydro(cj, e) : 0;
-#ifdef WITH_MPI
- const int ci_nodeID = ci->nodeID;
- const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
-#else
- const int ci_nodeID = e->nodeID;
- const int cj_nodeID = e->nodeID;
-#endif
-
- /* Only activate tasks that involve a local active cell. */
- if ((ci_active && ci_nodeID == e->nodeID) ||
- (cj_active && cj_nodeID == e->nodeID)) {
- scheduler_activate(s, t);
- }
- }
- }
- }
-}
-
-/**
- * @brief Un-skips all the hydro tasks associated with a given cell and checks
- * if the space needs to be rebuilt.
- *
- * @param c the #cell.
- * @param s the #scheduler.
- *
- * @return 1 If the space needs rebuilding. 0 otherwise.
- */
-int cell_unskip_hydro_tasks(struct cell *c, struct scheduler *s) {
- struct engine *e = s->space->e;
- const int nodeID = e->nodeID;
- const int with_feedback = e->policy & engine_policy_feedback;
- const int with_timestep_limiter =
- (e->policy & engine_policy_timestep_limiter);
-
-#ifdef WITH_MPI
- const int with_star_formation = e->policy & engine_policy_star_formation;
-#endif
- int rebuild = 0;
-
- /* Un-skip the density tasks involved with this cell. */
- for (struct link *l = c->hydro.density; l != NULL; l = l->next) {
- struct task *t = l->t;
- struct cell *ci = t->ci;
- struct cell *cj = t->cj;
- const int ci_active = cell_is_active_hydro(ci, e);
- const int cj_active = (cj != NULL) ? cell_is_active_hydro(cj, e) : 0;
-#ifdef WITH_MPI
- const int ci_nodeID = ci->nodeID;
- const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
-#else
- const int ci_nodeID = nodeID;
- const int cj_nodeID = nodeID;
-#endif
-
- /* Only activate tasks that involve a local active cell. */
- if ((ci_active && ci_nodeID == nodeID) ||
- (cj_active && cj_nodeID == nodeID)) {
- scheduler_activate(s, t);
-
- /* Activate hydro drift */
- if (t->type == task_type_self) {
- if (ci_nodeID == nodeID) cell_activate_drift_part(ci, s);
- if (ci_nodeID == nodeID && with_timestep_limiter)
- cell_activate_limiter(ci, s);
- }
-
- /* Set the correct sorting flags and activate hydro drifts */
- else if (t->type == task_type_pair) {
- /* Store some values. */
- atomic_or(&ci->hydro.requires_sorts, 1 << t->flags);
- atomic_or(&cj->hydro.requires_sorts, 1 << t->flags);
- ci->hydro.dx_max_sort_old = ci->hydro.dx_max_sort;
- cj->hydro.dx_max_sort_old = cj->hydro.dx_max_sort;
-
- /* Activate the drift tasks. */
- if (ci_nodeID == nodeID) cell_activate_drift_part(ci, s);
- if (cj_nodeID == nodeID) cell_activate_drift_part(cj, s);
-
- /* Activate the limiter tasks. */
- if (ci_nodeID == nodeID && with_timestep_limiter)
- cell_activate_limiter(ci, s);
- if (cj_nodeID == nodeID && with_timestep_limiter)
- cell_activate_limiter(cj, s);
-
- /* Check the sorts and activate them if needed. */
- cell_activate_hydro_sorts(ci, t->flags, s);
- cell_activate_hydro_sorts(cj, t->flags, s);
- }
-
- /* Store current values of dx_max and h_max. */
- else if (t->type == task_type_sub_self) {
- cell_activate_subcell_hydro_tasks(ci, NULL, s, with_timestep_limiter);
- }
-
- /* Store current values of dx_max and h_max. */
- else if (t->type == task_type_sub_pair) {
- cell_activate_subcell_hydro_tasks(ci, cj, s, with_timestep_limiter);
- }
- }
-
- /* Only interested in pair interactions as of here. */
- if (t->type == task_type_pair || t->type == task_type_sub_pair) {
- /* Check whether there was too much particle motion, i.e. the
- cell neighbour conditions were violated. */
- if (cell_need_rebuild_for_hydro_pair(ci, cj)) rebuild = 1;
-
-#ifdef WITH_MPI
- /* Activate the send/recv tasks. */
- if (ci_nodeID != nodeID) {
- /* If the local cell is active, receive data from the foreign cell. */
- if (cj_active) {
- scheduler_activate_recv(s, ci->mpi.recv, task_subtype_xv);
- if (ci_active) {
- scheduler_activate_recv(s, ci->mpi.recv, task_subtype_rho);
-
-#ifdef EXTRA_HYDRO_LOOP
- scheduler_activate_recv(s, ci->mpi.recv, task_subtype_gradient);
-#endif
- }
- }
-
- /* If the foreign cell is active, we want its particles for the limiter
- */
- if (ci_active && with_timestep_limiter)
- scheduler_activate_recv(s, ci->mpi.recv, task_subtype_limiter);
-
- /* If the foreign cell is active, we want its ti_end values. */
- if (ci_active)
- scheduler_activate_recv(s, ci->mpi.recv, task_subtype_tend_part);
-
- /* Is the foreign cell active and will need stuff from us? */
- if (ci_active) {
-
- scheduler_activate_send(s, cj->mpi.send, task_subtype_xv, ci_nodeID);
-
- /* Drift the cell which will be sent; note that not all sent
- particles will be drifted, only those that are needed. */
- cell_activate_drift_part(cj, s);
- if (with_timestep_limiter) cell_activate_limiter(cj, s);
-
- /* If the local cell is also active, more stuff will be needed. */
- if (cj_active) {
- scheduler_activate_send(s, cj->mpi.send, task_subtype_rho,
- ci_nodeID);
-
-#ifdef EXTRA_HYDRO_LOOP
- scheduler_activate_send(s, cj->mpi.send, task_subtype_gradient,
- ci_nodeID);
-#endif
- }
- }
-
- /* If the local cell is active, send its particles for the limiting. */
- if (cj_active && with_timestep_limiter)
- scheduler_activate_send(s, cj->mpi.send, task_subtype_limiter,
- ci_nodeID);
-
- /* If the local cell is active, send its ti_end values. */
- if (cj_active)
- scheduler_activate_send(s, cj->mpi.send, task_subtype_tend_part,
- ci_nodeID);
-
- /* Propagating new star counts? */
- if (with_star_formation && with_feedback) {
- if (ci_active && ci->hydro.count > 0) {
- scheduler_activate_recv(s, ci->mpi.recv, task_subtype_sf_counts);
- scheduler_activate_recv(s, ci->mpi.recv, task_subtype_tend_spart);
- }
- if (cj_active && cj->hydro.count > 0) {
- scheduler_activate_send(s, cj->mpi.send, task_subtype_sf_counts,
- ci_nodeID);
- scheduler_activate_send(s, cj->mpi.send, task_subtype_tend_spart,
- ci_nodeID);
- }
- }
-
- } else if (cj_nodeID != nodeID) {
- /* If the local cell is active, receive data from the foreign cell. */
- if (ci_active) {
- scheduler_activate_recv(s, cj->mpi.recv, task_subtype_xv);
- if (cj_active) {
- scheduler_activate_recv(s, cj->mpi.recv, task_subtype_rho);
-
-#ifdef EXTRA_HYDRO_LOOP
- scheduler_activate_recv(s, cj->mpi.recv, task_subtype_gradient);
-#endif
- }
- }
-
- /* If the foreign cell is active, we want its particles for the limiter
- */
- if (cj_active && with_timestep_limiter)
- scheduler_activate_recv(s, cj->mpi.recv, task_subtype_limiter);
-
- /* If the foreign cell is active, we want its ti_end values. */
- if (cj_active)
- scheduler_activate_recv(s, cj->mpi.recv, task_subtype_tend_part);
-
- /* Is the foreign cell active and will need stuff from us? */
- if (cj_active) {
-
- scheduler_activate_send(s, ci->mpi.send, task_subtype_xv, cj_nodeID);
-
- /* Drift the cell which will be sent; note that not all sent
- particles will be drifted, only those that are needed. */
- cell_activate_drift_part(ci, s);
- if (with_timestep_limiter) cell_activate_limiter(ci, s);
-
- /* If the local cell is also active, more stuff will be needed. */
- if (ci_active) {
-
- scheduler_activate_send(s, ci->mpi.send, task_subtype_rho,
- cj_nodeID);
-
-#ifdef EXTRA_HYDRO_LOOP
- scheduler_activate_send(s, ci->mpi.send, task_subtype_gradient,
- cj_nodeID);
-#endif
- }
- }
-
- /* If the local cell is active, send its particles for the limiting. */
- if (ci_active && with_timestep_limiter)
- scheduler_activate_send(s, ci->mpi.send, task_subtype_limiter,
- cj_nodeID);
-
- /* If the local cell is active, send its ti_end values. */
- if (ci_active)
- scheduler_activate_send(s, ci->mpi.send, task_subtype_tend_part,
- cj_nodeID);
-
- /* Propagating new star counts? */
- if (with_star_formation && with_feedback) {
- if (cj_active && cj->hydro.count > 0) {
- scheduler_activate_recv(s, cj->mpi.recv, task_subtype_sf_counts);
- scheduler_activate_recv(s, cj->mpi.recv, task_subtype_tend_spart);
- }
- if (ci_active && ci->hydro.count > 0) {
- scheduler_activate_send(s, ci->mpi.send, task_subtype_sf_counts,
- cj_nodeID);
- scheduler_activate_send(s, ci->mpi.send, task_subtype_tend_spart,
- cj_nodeID);
- }
- }
- }
-#endif
- }
- }
-
- /* Unskip all the other task types. */
- if (c->nodeID == nodeID && cell_is_active_hydro(c, e)) {
- for (struct link *l = c->hydro.gradient; l != NULL; l = l->next)
- scheduler_activate(s, l->t);
- for (struct link *l = c->hydro.force; l != NULL; l = l->next)
- scheduler_activate(s, l->t);
- for (struct link *l = c->hydro.limiter; l != NULL; l = l->next)
- scheduler_activate(s, l->t);
-
- if (c->hydro.extra_ghost != NULL)
- scheduler_activate(s, c->hydro.extra_ghost);
- if (c->hydro.ghost_in != NULL) cell_activate_hydro_ghosts(c, s, e);
- if (c->kick1 != NULL) scheduler_activate(s, c->kick1);
- if (c->kick2 != NULL) scheduler_activate(s, c->kick2);
- if (c->timestep != NULL) scheduler_activate(s, c->timestep);
- if (c->hydro.end_force != NULL) scheduler_activate(s, c->hydro.end_force);
- if (c->hydro.cooling_in != NULL) cell_activate_cooling(c, s, e);
-#ifdef WITH_LOGGER
- if (c->logger != NULL) scheduler_activate(s, c->logger);
-#endif
-
- if (c->top->hydro.star_formation != NULL) {
- cell_activate_star_formation_tasks(c->top, s, with_feedback);
- }
- if (c->top->hydro.sink_formation != NULL) {
- cell_activate_sink_formation_tasks(c->top, s);
- }
- }
-
- return rebuild;
-}
-
-/**
- * @brief Un-skips all the gravity tasks associated with a given cell and checks
- * if the space needs to be rebuilt.
- *
- * @param c the #cell.
- * @param s the #scheduler.
- *
- * @return 1 If the space needs rebuilding. 0 otherwise.
- */
-int cell_unskip_gravity_tasks(struct cell *c, struct scheduler *s) {
- struct engine *e = s->space->e;
- const int nodeID = e->nodeID;
- int rebuild = 0;
-
- /* Un-skip the gravity tasks involved with this cell. */
- for (struct link *l = c->grav.grav; l != NULL; l = l->next) {
- struct task *t = l->t;
- struct cell *ci = t->ci;
- struct cell *cj = t->cj;
- const int ci_active = cell_is_active_gravity(ci, e);
- const int cj_active = (cj != NULL) ? cell_is_active_gravity(cj, e) : 0;
-#ifdef WITH_MPI
- const int ci_nodeID = ci->nodeID;
- const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
-#else
- const int ci_nodeID = nodeID;
- const int cj_nodeID = nodeID;
-#endif
-
- /* Only activate tasks that involve a local active cell. */
- if ((ci_active && ci_nodeID == nodeID) ||
- (cj_active && cj_nodeID == nodeID)) {
- scheduler_activate(s, t);
-
- /* Set the drifting flags */
- if (t->type == task_type_self &&
- t->subtype == task_subtype_external_grav) {
- cell_activate_subcell_external_grav_tasks(ci, s);
- } else if (t->type == task_type_self && t->subtype == task_subtype_grav) {
- cell_activate_subcell_grav_tasks(ci, NULL, s);
- } else if (t->type == task_type_pair) {
- cell_activate_subcell_grav_tasks(ci, cj, s);
- } else if (t->type == task_type_grav_mm) {
-#ifdef SWIFT_DEBUG_CHECKS
- error("Incorrectly linked M-M task!");
-#endif
- }
- }
-
- if (t->type == task_type_pair) {
-#ifdef WITH_MPI
- /* Activate the send/recv tasks. */
- if (ci_nodeID != nodeID) {
- /* If the local cell is active, receive data from the foreign cell. */
- if (cj_active)
- scheduler_activate_recv(s, ci->mpi.recv, task_subtype_gpart);
-
- /* If the foreign cell is active, we want its ti_end values. */
- if (ci_active)
- scheduler_activate_recv(s, ci->mpi.recv, task_subtype_tend_gpart);
-
- /* Is the foreign cell active and will need stuff from us? */
- if (ci_active) {
-
- scheduler_activate_send(s, cj->mpi.send, task_subtype_gpart,
- ci_nodeID);
-
- /* Drift the cell which will be sent at the level at which it is
- sent, i.e. drift the cell specified in the send task (l->t)
- itself. */
- cell_activate_drift_gpart(cj, s);
- }
-
- /* If the local cell is active, send its ti_end values. */
- if (cj_active)
- scheduler_activate_send(s, cj->mpi.send, task_subtype_tend_gpart,
- ci_nodeID);
-
- } else if (cj_nodeID != nodeID) {
- /* If the local cell is active, receive data from the foreign cell. */
- if (ci_active)
- scheduler_activate_recv(s, cj->mpi.recv, task_subtype_gpart);
-
- /* If the foreign cell is active, we want its ti_end values. */
- if (cj_active)
- scheduler_activate_recv(s, cj->mpi.recv, task_subtype_tend_gpart);
-
- /* Is the foreign cell active and will need stuff from us? */
- if (cj_active) {
-
- scheduler_activate_send(s, ci->mpi.send, task_subtype_gpart,
- cj_nodeID);
-
- /* Drift the cell which will be sent at the level at which it is
- sent, i.e. drift the cell specified in the send task (l->t)
- itself. */
- cell_activate_drift_gpart(ci, s);
- }
-
- /* If the local cell is active, send its ti_end values. */
- if (ci_active)
- scheduler_activate_send(s, ci->mpi.send, task_subtype_tend_gpart,
- cj_nodeID);
- }
-#endif
- }
- }
-
- for (struct link *l = c->grav.mm; l != NULL; l = l->next) {
- struct task *t = l->t;
- struct cell *ci = t->ci;
- struct cell *cj = t->cj;
- const int ci_active = cell_is_active_gravity_mm(ci, e);
- const int cj_active = cell_is_active_gravity_mm(cj, e);
-#ifdef WITH_MPI
- const int ci_nodeID = ci->nodeID;
- const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
-#else
- const int ci_nodeID = nodeID;
- const int cj_nodeID = nodeID;
-#endif
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (t->type != task_type_grav_mm) error("Incorrectly linked gravity task!");
-#endif
-
- /* Only activate tasks that involve a local active cell. */
- if ((ci_active && ci_nodeID == nodeID) ||
- (cj_active && cj_nodeID == nodeID)) {
- scheduler_activate(s, t);
- }
- }
-
- /* Unskip all the other task types. */
- if (c->nodeID == nodeID && cell_is_active_gravity(c, e)) {
- if (c->grav.init != NULL) scheduler_activate(s, c->grav.init);
- if (c->grav.init_out != NULL) scheduler_activate(s, c->grav.init_out);
- if (c->kick1 != NULL) scheduler_activate(s, c->kick1);
- if (c->kick2 != NULL) scheduler_activate(s, c->kick2);
- if (c->timestep != NULL) scheduler_activate(s, c->timestep);
- if (c->grav.down != NULL) scheduler_activate(s, c->grav.down);
- if (c->grav.down_in != NULL) scheduler_activate(s, c->grav.down_in);
- if (c->grav.long_range != NULL) scheduler_activate(s, c->grav.long_range);
- if (c->grav.end_force != NULL) scheduler_activate(s, c->grav.end_force);
-#ifdef WITH_LOGGER
- if (c->logger != NULL) scheduler_activate(s, c->logger);
-#endif
- }
-
- return rebuild;
-}
-
-/**
- * @brief Un-skips all the stars tasks associated with a given cell and checks
- * if the space needs to be rebuilt.
- *
- * @param c the #cell.
- * @param s the #scheduler.
- * @param with_star_formation Are we running with star formation switched on?
- *
- * @return 1 If the space needs rebuilding. 0 otherwise.
- */
-int cell_unskip_stars_tasks(struct cell *c, struct scheduler *s,
- const int with_star_formation) {
-
- struct engine *e = s->space->e;
- const int with_timestep_sync = (e->policy & engine_policy_timestep_sync);
- const int nodeID = e->nodeID;
- int rebuild = 0;
-
- if (c->stars.drift != NULL) {
- if (cell_is_active_stars(c, e) ||
- (with_star_formation && cell_is_active_hydro(c, e))) {
-
- cell_activate_drift_spart(c, s);
- }
- }
-
- /* Un-skip the density tasks involved with this cell. */
- for (struct link *l = c->stars.density; l != NULL; l = l->next) {
- struct task *t = l->t;
- struct cell *ci = t->ci;
- struct cell *cj = t->cj;
-#ifdef WITH_MPI
- const int ci_nodeID = ci->nodeID;
- const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
-#else
- const int ci_nodeID = nodeID;
- const int cj_nodeID = nodeID;
-#endif
-
- const int ci_active = cell_is_active_stars(ci, e) ||
- (with_star_formation && cell_is_active_hydro(ci, e));
-
- const int cj_active =
- (cj != NULL) && (cell_is_active_stars(cj, e) ||
- (with_star_formation && cell_is_active_hydro(cj, e)));
-
- /* Activate the drifts */
- if (t->type == task_type_self && ci_active) {
- cell_activate_drift_spart(ci, s);
- cell_activate_drift_part(ci, s);
- if (with_timestep_sync) cell_activate_sync_part(ci, s);
- }
-
- /* Only activate tasks that involve a local active cell. */
- if ((ci_active || cj_active) &&
- (ci_nodeID == nodeID || cj_nodeID == nodeID)) {
- scheduler_activate(s, t);
-
- if (t->type == task_type_pair) {
- /* Do ci */
- if (ci_active) {
- /* stars for ci */
- atomic_or(&ci->stars.requires_sorts, 1 << t->flags);
- ci->stars.dx_max_sort_old = ci->stars.dx_max_sort;
-
- /* hydro for cj */
- atomic_or(&cj->hydro.requires_sorts, 1 << t->flags);
- cj->hydro.dx_max_sort_old = cj->hydro.dx_max_sort;
-
- /* Activate the drift tasks. */
- if (ci_nodeID == nodeID) cell_activate_drift_spart(ci, s);
- if (cj_nodeID == nodeID) cell_activate_drift_part(cj, s);
- if (cj_nodeID == nodeID && with_timestep_sync)
- cell_activate_sync_part(cj, s);
-
- /* Check the sorts and activate them if needed. */
- cell_activate_stars_sorts(ci, t->flags, s);
- cell_activate_hydro_sorts(cj, t->flags, s);
- }
-
- /* Do cj */
- if (cj_active) {
- /* hydro for ci */
- atomic_or(&ci->hydro.requires_sorts, 1 << t->flags);
- ci->hydro.dx_max_sort_old = ci->hydro.dx_max_sort;
-
- /* stars for cj */
- atomic_or(&cj->stars.requires_sorts, 1 << t->flags);
- cj->stars.dx_max_sort_old = cj->stars.dx_max_sort;
-
- /* Activate the drift tasks. */
- if (cj_nodeID == nodeID) cell_activate_drift_spart(cj, s);
- if (ci_nodeID == nodeID) cell_activate_drift_part(ci, s);
- if (ci_nodeID == nodeID && with_timestep_sync)
- cell_activate_sync_part(ci, s);
-
- /* Check the sorts and activate them if needed. */
- cell_activate_hydro_sorts(ci, t->flags, s);
- cell_activate_stars_sorts(cj, t->flags, s);
- }
- }
-
- else if (t->type == task_type_sub_self) {
- cell_activate_subcell_stars_tasks(ci, NULL, s, with_star_formation,
- with_timestep_sync);
- }
-
- else if (t->type == task_type_sub_pair) {
- cell_activate_subcell_stars_tasks(ci, cj, s, with_star_formation,
- with_timestep_sync);
- }
- }
-
- /* Only interested in pair interactions as of here. */
- if (t->type == task_type_pair || t->type == task_type_sub_pair) {
- /* Check whether there was too much particle motion, i.e. the
- cell neighbour conditions were violated. */
- if (cell_need_rebuild_for_stars_pair(ci, cj)) rebuild = 1;
- if (cell_need_rebuild_for_stars_pair(cj, ci)) rebuild = 1;
-
-#ifdef WITH_MPI
- /* Activate the send/recv tasks. */
- if (ci_nodeID != nodeID) {
- if (cj_active) {
- scheduler_activate_recv(s, ci->mpi.recv, task_subtype_xv);
- scheduler_activate_recv(s, ci->mpi.recv, task_subtype_rho);
-
- /* If the local cell is active, more stuff will be needed. */
- scheduler_activate_send(s, cj->mpi.send, task_subtype_spart,
- ci_nodeID);
- cell_activate_drift_spart(cj, s);
-
- /* If the local cell is active, send its ti_end values. */
- scheduler_activate_send(s, cj->mpi.send, task_subtype_tend_spart,
- ci_nodeID);
- }
-
- if (ci_active) {
- scheduler_activate_recv(s, ci->mpi.recv, task_subtype_spart);
-
- /* If the foreign cell is active, we want its ti_end values. */
- scheduler_activate_recv(s, ci->mpi.recv, task_subtype_tend_spart);
-
- /* Is the foreign cell active and will need stuff from us? */
- scheduler_activate_send(s, cj->mpi.send, task_subtype_xv, ci_nodeID);
- scheduler_activate_send(s, cj->mpi.send, task_subtype_rho, ci_nodeID);
-
- /* Drift the cell which will be sent; note that not all sent
- particles will be drifted, only those that are needed. */
- cell_activate_drift_part(cj, s);
- }
-
- } else if (cj_nodeID != nodeID) {
- /* If the local cell is active, receive data from the foreign cell. */
- if (ci_active) {
- scheduler_activate_recv(s, cj->mpi.recv, task_subtype_xv);
- scheduler_activate_recv(s, cj->mpi.recv, task_subtype_rho);
-
- /* If the local cell is active, more stuff will be needed. */
- scheduler_activate_send(s, ci->mpi.send, task_subtype_spart,
- cj_nodeID);
- cell_activate_drift_spart(ci, s);
-
- /* If the local cell is active, send its ti_end values. */
- scheduler_activate_send(s, ci->mpi.send, task_subtype_tend_spart,
- cj_nodeID);
- }
-
- if (cj_active) {
- scheduler_activate_recv(s, cj->mpi.recv, task_subtype_spart);
-
- /* If the foreign cell is active, we want its ti_end values. */
- scheduler_activate_recv(s, cj->mpi.recv, task_subtype_tend_spart);
-
- /* Is the foreign cell active and will need stuff from us? */
- scheduler_activate_send(s, ci->mpi.send, task_subtype_xv, cj_nodeID);
- scheduler_activate_send(s, ci->mpi.send, task_subtype_rho, cj_nodeID);
-
- /* Drift the cell which will be sent; note that not all sent
- particles will be drifted, only those that are needed. */
- cell_activate_drift_part(ci, s);
- }
- }
-#endif
- }
- }
-
- /* Un-skip the feedback tasks involved with this cell. */
- for (struct link *l = c->stars.feedback; l != NULL; l = l->next) {
- struct task *t = l->t;
- struct cell *ci = t->ci;
- struct cell *cj = t->cj;
-#ifdef WITH_MPI
- const int ci_nodeID = ci->nodeID;
- const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
-#else
- const int ci_nodeID = nodeID;
- const int cj_nodeID = nodeID;
-#endif
-
- const int ci_active = cell_is_active_stars(ci, e) ||
- (with_star_formation && cell_is_active_hydro(ci, e));
-
- const int cj_active =
- (cj != NULL) && (cell_is_active_stars(cj, e) ||
- (with_star_formation && cell_is_active_hydro(cj, e)));
-
- if (t->type == task_type_self && ci_active) {
- scheduler_activate(s, t);
- }
-
- else if (t->type == task_type_sub_self && ci_active) {
- scheduler_activate(s, t);
- }
-
- else if (t->type == task_type_pair || t->type == task_type_sub_pair) {
- /* We only want to activate the task if the cell is active and is
- going to update some gas on the *local* node */
- if ((ci_nodeID == nodeID && cj_nodeID == nodeID) &&
- (ci_active || cj_active)) {
- scheduler_activate(s, t);
-
- } else if ((ci_nodeID == nodeID && cj_nodeID != nodeID) && (cj_active)) {
- scheduler_activate(s, t);
-
- } else if ((ci_nodeID != nodeID && cj_nodeID == nodeID) && (ci_active)) {
- scheduler_activate(s, t);
- }
- }
-
- /* Nothing more to do here, all drifts and sorts activated above */
- }
-
- /* Unskip all the other task types. */
- if (c->nodeID == nodeID) {
- if (cell_is_active_stars(c, e) ||
- (with_star_formation && cell_is_active_hydro(c, e))) {
-
- if (c->stars.ghost != NULL) scheduler_activate(s, c->stars.ghost);
- if (c->stars.stars_in != NULL) scheduler_activate(s, c->stars.stars_in);
- if (c->stars.stars_out != NULL) scheduler_activate(s, c->stars.stars_out);
- if (c->kick1 != NULL) scheduler_activate(s, c->kick1);
- if (c->kick2 != NULL) scheduler_activate(s, c->kick2);
- if (c->timestep != NULL) scheduler_activate(s, c->timestep);
-#ifdef WITH_LOGGER
- if (c->logger != NULL) scheduler_activate(s, c->logger);
-#endif
- }
- }
-
- return rebuild;
-}
-
-/**
- * @brief Un-skips all the black hole tasks associated with a given cell and
- * checks if the space needs to be rebuilt.
- *
- * @param c the #cell.
- * @param s the #scheduler.
- *
- * @return 1 If the space needs rebuilding. 0 otherwise.
- */
-int cell_unskip_black_holes_tasks(struct cell *c, struct scheduler *s) {
-
- struct engine *e = s->space->e;
- const int with_timestep_sync = (e->policy & engine_policy_timestep_sync);
- const int nodeID = e->nodeID;
- int rebuild = 0;
-
- if (c->black_holes.drift != NULL && cell_is_active_black_holes(c, e)) {
- cell_activate_drift_bpart(c, s);
- }
-
- /* Un-skip the density tasks involved with this cell. */
- for (struct link *l = c->black_holes.density; l != NULL; l = l->next) {
- struct task *t = l->t;
- struct cell *ci = t->ci;
- struct cell *cj = t->cj;
- const int ci_active = cell_is_active_black_holes(ci, e);
- const int cj_active = (cj != NULL) ? cell_is_active_black_holes(cj, e) : 0;
-#ifdef WITH_MPI
- const int ci_nodeID = ci->nodeID;
- const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
-#else
- const int ci_nodeID = nodeID;
- const int cj_nodeID = nodeID;
-#endif
-
- /* Only activate tasks that involve a local active cell. */
- if ((ci_active || cj_active) &&
- (ci_nodeID == nodeID || cj_nodeID == nodeID)) {
-
- scheduler_activate(s, t);
-
- /* Activate the drifts */
- if (t->type == task_type_self) {
- cell_activate_drift_part(ci, s);
- cell_activate_drift_bpart(ci, s);
- }
-
- /* Activate the drifts */
- else if (t->type == task_type_pair) {
-
- /* Activate the drift tasks. */
- if (ci_nodeID == nodeID) cell_activate_drift_bpart(ci, s);
- if (ci_nodeID == nodeID) cell_activate_drift_part(ci, s);
-
- if (cj_nodeID == nodeID) cell_activate_drift_part(cj, s);
- if (cj_nodeID == nodeID) cell_activate_drift_bpart(cj, s);
- }
-
- /* Store current values of dx_max and h_max. */
- else if (t->type == task_type_sub_self) {
- cell_activate_subcell_black_holes_tasks(ci, NULL, s,
- with_timestep_sync);
- }
-
- /* Store current values of dx_max and h_max. */
- else if (t->type == task_type_sub_pair) {
- cell_activate_subcell_black_holes_tasks(ci, cj, s, with_timestep_sync);
- }
- }
-
- /* Only interested in pair interactions as of here. */
- if (t->type == task_type_pair || t->type == task_type_sub_pair) {
-
- /* Check whether there was too much particle motion, i.e. the
- cell neighbour conditions were violated. */
- if (cell_need_rebuild_for_black_holes_pair(ci, cj)) rebuild = 1;
- if (cell_need_rebuild_for_black_holes_pair(cj, ci)) rebuild = 1;
-
- scheduler_activate(s, ci->hydro.super->black_holes.swallow_ghost[0]);
- scheduler_activate(s, cj->hydro.super->black_holes.swallow_ghost[0]);
-
-#ifdef WITH_MPI
- /* Activate the send/recv tasks. */
- if (ci_nodeID != nodeID) {
-
- /* Receive the foreign parts to compute BH accretion rates and do the
- * swallowing */
- scheduler_activate_recv(s, ci->mpi.recv, task_subtype_rho);
- scheduler_activate_recv(s, ci->mpi.recv, task_subtype_part_swallow);
-
- /* Send the local BHs to tag the particles to swallow and to do feedback
- */
- scheduler_activate_send(s, cj->mpi.send, task_subtype_bpart_rho,
- ci_nodeID);
- scheduler_activate_send(s, cj->mpi.send, task_subtype_bpart_feedback,
- ci_nodeID);
-
- /* Drift before you send */
- cell_activate_drift_bpart(cj, s);
-
- /* Send the new BH time-steps */
- scheduler_activate_send(s, cj->mpi.send, task_subtype_tend_bpart,
- ci_nodeID);
-
- /* Receive the foreign BHs to tag particles to swallow and for feedback
- */
- scheduler_activate_recv(s, ci->mpi.recv, task_subtype_bpart_rho);
- scheduler_activate_recv(s, ci->mpi.recv, task_subtype_bpart_feedback);
-
- /* Receive the foreign BH time-steps */
- scheduler_activate_recv(s, ci->mpi.recv, task_subtype_tend_bpart);
-
- /* Send the local part information */
- scheduler_activate_send(s, cj->mpi.send, task_subtype_rho, ci_nodeID);
- scheduler_activate_send(s, cj->mpi.send, task_subtype_part_swallow,
- ci_nodeID);
-
- /* Drift the cell which will be sent; note that not all sent
- particles will be drifted, only those that are needed. */
- cell_activate_drift_part(cj, s);
-
- } else if (cj_nodeID != nodeID) {
-
- /* Receive the foreign parts to compute BH accretion rates and do the
- * swallowing */
- scheduler_activate_recv(s, cj->mpi.recv, task_subtype_rho);
- scheduler_activate_recv(s, cj->mpi.recv, task_subtype_part_swallow);
-
- /* Send the local BHs to tag the particles to swallow and to do feedback
- */
- scheduler_activate_send(s, ci->mpi.send, task_subtype_bpart_rho,
- cj_nodeID);
- scheduler_activate_send(s, ci->mpi.send, task_subtype_bpart_feedback,
- cj_nodeID);
-
- /* Drift before you send */
- cell_activate_drift_bpart(ci, s);
-
- /* Send the new BH time-steps */
- scheduler_activate_send(s, ci->mpi.send, task_subtype_tend_bpart,
- cj_nodeID);
-
- /* Receive the foreign BHs to tag particles to swallow and for feedback
- */
- scheduler_activate_recv(s, cj->mpi.recv, task_subtype_bpart_rho);
- scheduler_activate_recv(s, cj->mpi.recv, task_subtype_bpart_feedback);
-
- /* Receive the foreign BH time-steps */
- scheduler_activate_recv(s, cj->mpi.recv, task_subtype_tend_bpart);
-
- /* Send the local part information */
- scheduler_activate_send(s, ci->mpi.send, task_subtype_rho, cj_nodeID);
- scheduler_activate_send(s, ci->mpi.send, task_subtype_part_swallow,
- cj_nodeID);
-
- /* Drift the cell which will be sent; note that not all sent
- particles will be drifted, only those that are needed. */
- cell_activate_drift_part(ci, s);
- }
-#endif
- }
- }
-
- /* Un-skip the swallow tasks involved with this cell. */
- for (struct link *l = c->black_holes.swallow; l != NULL; l = l->next) {
- struct task *t = l->t;
- struct cell *ci = t->ci;
- struct cell *cj = t->cj;
- const int ci_active = cell_is_active_black_holes(ci, e);
- const int cj_active = (cj != NULL) ? cell_is_active_black_holes(cj, e) : 0;
-#ifdef WITH_MPI
- const int ci_nodeID = ci->nodeID;
- const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
-#else
- const int ci_nodeID = nodeID;
- const int cj_nodeID = nodeID;
-#endif
-
- /* Only activate tasks that involve a local active cell. */
- if ((ci_active || cj_active) &&
- (ci_nodeID == nodeID || cj_nodeID == nodeID)) {
-
- scheduler_activate(s, t);
- }
- }
-
- /* Un-skip the swallow tasks involved with this cell. */
- for (struct link *l = c->black_holes.do_gas_swallow; l != NULL; l = l->next) {
- struct task *t = l->t;
- struct cell *ci = t->ci;
- struct cell *cj = t->cj;
- const int ci_active = cell_is_active_black_holes(ci, e);
- const int cj_active = (cj != NULL) ? cell_is_active_black_holes(cj, e) : 0;
-#ifdef WITH_MPI
- const int ci_nodeID = ci->nodeID;
- const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
-#else
- const int ci_nodeID = nodeID;
- const int cj_nodeID = nodeID;
-#endif
-
- /* Only activate tasks that involve a local active cell. */
- if ((ci_active || cj_active) &&
- (ci_nodeID == nodeID || cj_nodeID == nodeID)) {
-
- scheduler_activate(s, t);
- }
- }
-
- /* Un-skip the swallow tasks involved with this cell. */
- for (struct link *l = c->black_holes.do_bh_swallow; l != NULL; l = l->next) {
- struct task *t = l->t;
- struct cell *ci = t->ci;
- struct cell *cj = t->cj;
- const int ci_active = cell_is_active_black_holes(ci, e);
- const int cj_active = (cj != NULL) ? cell_is_active_black_holes(cj, e) : 0;
-#ifdef WITH_MPI
- const int ci_nodeID = ci->nodeID;
- const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
-#else
- const int ci_nodeID = nodeID;
- const int cj_nodeID = nodeID;
-#endif
-
- /* Only activate tasks that involve a local active cell. */
- if ((ci_active || cj_active) &&
- (ci_nodeID == nodeID || cj_nodeID == nodeID)) {
-
- scheduler_activate(s, t);
- }
- }
-
- /* Un-skip the feedback tasks involved with this cell. */
- for (struct link *l = c->black_holes.feedback; l != NULL; l = l->next) {
- struct task *t = l->t;
- struct cell *ci = t->ci;
- struct cell *cj = t->cj;
- const int ci_active = cell_is_active_black_holes(ci, e);
- const int cj_active = (cj != NULL) ? cell_is_active_black_holes(cj, e) : 0;
-#ifdef WITH_MPI
- const int ci_nodeID = ci->nodeID;
- const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
-#else
- const int ci_nodeID = nodeID;
- const int cj_nodeID = nodeID;
-#endif
-
- /* Only activate tasks that involve a local active cell. */
- if ((ci_active || cj_active) &&
- (ci_nodeID == nodeID || cj_nodeID == nodeID)) {
-
- scheduler_activate(s, t);
- }
- }
-
- /* Unskip all the other task types. */
- if (c->nodeID == nodeID && cell_is_active_black_holes(c, e)) {
-
- if (c->black_holes.density_ghost != NULL)
- scheduler_activate(s, c->black_holes.density_ghost);
- if (c->black_holes.swallow_ghost[0] != NULL)
- scheduler_activate(s, c->black_holes.swallow_ghost[0]);
- if (c->black_holes.swallow_ghost[1] != NULL)
- scheduler_activate(s, c->black_holes.swallow_ghost[1]);
- if (c->black_holes.swallow_ghost[2] != NULL)
- scheduler_activate(s, c->black_holes.swallow_ghost[2]);
- if (c->black_holes.black_holes_in != NULL)
- scheduler_activate(s, c->black_holes.black_holes_in);
- if (c->black_holes.black_holes_out != NULL)
- scheduler_activate(s, c->black_holes.black_holes_out);
- if (c->kick1 != NULL) scheduler_activate(s, c->kick1);
- if (c->kick2 != NULL) scheduler_activate(s, c->kick2);
- if (c->timestep != NULL) scheduler_activate(s, c->timestep);
-#ifdef WITH_LOGGER
- if (c->logger != NULL) scheduler_activate(s, c->logger);
-#endif
- }
-
- return rebuild;
-}
-
-/**
- * @brief Un-skips all the sinks tasks associated with a given cell and
- * checks if the space needs to be rebuilt.
- *
- * @param c the #cell.
- * @param s the #scheduler.
- *
- * @return 1 If the space needs rebuilding. 0 otherwise.
- */
-int cell_unskip_sinks_tasks(struct cell *c, struct scheduler *s) {
-
- struct engine *e = s->space->e;
- const int nodeID = e->nodeID;
- int rebuild = 0;
-
- if (c->sinks.drift != NULL)
- if (cell_is_active_sinks(c, e) || cell_is_active_hydro(c, e)) {
- cell_activate_drift_sink(c, s);
- }
-
- /* Unskip all the other task types. */
- if (c->nodeID == nodeID)
- if (cell_is_active_sinks(c, e) || cell_is_active_hydro(c, e)) {
- if (c->sinks.sink_in != NULL) scheduler_activate(s, c->sinks.sink_in);
- if (c->sinks.sink_out != NULL) scheduler_activate(s, c->sinks.sink_out);
- if (c->kick1 != NULL) scheduler_activate(s, c->kick1);
- if (c->kick2 != NULL) scheduler_activate(s, c->kick2);
- if (c->timestep != NULL) scheduler_activate(s, c->timestep);
-#ifdef WITH_LOGGER
- if (c->logger != NULL) scheduler_activate(s, c->logger);
-#endif
- }
-
- return rebuild;
-}
-
-/**
- * @brief Un-skips all the RT tasks associated with a given cell and checks
- * if the space needs to be rebuilt.
- *
- * @param c the #cell.
- * @param s the #scheduler.
- *
- * @return 1 If the space needs rebuilding. 0 otherwise.
- */
-int cell_unskip_rt_tasks(struct cell *c, struct scheduler *s) {
- struct engine *e = s->space->e;
- const int nodeID = e->nodeID;
-
- /* TODO: implement rebuild conditions */
- int rebuild = 0;
-
- for (struct link *l = c->hydro.rt_inject; l != NULL; l = l->next) {
- struct task *t = l->t;
- struct cell *ci = t->ci;
- struct cell *cj = t->cj;
- const int ci_active = cell_is_active_hydro(ci, e);
- const int cj_active = (cj != NULL) ? cell_is_active_hydro(cj, e) : 0;
-#ifdef WITH_MPI
- const int ci_nodeID = ci->nodeID;
- const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
-#else
- const int ci_nodeID = nodeID;
- const int cj_nodeID = nodeID;
-#endif
-
- /* Only activate tasks that involve a local active cell. */
- if ((ci_active && ci_nodeID == nodeID) ||
- (cj_active && cj_nodeID == nodeID)) {
-
- scheduler_activate(s, t);
-
- if (t->type == task_type_sub_self) {
- cell_activate_subcell_rt_tasks(ci, NULL, s);
- }
-
- else if (t->type == task_type_sub_pair) {
- cell_activate_subcell_rt_tasks(ci, cj, s);
- }
- }
- }
-
- /* Unskip all the other task types */
- if (c->nodeID == nodeID) {
- if (cell_is_active_hydro(c, e)) {
- if (c->hydro.rt_in != NULL) scheduler_activate(s, c->hydro.rt_in);
- if (c->hydro.rt_ghost1 != NULL) scheduler_activate(s, c->hydro.rt_ghost1);
- if (c->hydro.rt_out != NULL) scheduler_activate(s, c->hydro.rt_out);
- }
- }
-
- return rebuild;
-}
-
-/**
- * @brief Set the super-cell pointers for all cells in a hierarchy.
- *
- * @param c The top-level #cell to play with.
- * @param super Pointer to the deepest cell with tasks in this part of the
- * tree.
- * @param with_hydro Are we running with hydrodynamics on?
- * @param with_grav Are we running with gravity on?
- */
-void cell_set_super(struct cell *c, struct cell *super, const int with_hydro,
- const int with_grav) {
- /* Are we in a cell which is either the hydro or gravity super? */
- if (super == NULL && ((with_hydro && c->hydro.super != NULL) ||
- (with_grav && c->grav.super != NULL)))
- super = c;
-
- /* Set the super-cell */
- c->super = super;
-
- /* Recurse */
- if (c->split)
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL)
- cell_set_super(c->progeny[k], super, with_hydro, with_grav);
-}
-
-/**
- * @brief Set the super-cell pointers for all cells in a hierarchy.
- *
- * @param c The top-level #cell to play with.
- * @param super_hydro Pointer to the deepest cell with tasks in this part of
- * the tree.
- */
-void cell_set_super_hydro(struct cell *c, struct cell *super_hydro) {
- /* Are we in a cell with some kind of self/pair task ? */
- if (super_hydro == NULL && c->hydro.density != NULL) super_hydro = c;
-
- /* Set the super-cell */
- c->hydro.super = super_hydro;
-
- /* Recurse */
- if (c->split)
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL)
- cell_set_super_hydro(c->progeny[k], super_hydro);
-}
-
-/**
- * @brief Set the super-cell pointers for all cells in a hierarchy.
- *
- * @param c The top-level #cell to play with.
- * @param super_gravity Pointer to the deepest cell with tasks in this part of
- * the tree.
- */
-void cell_set_super_gravity(struct cell *c, struct cell *super_gravity) {
- /* Are we in a cell with some kind of self/pair task ? */
- if (super_gravity == NULL && (c->grav.grav != NULL || c->grav.mm != NULL))
- super_gravity = c;
-
- /* Set the super-cell */
- c->grav.super = super_gravity;
-
- /* Recurse */
- if (c->split)
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL)
- cell_set_super_gravity(c->progeny[k], super_gravity);
-}
-
-/**
- * @brief Mapper function to set the super pointer of the cells.
- *
- * @param map_data The top-level cells.
- * @param num_elements The number of top-level cells.
- * @param extra_data Unused parameter.
- */
-void cell_set_super_mapper(void *map_data, int num_elements, void *extra_data) {
- const struct engine *e = (const struct engine *)extra_data;
-
- const int with_hydro = (e->policy & engine_policy_hydro);
- const int with_grav = (e->policy & engine_policy_self_gravity) ||
- (e->policy & engine_policy_external_gravity);
-
- for (int ind = 0; ind < num_elements; ind++) {
- struct cell *c = &((struct cell *)map_data)[ind];
-
- /* All top-level cells get an MPI tag. */
-#ifdef WITH_MPI
- cell_ensure_tagged(c);
-#endif
-
- /* Super-pointer for hydro */
- if (with_hydro) cell_set_super_hydro(c, NULL);
-
- /* Super-pointer for gravity */
- if (with_grav) cell_set_super_gravity(c, NULL);
-
- /* Super-pointer for common operations */
- cell_set_super(c, NULL, with_hydro, with_grav);
- }
-}
-
-/**
- * @brief Does this cell or any of its children have any task ?
- *
- * We use the timestep-related tasks to probe this as these always
- * exist in a cell hierarchy that has any kind of task.
- *
- * @param c The #cell to probe.
- */
-int cell_has_tasks(struct cell *c) {
-#ifdef WITH_MPI
- if (c->timestep != NULL || c->mpi.recv != NULL) return 1;
-#else
- if (c->timestep != NULL) return 1;
-#endif
-
- if (c->split) {
- int count = 0;
- for (int k = 0; k < 8; ++k)
- if (c->progeny[k] != NULL) count += cell_has_tasks(c->progeny[k]);
- return count;
- } else {
- return 0;
- }
-}
-
-/**
- * @brief Recursively drifts the #part in a cell hierarchy.
- *
- * @param c The #cell.
- * @param e The #engine (to get ti_current).
- * @param force Drift the particles irrespective of the #cell flags.
- */
-void cell_drift_part(struct cell *c, const struct engine *e, int force) {
- const int periodic = e->s->periodic;
- const double dim[3] = {e->s->dim[0], e->s->dim[1], e->s->dim[2]};
- const int with_cosmology = (e->policy & engine_policy_cosmology);
- const float hydro_h_max = e->hydro_properties->h_max;
- const float hydro_h_min = e->hydro_properties->h_min;
- const integertime_t ti_old_part = c->hydro.ti_old_part;
- const integertime_t ti_current = e->ti_current;
- struct part *const parts = c->hydro.parts;
- struct xpart *const xparts = c->hydro.xparts;
-
- float dx_max = 0.f, dx2_max = 0.f;
- float dx_max_sort = 0.0f, dx2_max_sort = 0.f;
- float cell_h_max = 0.f;
-
- /* Drift irrespective of cell flags? */
- force = (force || cell_get_flag(c, cell_flag_do_hydro_drift));
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Check that we only drift local cells. */
- if (c->nodeID != engine_rank) error("Drifting a foreign cell is nope.");
-
- /* Check that we are actually going to move forward. */
- if (ti_current < ti_old_part) error("Attempt to drift to the past");
-#endif
-
- /* Early abort? */
- if (c->hydro.count == 0) {
- /* Clear the drift flags. */
- cell_clear_flag(c, cell_flag_do_hydro_drift | cell_flag_do_hydro_sub_drift);
-
- /* Update the time of the last drift */
- c->hydro.ti_old_part = ti_current;
-
- return;
- }
-
- /* Ok, we have some particles somewhere in the hierarchy to drift */
-
- /* Are we not in a leaf ? */
- if (c->split && (force || cell_get_flag(c, cell_flag_do_hydro_sub_drift))) {
-
- /* Loop over the progeny and collect their data. */
- for (int k = 0; k < 8; k++) {
- if (c->progeny[k] != NULL) {
- struct cell *cp = c->progeny[k];
-
- /* Collect */
- cell_drift_part(cp, e, force);
-
- /* Update */
- dx_max = max(dx_max, cp->hydro.dx_max_part);
- dx_max_sort = max(dx_max_sort, cp->hydro.dx_max_sort);
- cell_h_max = max(cell_h_max, cp->hydro.h_max);
- }
- }
-
- /* Store the values */
- c->hydro.h_max = cell_h_max;
- c->hydro.dx_max_part = dx_max;
- c->hydro.dx_max_sort = dx_max_sort;
-
- /* Update the time of the last drift */
- c->hydro.ti_old_part = ti_current;
-
- } else if (!c->split && force && ti_current > ti_old_part) {
- /* Drift from the last time the cell was drifted to the current time */
- double dt_drift, dt_kick_grav, dt_kick_hydro, dt_therm;
- if (with_cosmology) {
- dt_drift =
- cosmology_get_drift_factor(e->cosmology, ti_old_part, ti_current);
- dt_kick_grav =
- cosmology_get_grav_kick_factor(e->cosmology, ti_old_part, ti_current);
- dt_kick_hydro = cosmology_get_hydro_kick_factor(e->cosmology, ti_old_part,
- ti_current);
- dt_therm = cosmology_get_therm_kick_factor(e->cosmology, ti_old_part,
- ti_current);
- } else {
- dt_drift = (ti_current - ti_old_part) * e->time_base;
- dt_kick_grav = (ti_current - ti_old_part) * e->time_base;
- dt_kick_hydro = (ti_current - ti_old_part) * e->time_base;
- dt_therm = (ti_current - ti_old_part) * e->time_base;
- }
-
- /* Loop over all the gas particles in the cell */
- const size_t nr_parts = c->hydro.count;
- for (size_t k = 0; k < nr_parts; k++) {
- /* Get a handle on the part. */
- struct part *const p = &parts[k];
- struct xpart *const xp = &xparts[k];
-
- /* Ignore inhibited particles */
- if (part_is_inhibited(p, e)) continue;
-
- /* Apply the effects of feedback on this particle
- * (Note: Only used in schemes that have a delayed feedback mechanism
- * otherwise just an empty function) */
- feedback_update_part(p, xp, e);
-
- /* Drift... */
- drift_part(p, xp, dt_drift, dt_kick_hydro, dt_kick_grav, dt_therm,
- ti_old_part, ti_current, e->cosmology, e->hydro_properties,
- e->entropy_floor);
-
- /* Update the tracers properties */
- tracers_after_drift(p, xp, e->internal_units, e->physical_constants,
- with_cosmology, e->cosmology, e->hydro_properties,
- e->cooling_func, e->time);
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Make sure the particle does not drift by more than a box length. */
- if (fabs(xp->v_full[0] * dt_drift) > e->s->dim[0] ||
- fabs(xp->v_full[1] * dt_drift) > e->s->dim[1] ||
- fabs(xp->v_full[2] * dt_drift) > e->s->dim[2]) {
- error(
- "Particle drifts by more than a box length! id %llu xp->v_full "
- "%.5e %.5e %.5e p->v %.5e %.5e %.5e",
- p->id, xp->v_full[0], xp->v_full[1], xp->v_full[2], p->v[0],
- p->v[1], p->v[2]);
- }
-#endif
-
- /* In non-periodic BC runs, remove particles that crossed the border */
- if (!periodic) {
-
- /* Did the particle leave the box? */
- if ((p->x[0] > dim[0]) || (p->x[0] < 0.) || // x
- (p->x[1] > dim[1]) || (p->x[1] < 0.) || // y
- (p->x[2] > dim[2]) || (p->x[2] < 0.)) { // z
-
- lock_lock(&e->s->lock);
-
- /* Re-check that the particle has not been removed
- * by another thread before we do the deed. */
- if (!part_is_inhibited(p, e)) {
-
-#ifdef WITH_LOGGER
- if (e->policy & engine_policy_logger) {
- /* Log the particle one last time. */
- logger_log_part(
- e->logger, p, xp, e, /* log_all */ 1,
- logger_pack_flags_and_data(logger_flag_delete, 0));
- }
-#endif
-
- /* One last action before death? */
- hydro_remove_part(p, xp);
-
- /* Remove the particle entirely */
- cell_remove_part(e, c, p, xp);
- }
-
- if (lock_unlock(&e->s->lock) != 0)
- error("Failed to unlock the space!");
-
- continue;
- }
- }
-
- /* Limit h to within the allowed range */
- p->h = min(p->h, hydro_h_max);
- p->h = max(p->h, hydro_h_min);
-
- /* Compute (square of) motion since last cell construction */
- const float dx2 = xp->x_diff[0] * xp->x_diff[0] +
- xp->x_diff[1] * xp->x_diff[1] +
- xp->x_diff[2] * xp->x_diff[2];
- dx2_max = max(dx2_max, dx2);
- const float dx2_sort = xp->x_diff_sort[0] * xp->x_diff_sort[0] +
- xp->x_diff_sort[1] * xp->x_diff_sort[1] +
- xp->x_diff_sort[2] * xp->x_diff_sort[2];
- dx2_max_sort = max(dx2_max_sort, dx2_sort);
-
- /* Update the maximal smoothing length in the cell */
- cell_h_max = max(cell_h_max, p->h);
-
- /* Mark the particle has not being swallowed */
- black_holes_mark_part_as_not_swallowed(&p->black_holes_data);
-
- /* Get ready for a density calculation */
- if (part_is_active(p, e)) {
- hydro_init_part(p, &e->s->hs);
- black_holes_init_potential(&p->black_holes_data);
- chemistry_init_part(p, e->chemistry);
- pressure_floor_init_part(p, xp);
- star_formation_init_part(p, e->star_formation);
- tracers_after_init(p, xp, e->internal_units, e->physical_constants,
- with_cosmology, e->cosmology, e->hydro_properties,
- e->cooling_func, e->time);
- rt_init_part(p);
- }
- }
-
- /* Now, get the maximal particle motion from its square */
- dx_max = sqrtf(dx2_max);
- dx_max_sort = sqrtf(dx2_max_sort);
-
- /* Store the values */
- c->hydro.h_max = cell_h_max;
- c->hydro.dx_max_part = dx_max;
- c->hydro.dx_max_sort = dx_max_sort;
-
- /* Update the time of the last drift */
- c->hydro.ti_old_part = ti_current;
- }
-
- /* Clear the drift flags. */
- cell_clear_flag(c, cell_flag_do_hydro_drift | cell_flag_do_hydro_sub_drift);
-}
-
-/**
- * @brief Recursively drifts the #gpart in a cell hierarchy.
- *
- * @param c The #cell.
- * @param e The #engine (to get ti_current).
- * @param force Drift the particles irrespective of the #cell flags.
- */
-void cell_drift_gpart(struct cell *c, const struct engine *e, int force) {
- const int periodic = e->s->periodic;
- const double dim[3] = {e->s->dim[0], e->s->dim[1], e->s->dim[2]};
- const int with_cosmology = (e->policy & engine_policy_cosmology);
- const integertime_t ti_old_gpart = c->grav.ti_old_part;
- const integertime_t ti_current = e->ti_current;
- struct gpart *const gparts = c->grav.parts;
- const struct gravity_props *grav_props = e->gravity_properties;
-
- /* Drift irrespective of cell flags? */
- force = (force || cell_get_flag(c, cell_flag_do_grav_drift));
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Check that we only drift local cells. */
- if (c->nodeID != engine_rank) error("Drifting a foreign cell is nope.");
-
- /* Check that we are actually going to move forward. */
- if (ti_current < ti_old_gpart) error("Attempt to drift to the past");
-#endif
-
- /* Early abort? */
- if (c->grav.count == 0) {
- /* Clear the drift flags. */
- cell_clear_flag(c, cell_flag_do_grav_drift | cell_flag_do_grav_sub_drift);
-
- /* Update the time of the last drift */
- c->grav.ti_old_part = ti_current;
-
- return;
- }
-
- /* Ok, we have some particles somewhere in the hierarchy to drift */
-
- /* Are we not in a leaf ? */
- if (c->split && (force || cell_get_flag(c, cell_flag_do_grav_sub_drift))) {
-
- /* Loop over the progeny and collect their data. */
- for (int k = 0; k < 8; k++) {
- if (c->progeny[k] != NULL) {
- struct cell *cp = c->progeny[k];
-
- /* Recurse */
- cell_drift_gpart(cp, e, force);
- }
- }
-
- /* Update the time of the last drift */
- c->grav.ti_old_part = ti_current;
-
- } else if (!c->split && force && ti_current > ti_old_gpart) {
- /* Drift from the last time the cell was drifted to the current time */
- double dt_drift;
- if (with_cosmology) {
- dt_drift =
- cosmology_get_drift_factor(e->cosmology, ti_old_gpart, ti_current);
- } else {
- dt_drift = (ti_current - ti_old_gpart) * e->time_base;
- }
-
- /* Loop over all the g-particles in the cell */
- const size_t nr_gparts = c->grav.count;
- for (size_t k = 0; k < nr_gparts; k++) {
- /* Get a handle on the gpart. */
- struct gpart *const gp = &gparts[k];
-
- /* Ignore inhibited particles */
- if (gpart_is_inhibited(gp, e)) continue;
-
- /* Drift... */
- drift_gpart(gp, dt_drift, ti_old_gpart, ti_current, grav_props, e);
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Make sure the particle does not drift by more than a box length. */
- if (fabs(gp->v_full[0] * dt_drift) > e->s->dim[0] ||
- fabs(gp->v_full[1] * dt_drift) > e->s->dim[1] ||
- fabs(gp->v_full[2] * dt_drift) > e->s->dim[2]) {
- error(
- "Particle drifts by more than a box length! gp->v_full %.5e %.5e "
- "%.5e",
- gp->v_full[0], gp->v_full[1], gp->v_full[2]);
- }
-#endif
-
- /* In non-periodic BC runs, remove particles that crossed the border */
- if (!periodic) {
-
- /* Did the particle leave the box? */
- if ((gp->x[0] > dim[0]) || (gp->x[0] < 0.) || // x
- (gp->x[1] > dim[1]) || (gp->x[1] < 0.) || // y
- (gp->x[2] > dim[2]) || (gp->x[2] < 0.)) { // z
-
- lock_lock(&e->s->lock);
-
- /* Re-check that the particle has not been removed
- * by another thread before we do the deed. */
- if (!gpart_is_inhibited(gp, e)) {
-
- /* Remove the particle entirely */
- if (gp->type == swift_type_dark_matter) {
-
-#ifdef WITH_LOGGER
- if (e->policy & engine_policy_logger) {
- /* Log the particle one last time. */
- logger_log_gpart(
- e->logger, gp, e, /* log_all */ 1,
- logger_pack_flags_and_data(logger_flag_delete, 0));
- }
-#endif
-
- /* Remove the particle */
- cell_remove_gpart(e, c, gp);
- }
- }
-
- if (lock_unlock(&e->s->lock) != 0)
- error("Failed to unlock the space!");
-
- continue;
- }
- }
-
- /* Init gravity force fields. */
- if (gpart_is_active(gp, e)) {
- gravity_init_gpart(gp);
- }
- }
-
- /* Update the time of the last drift */
- c->grav.ti_old_part = ti_current;
- }
-
- /* Clear the drift flags. */
- cell_clear_flag(c, cell_flag_do_grav_drift | cell_flag_do_grav_sub_drift);
-}
-
-/**
- * @brief Recursively drifts the #spart in a cell hierarchy.
- *
- * @param c The #cell.
- * @param e The #engine (to get ti_current).
- * @param force Drift the particles irrespective of the #cell flags.
- */
-void cell_drift_spart(struct cell *c, const struct engine *e, int force) {
- const int periodic = e->s->periodic;
- const double dim[3] = {e->s->dim[0], e->s->dim[1], e->s->dim[2]};
- const int with_cosmology = (e->policy & engine_policy_cosmology);
- const float stars_h_max = e->hydro_properties->h_max;
- const float stars_h_min = e->hydro_properties->h_min;
- const integertime_t ti_old_spart = c->stars.ti_old_part;
- const integertime_t ti_current = e->ti_current;
- struct spart *const sparts = c->stars.parts;
-
- float dx_max = 0.f, dx2_max = 0.f;
- float dx_max_sort = 0.0f, dx2_max_sort = 0.f;
- float cell_h_max = 0.f;
-
- /* Drift irrespective of cell flags? */
- force = (force || cell_get_flag(c, cell_flag_do_stars_drift));
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Check that we only drift local cells. */
- if (c->nodeID != engine_rank) error("Drifting a foreign cell is nope.");
-
- /* Check that we are actually going to move forward. */
- if (ti_current < ti_old_spart) error("Attempt to drift to the past");
-#endif
-
- /* Early abort? */
- if (c->stars.count == 0) {
- /* Clear the drift flags. */
- cell_clear_flag(c, cell_flag_do_stars_drift | cell_flag_do_stars_sub_drift);
-
- /* Update the time of the last drift */
- c->stars.ti_old_part = ti_current;
-
- return;
- }
-
- /* Ok, we have some particles somewhere in the hierarchy to drift */
-
- /* Are we not in a leaf ? */
- if (c->split && (force || cell_get_flag(c, cell_flag_do_stars_sub_drift))) {
-
- /* Loop over the progeny and collect their data. */
- for (int k = 0; k < 8; k++) {
- if (c->progeny[k] != NULL) {
- struct cell *cp = c->progeny[k];
-
- /* Recurse */
- cell_drift_spart(cp, e, force);
-
- /* Update */
- dx_max = max(dx_max, cp->stars.dx_max_part);
- dx_max_sort = max(dx_max_sort, cp->stars.dx_max_sort);
- cell_h_max = max(cell_h_max, cp->stars.h_max);
- }
- }
-
- /* Store the values */
- c->stars.h_max = cell_h_max;
- c->stars.dx_max_part = dx_max;
- c->stars.dx_max_sort = dx_max_sort;
-
- /* Update the time of the last drift */
- c->stars.ti_old_part = ti_current;
-
- } else if (!c->split && force && ti_current > ti_old_spart) {
- /* Drift from the last time the cell was drifted to the current time */
- double dt_drift;
- if (with_cosmology) {
- dt_drift =
- cosmology_get_drift_factor(e->cosmology, ti_old_spart, ti_current);
- } else {
- dt_drift = (ti_current - ti_old_spart) * e->time_base;
- }
-
- /* Loop over all the star particles in the cell */
- const size_t nr_sparts = c->stars.count;
- for (size_t k = 0; k < nr_sparts; k++) {
- /* Get a handle on the spart. */
- struct spart *const sp = &sparts[k];
-
- /* Ignore inhibited particles */
- if (spart_is_inhibited(sp, e)) continue;
-
- /* Drift... */
- drift_spart(sp, dt_drift, ti_old_spart, ti_current);
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Make sure the particle does not drift by more than a box length. */
- if (fabs(sp->v[0] * dt_drift) > e->s->dim[0] ||
- fabs(sp->v[1] * dt_drift) > e->s->dim[1] ||
- fabs(sp->v[2] * dt_drift) > e->s->dim[2]) {
- error("Particle drifts by more than a box length!");
- }
-#endif
-
- /* In non-periodic BC runs, remove particles that crossed the border */
- if (!periodic) {
-
- /* Did the particle leave the box? */
- if ((sp->x[0] > dim[0]) || (sp->x[0] < 0.) || // x
- (sp->x[1] > dim[1]) || (sp->x[1] < 0.) || // y
- (sp->x[2] > dim[2]) || (sp->x[2] < 0.)) { // z
-
- lock_lock(&e->s->lock);
-
- /* Re-check that the particle has not been removed
- * by another thread before we do the deed. */
- if (!spart_is_inhibited(sp, e)) {
-
-#ifdef WITH_LOGGER
- if (e->policy & engine_policy_logger) {
- /* Log the particle one last time. */
- logger_log_spart(
- e->logger, sp, e, /* log_all */ 1,
- logger_pack_flags_and_data(logger_flag_delete, 0));
- }
-#endif
-
- /* Remove the particle entirely */
- cell_remove_spart(e, c, sp);
- }
-
- if (lock_unlock(&e->s->lock) != 0)
- error("Failed to unlock the space!");
-
- continue;
- }
- }
-
- /* Limit h to within the allowed range */
- sp->h = min(sp->h, stars_h_max);
- sp->h = max(sp->h, stars_h_min);
-
- /* Compute (square of) motion since last cell construction */
- const float dx2 = sp->x_diff[0] * sp->x_diff[0] +
- sp->x_diff[1] * sp->x_diff[1] +
- sp->x_diff[2] * sp->x_diff[2];
- dx2_max = max(dx2_max, dx2);
-
- const float dx2_sort = sp->x_diff_sort[0] * sp->x_diff_sort[0] +
- sp->x_diff_sort[1] * sp->x_diff_sort[1] +
- sp->x_diff_sort[2] * sp->x_diff_sort[2];
-
- dx2_max_sort = max(dx2_max_sort, dx2_sort);
-
- /* Maximal smoothing length */
- cell_h_max = max(cell_h_max, sp->h);
-
- /* Get ready for a density calculation */
- if (spart_is_active(sp, e)) {
- stars_init_spart(sp);
- feedback_init_spart(sp);
- rt_init_spart(sp);
- }
- }
-
- /* Now, get the maximal particle motion from its square */
- dx_max = sqrtf(dx2_max);
- dx_max_sort = sqrtf(dx2_max_sort);
-
- /* Store the values */
- c->stars.h_max = cell_h_max;
- c->stars.dx_max_part = dx_max;
- c->stars.dx_max_sort = dx_max_sort;
-
- /* Update the time of the last drift */
- c->stars.ti_old_part = ti_current;
- }
-
- /* Clear the drift flags. */
- cell_clear_flag(c, cell_flag_do_stars_drift | cell_flag_do_stars_sub_drift);
-}
-
-/**
- * @brief Recursively drifts the #bpart in a cell hierarchy.
- *
- * @param c The #cell.
- * @param e The #engine (to get ti_current).
- * @param force Drift the particles irrespective of the #cell flags.
- */
-void cell_drift_bpart(struct cell *c, const struct engine *e, int force) {
-
- const int periodic = e->s->periodic;
- const double dim[3] = {e->s->dim[0], e->s->dim[1], e->s->dim[2]};
- const int with_cosmology = (e->policy & engine_policy_cosmology);
- const float black_holes_h_max = e->hydro_properties->h_max;
- const float black_holes_h_min = e->hydro_properties->h_min;
- const integertime_t ti_old_bpart = c->black_holes.ti_old_part;
- const integertime_t ti_current = e->ti_current;
- struct bpart *const bparts = c->black_holes.parts;
-
- float dx_max = 0.f, dx2_max = 0.f;
- float cell_h_max = 0.f;
-
- /* Drift irrespective of cell flags? */
- force = (force || cell_get_flag(c, cell_flag_do_bh_drift));
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Check that we only drift local cells. */
- if (c->nodeID != engine_rank) error("Drifting a foreign cell is nope.");
-
- /* Check that we are actually going to move forward. */
- if (ti_current < ti_old_bpart) error("Attempt to drift to the past");
-#endif
-
- /* Early abort? */
- if (c->black_holes.count == 0) {
-
- /* Clear the drift flags. */
- cell_clear_flag(c, cell_flag_do_bh_drift | cell_flag_do_bh_sub_drift);
-
- /* Update the time of the last drift */
- c->black_holes.ti_old_part = ti_current;
-
- return;
- }
-
- /* Ok, we have some particles somewhere in the hierarchy to drift */
-
- /* Are we not in a leaf ? */
- if (c->split && (force || cell_get_flag(c, cell_flag_do_bh_sub_drift))) {
-
- /* Loop over the progeny and collect their data. */
- for (int k = 0; k < 8; k++) {
- if (c->progeny[k] != NULL) {
- struct cell *cp = c->progeny[k];
-
- /* Recurse */
- cell_drift_bpart(cp, e, force);
-
- /* Update */
- dx_max = max(dx_max, cp->black_holes.dx_max_part);
- cell_h_max = max(cell_h_max, cp->black_holes.h_max);
- }
- }
-
- /* Store the values */
- c->black_holes.h_max = cell_h_max;
- c->black_holes.dx_max_part = dx_max;
-
- /* Update the time of the last drift */
- c->black_holes.ti_old_part = ti_current;
-
- } else if (!c->split && force && ti_current > ti_old_bpart) {
-
- /* Drift from the last time the cell was drifted to the current time */
- double dt_drift;
- if (with_cosmology) {
- dt_drift =
- cosmology_get_drift_factor(e->cosmology, ti_old_bpart, ti_current);
- } else {
- dt_drift = (ti_current - ti_old_bpart) * e->time_base;
- }
-
- /* Loop over all the star particles in the cell */
- const size_t nr_bparts = c->black_holes.count;
- for (size_t k = 0; k < nr_bparts; k++) {
-
- /* Get a handle on the bpart. */
- struct bpart *const bp = &bparts[k];
-
- /* Ignore inhibited particles */
- if (bpart_is_inhibited(bp, e)) continue;
-
- /* Drift... */
- drift_bpart(bp, dt_drift, ti_old_bpart, ti_current);
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Make sure the particle does not drift by more than a box length. */
- if (fabs(bp->v[0] * dt_drift) > e->s->dim[0] ||
- fabs(bp->v[1] * dt_drift) > e->s->dim[1] ||
- fabs(bp->v[2] * dt_drift) > e->s->dim[2]) {
- error("Particle drifts by more than a box length!");
- }
-#endif
-
- /* In non-periodic BC runs, remove particles that crossed the border */
- if (!periodic) {
-
- /* Did the particle leave the box? */
- if ((bp->x[0] > dim[0]) || (bp->x[0] < 0.) || // x
- (bp->x[1] > dim[1]) || (bp->x[1] < 0.) || // y
- (bp->x[2] > dim[2]) || (bp->x[2] < 0.)) { // z
-
- lock_lock(&e->s->lock);
-
- /* Re-check that the particle has not been removed
- * by another thread before we do the deed. */
- if (!bpart_is_inhibited(bp, e)) {
-
-#ifdef WITH_LOGGER
- if (e->policy & engine_policy_logger) {
- error("Logging of black hole particles is not yet implemented.");
- }
-#endif
-
- /* Remove the particle entirely */
- cell_remove_bpart(e, c, bp);
- }
-
- if (lock_unlock(&e->s->lock) != 0)
- error("Failed to unlock the space!");
-
- continue;
- }
- }
-
- /* Limit h to within the allowed range */
- bp->h = min(bp->h, black_holes_h_max);
- bp->h = max(bp->h, black_holes_h_min);
-
- /* Compute (square of) motion since last cell construction */
- const float dx2 = bp->x_diff[0] * bp->x_diff[0] +
- bp->x_diff[1] * bp->x_diff[1] +
- bp->x_diff[2] * bp->x_diff[2];
- dx2_max = max(dx2_max, dx2);
-
- /* Maximal smoothing length */
- cell_h_max = max(cell_h_max, bp->h);
-
- /* Mark the particle has not being swallowed */
- black_holes_mark_bpart_as_not_swallowed(&bp->merger_data);
-
- /* Get ready for a density calculation */
- if (bpart_is_active(bp, e)) {
- black_holes_init_bpart(bp);
- }
- }
-
- /* Now, get the maximal particle motion from its square */
- dx_max = sqrtf(dx2_max);
-
- /* Store the values */
- c->black_holes.h_max = cell_h_max;
- c->black_holes.dx_max_part = dx_max;
-
- /* Update the time of the last drift */
- c->black_holes.ti_old_part = ti_current;
- }
-
- /* Clear the drift flags. */
- cell_clear_flag(c, cell_flag_do_bh_drift | cell_flag_do_bh_sub_drift);
-}
-
-/**
- * @brief Recursively drifts the #sink's in a cell hierarchy.
- *
- * @param c The #cell.
- * @param e The #engine (to get ti_current).
- * @param force Drift the particles irrespective of the #cell flags.
- */
-void cell_drift_sink(struct cell *c, const struct engine *e, int force) {
-
- const int periodic = e->s->periodic;
- const double dim[3] = {e->s->dim[0], e->s->dim[1], e->s->dim[2]};
- const int with_cosmology = (e->policy & engine_policy_cosmology);
- const integertime_t ti_old_sink = c->sinks.ti_old_part;
- const integertime_t ti_current = e->ti_current;
- struct sink *const sinks = c->sinks.parts;
-
- float dx_max = 0.f, dx2_max = 0.f;
- float cell_r_max = 0.f;
-
- /* Drift irrespective of cell flags? */
- force = (force || cell_get_flag(c, cell_flag_do_sink_drift));
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Check that we only drift local cells. */
- if (c->nodeID != engine_rank) error("Drifting a foreign cell is nope.");
-
- /* Check that we are actually going to move forward. */
- if (ti_current < ti_old_sink) error("Attempt to drift to the past");
-#endif
-
- /* Early abort? */
- if (c->sinks.count == 0) {
-
- /* Clear the drift flags. */
- cell_clear_flag(c, cell_flag_do_sink_drift | cell_flag_do_sink_sub_drift);
-
- /* Update the time of the last drift */
- c->sinks.ti_old_part = ti_current;
-
- return;
- }
-
- /* Ok, we have some particles somewhere in the hierarchy to drift */
-
- /* Are we not in a leaf ? */
- if (c->split && (force || cell_get_flag(c, cell_flag_do_sink_sub_drift))) {
-
- /* Loop over the progeny and collect their data. */
- for (int k = 0; k < 8; k++) {
- if (c->progeny[k] != NULL) {
- struct cell *cp = c->progeny[k];
-
- /* Recurse */
- cell_drift_sink(cp, e, force);
-
- /* Update */
- dx_max = max(dx_max, cp->sinks.dx_max_part);
- cell_r_max = max(cell_r_max, cp->sinks.r_cut_max);
- }
- }
-
- /* Store the values */
- c->sinks.r_cut_max = cell_r_max;
- c->sinks.dx_max_part = dx_max;
-
- /* Update the time of the last drift */
- c->sinks.ti_old_part = ti_current;
-
- } else if (!c->split && force && ti_current > ti_old_sink) {
-
- /* Drift from the last time the cell was drifted to the current time */
- double dt_drift;
- if (with_cosmology) {
- dt_drift =
- cosmology_get_drift_factor(e->cosmology, ti_old_sink, ti_current);
- } else {
- dt_drift = (ti_current - ti_old_sink) * e->time_base;
- }
-
- /* Loop over all the star particles in the cell */
- const size_t nr_sinks = c->sinks.count;
- for (size_t k = 0; k < nr_sinks; k++) {
-
- /* Get a handle on the sink. */
- struct sink *const sink = &sinks[k];
-
- /* Ignore inhibited particles */
- if (sink_is_inhibited(sink, e)) continue;
-
- /* Drift... */
- drift_sink(sink, dt_drift, ti_old_sink, ti_current);
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Make sure the particle does not drift by more than a box length. */
- if (fabs(sink->v[0] * dt_drift) > e->s->dim[0] ||
- fabs(sink->v[1] * dt_drift) > e->s->dim[1] ||
- fabs(sink->v[2] * dt_drift) > e->s->dim[2]) {
- error("Particle drifts by more than a box length!");
- }
-#endif
-
- /* In non-periodic BC runs, remove particles that crossed the border */
- if (!periodic) {
-
- /* Did the particle leave the box? */
- if ((sink->x[0] > dim[0]) || (sink->x[0] < 0.) || // x
- (sink->x[1] > dim[1]) || (sink->x[1] < 0.) || // y
- (sink->x[2] > dim[2]) || (sink->x[2] < 0.)) { // z
-
- lock_lock(&e->s->lock);
-
- /* Re-check that the particle has not been removed
- * by another thread before we do the deed. */
- if (!sink_is_inhibited(sink, e)) {
-
-#ifdef WITH_LOGGER
- if (e->policy & engine_policy_logger) {
- error("Logging of sink particles is not yet implemented.");
- }
-#endif
-
- /* Remove the particle entirely */
- // cell_remove_sink(e, c, bp);
- error("TODO: loic implement cell_remove_sink");
- }
-
- if (lock_unlock(&e->s->lock) != 0)
- error("Failed to unlock the space!");
-
- continue;
- }
- }
-
- /* sp->h does not need to be limited. */
-
- /* Compute (square of) motion since last cell construction */
- const float dx2 = sink->x_diff[0] * sink->x_diff[0] +
- sink->x_diff[1] * sink->x_diff[1] +
- sink->x_diff[2] * sink->x_diff[2];
- dx2_max = max(dx2_max, dx2);
-
- /* Maximal smoothing length */
- cell_r_max = max(cell_r_max, sink->r_cut);
-
- /* Get ready for a density calculation */
- if (sink_is_active(sink, e)) {
- sink_init_sink(sink);
- }
- }
-
- /* Now, get the maximal particle motion from its square */
- dx_max = sqrtf(dx2_max);
-
- /* Store the values */
- c->sinks.r_cut_max = cell_r_max;
- c->sinks.dx_max_part = dx_max;
-
- /* Update the time of the last drift */
- c->sinks.ti_old_part = ti_current;
- }
-
- /* Clear the drift flags. */
- cell_clear_flag(c, cell_flag_do_sink_drift | cell_flag_do_sink_sub_drift);
-}
-
-/**
- * @brief Recursively drifts all multipoles in a cell hierarchy.
- *
- * @param c The #cell.
- * @param e The #engine (to get ti_current).
- */
-void cell_drift_all_multipoles(struct cell *c, const struct engine *e) {
- const integertime_t ti_old_multipole = c->grav.ti_old_multipole;
- const integertime_t ti_current = e->ti_current;
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Check that we are actually going to move forward. */
- if (ti_current < ti_old_multipole) error("Attempt to drift to the past");
-#endif
-
- /* Drift from the last time the cell was drifted to the current time */
- double dt_drift;
- if (e->policy & engine_policy_cosmology)
- dt_drift =
- cosmology_get_drift_factor(e->cosmology, ti_old_multipole, ti_current);
- else
- dt_drift = (ti_current - ti_old_multipole) * e->time_base;
-
- /* Drift the multipole */
- if (ti_current > ti_old_multipole) gravity_drift(c->grav.multipole, dt_drift);
-
- /* Are we not in a leaf ? */
- if (c->split) {
- /* Loop over the progeny and recurse. */
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL) cell_drift_all_multipoles(c->progeny[k], e);
- }
-
- /* Update the time of the last drift */
- c->grav.ti_old_multipole = ti_current;
-}
-
-/**
- * @brief Drifts the multipole of a cell to the current time.
- *
- * Only drifts the multipole at this level. Multipoles deeper in the
- * tree are not updated.
- *
- * @param c The #cell.
- * @param e The #engine (to get ti_current).
- */
-void cell_drift_multipole(struct cell *c, const struct engine *e) {
- const integertime_t ti_old_multipole = c->grav.ti_old_multipole;
- const integertime_t ti_current = e->ti_current;
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Check that we are actually going to move forward. */
- if (ti_current < ti_old_multipole) error("Attempt to drift to the past");
-#endif
-
- /* Drift from the last time the cell was drifted to the current time */
- double dt_drift;
- if (e->policy & engine_policy_cosmology)
- dt_drift =
- cosmology_get_drift_factor(e->cosmology, ti_old_multipole, ti_current);
- else
- dt_drift = (ti_current - ti_old_multipole) * e->time_base;
-
- if (ti_current > ti_old_multipole) gravity_drift(c->grav.multipole, dt_drift);
-
- /* Update the time of the last drift */
- c->grav.ti_old_multipole = ti_current;
-}
-
-/**
- * @brief Resets all the sorting properties for the stars in a given cell
- * hierarchy.
- *
- * The clear_unused_flags argument can be used to additionally clean up all
- * the flags demanding a sort for the given cell. This should be used with
- * caution as it will prevent the sort tasks from doing anything on that cell
- * until these flags are reset.
- *
- * @param c The #cell to clean.
- * @param clear_unused_flags Do we also clean the flags demanding a sort?
- */
-void cell_clear_stars_sort_flags(struct cell *c, const int clear_unused_flags) {
-
- /* Clear the flags that have not been reset by the sort task? */
- if (clear_unused_flags) {
- c->stars.requires_sorts = 0;
- c->stars.do_sort = 0;
- cell_clear_flag(c, cell_flag_do_stars_sub_sort);
- }
-
- /* Indicate that the cell is not sorted and cancel the pointer sorting
- * arrays.
- */
- c->stars.sorted = 0;
- cell_free_stars_sorts(c);
-
- /* Recurse if possible */
- if (c->split) {
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL)
- cell_clear_stars_sort_flags(c->progeny[k], clear_unused_flags);
- }
-}
-
-/**
- * @brief Resets all the sorting properties for the hydro in a given cell
- * hierarchy.
- *
- * The clear_unused_flags argument can be used to additionally clean up all
- * the flags demanding a sort for the given cell. This should be used with
- * caution as it will prevent the sort tasks from doing anything on that cell
- * until these flags are reset.
- *
- * @param c The #cell to clean.
- * @param clear_unused_flags Do we also clean the flags demanding a sort?
- */
-void cell_clear_hydro_sort_flags(struct cell *c, const int clear_unused_flags) {
-
- /* Clear the flags that have not been reset by the sort task? */
- if (clear_unused_flags) {
- c->hydro.do_sort = 0;
- c->hydro.requires_sorts = 0;
- cell_clear_flag(c, cell_flag_do_hydro_sub_sort);
- }
-
- /* Indicate that the cell is not sorted and cancel the pointer sorting
- * arrays.
- */
- c->hydro.sorted = 0;
- cell_free_hydro_sorts(c);
-
- /* Recurse if possible */
- if (c->split) {
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL)
- cell_clear_hydro_sort_flags(c->progeny[k], clear_unused_flags);
- }
-}
-
-/**
- * @brief Recursively checks that all particles in a cell have a time-step
- */
-void cell_check_timesteps(const struct cell *c, const integertime_t ti_current,
- const timebin_t max_bin) {
-#ifdef SWIFT_DEBUG_CHECKS
-
- if (c->hydro.ti_end_min == 0 && c->grav.ti_end_min == 0 &&
- c->stars.ti_end_min == 0 && c->black_holes.ti_end_min == 0 &&
- c->sinks.ti_end_min == 0 && c->nr_tasks > 0)
- error("Cell without assigned time-step");
-
- if (c->split) {
- for (int k = 0; k < 8; ++k)
- if (c->progeny[k] != NULL)
- cell_check_timesteps(c->progeny[k], ti_current, max_bin);
- } else {
- if (c->nodeID == engine_rank)
- for (int i = 0; i < c->hydro.count; ++i)
- if (c->hydro.parts[i].time_bin == 0)
- error("Particle without assigned time-bin");
- }
-
- /* Other checks not relevent when starting-up */
- if (ti_current == 0) return;
-
- integertime_t ti_end_min = max_nr_timesteps;
- integertime_t ti_end_max = 0;
- integertime_t ti_beg_max = 0;
-
- int count = 0;
-
- for (int i = 0; i < c->hydro.count; ++i) {
-
- const struct part *p = &c->hydro.parts[i];
- if (p->time_bin == time_bin_inhibited) continue;
- if (p->time_bin == time_bin_not_created) continue;
-
- ++count;
-
- integertime_t ti_end, ti_beg;
-
- if (p->time_bin <= max_bin) {
- integertime_t time_step = get_integer_timestep(p->time_bin);
- ti_end = get_integer_time_end(ti_current, p->time_bin) + time_step;
- ti_beg = get_integer_time_begin(ti_current + 1, p->time_bin);
- } else {
- ti_end = get_integer_time_end(ti_current, p->time_bin);
- ti_beg = get_integer_time_begin(ti_current + 1, p->time_bin);
- }
-
- ti_end_min = min(ti_end, ti_end_min);
- ti_end_max = max(ti_end, ti_end_max);
- ti_beg_max = max(ti_beg, ti_beg_max);
- }
-
- /* Only check cells that have at least one non-inhibited particle */
- if (count > 0) {
-
- if (count != c->hydro.count) {
-
- /* Note that we use a < as the particle with the smallest time-bin
- might have been swallowed. This means we will run this cell with
- 0 active particles but that's not wrong */
- if (ti_end_min < c->hydro.ti_end_min)
- error(
- "Non-matching ti_end_min. Cell=%lld true=%lld ti_current=%lld "
- "depth=%d",
- c->hydro.ti_end_min, ti_end_min, ti_current, c->depth);
-
- } else /* Normal case: nothing was swallowed/converted */ {
- if (ti_end_min != c->hydro.ti_end_min)
- error(
- "Non-matching ti_end_min. Cell=%lld true=%lld ti_current=%lld "
- "depth=%d",
- c->hydro.ti_end_min, ti_end_min, ti_current, c->depth);
- }
-
- if (ti_end_max > c->hydro.ti_end_max)
- error(
- "Non-matching ti_end_max. Cell=%lld true=%lld ti_current=%lld "
- "depth=%d",
- c->hydro.ti_end_max, ti_end_max, ti_current, c->depth);
-
- if (ti_beg_max != c->hydro.ti_beg_max)
- error(
- "Non-matching ti_beg_max. Cell=%lld true=%lld ti_current=%lld "
- "depth=%d",
- c->hydro.ti_beg_max, ti_beg_max, ti_current, c->depth);
- }
-
-#else
- error("Calling debugging code without debugging flag activated.");
-#endif
-}
-
-void cell_check_spart_pos(const struct cell *c,
- const struct spart *global_sparts) {
-#ifdef SWIFT_DEBUG_CHECKS
-
- /* Recurse */
- if (c->split) {
- for (int k = 0; k < 8; ++k)
- if (c->progeny[k] != NULL)
- cell_check_spart_pos(c->progeny[k], global_sparts);
- }
-
- struct spart *sparts = c->stars.parts;
- const int count = c->stars.count;
- for (int i = 0; i < count; ++i) {
- const struct spart *sp = &sparts[i];
- if ((sp->x[0] < c->loc[0] / space_stretch) ||
- (sp->x[1] < c->loc[1] / space_stretch) ||
- (sp->x[2] < c->loc[2] / space_stretch) ||
- (sp->x[0] >= (c->loc[0] + c->width[0]) * space_stretch) ||
- (sp->x[1] >= (c->loc[1] + c->width[1]) * space_stretch) ||
- (sp->x[2] >= (c->loc[2] + c->width[2]) * space_stretch))
- error("spart not in its cell!");
-
- if (sp->time_bin != time_bin_not_created &&
- sp->time_bin != time_bin_inhibited) {
- const struct gpart *gp = sp->gpart;
- if (gp == NULL && sp->time_bin != time_bin_not_created)
- error("Unlinked spart!");
-
- if (&global_sparts[-gp->id_or_neg_offset] != sp)
- error("Incorrectly linked spart!");
- }
- }
-
-#else
- error("Calling a degugging function outside debugging mode.");
-#endif
-}
-
-/**
- * @brief Checks that a cell and all its progenies have cleared their sort
- * flags.
- *
- * Should only be used for debugging purposes.
- *
- * @param c The #cell to check.
- */
-void cell_check_sort_flags(const struct cell *c) {
-
-#ifdef SWIFT_DEBUG_CHECKS
- const int do_hydro_sub_sort = cell_get_flag(c, cell_flag_do_hydro_sub_sort);
- const int do_stars_sub_sort = cell_get_flag(c, cell_flag_do_stars_sub_sort);
-
- if (do_hydro_sub_sort)
- error("cell %d has a hydro sub_sort flag set. Node=%d depth=%d maxdepth=%d",
- c->cellID, c->nodeID, c->depth, c->maxdepth);
-
- if (do_stars_sub_sort)
- error("cell %d has a stars sub_sort flag set. Node=%d depth=%d maxdepth=%d",
- c->cellID, c->nodeID, c->depth, c->maxdepth);
-
- if (c->split) {
- for (int k = 0; k < 8; ++k) {
- if (c->progeny[k] != NULL) cell_check_sort_flags(c->progeny[k]);
- }
- }
-#endif
-}
-
-/**
- * @brief Recursively update the pointer and counter for #spart after the
- * addition of a new particle.
- *
- * @param c The cell we are working on.
- * @param progeny_list The list of the progeny index at each level for the
- * leaf-cell where the particle was added.
- * @param main_branch Are we in a cell directly above the leaf where the new
- * particle was added?
- */
-void cell_recursively_shift_sparts(struct cell *c,
- const int progeny_list[space_cell_maxdepth],
- const int main_branch) {
- if (c->split) {
- /* No need to recurse in progenies located before the insestion point */
- const int first_progeny = main_branch ? progeny_list[(int)c->depth] : 0;
-
- for (int k = first_progeny; k < 8; ++k) {
- if (c->progeny[k] != NULL)
- cell_recursively_shift_sparts(c->progeny[k], progeny_list,
- main_branch && (k == first_progeny));
- }
- }
-
- /* When directly above the leaf with the new particle: increase the particle
- * count */
- /* When after the leaf with the new particle: shift by one position */
- if (main_branch) {
- c->stars.count++;
-
- /* Indicate that the cell is not sorted and cancel the pointer sorting
- * arrays. */
- c->stars.sorted = 0;
- cell_free_stars_sorts(c);
-
- } else {
- c->stars.parts++;
- }
-}
-
-/**
- * @brief Recursively update the pointer and counter for #sink after the
- * addition of a new particle.
- *
- * @param c The cell we are working on.
- * @param progeny_list The list of the progeny index at each level for the
- * leaf-cell where the particle was added.
- * @param main_branch Are we in a cell directly above the leaf where the new
- * particle was added?
- */
-void cell_recursively_shift_sinks(struct cell *c,
- const int progeny_list[space_cell_maxdepth],
- const int main_branch) {
- if (c->split) {
- /* No need to recurse in progenies located before the insestion point */
- const int first_progeny = main_branch ? progeny_list[(int)c->depth] : 0;
-
- for (int k = first_progeny; k < 8; ++k) {
- if (c->progeny[k] != NULL)
- cell_recursively_shift_sinks(c->progeny[k], progeny_list,
- main_branch && (k == first_progeny));
- }
- }
-
- /* When directly above the leaf with the new particle: increase the particle
- * count */
- /* When after the leaf with the new particle: shift by one position */
- if (main_branch) {
- c->sinks.count++;
- } else {
- c->sinks.parts++;
- }
-}
-
-/**
- * @brief Recursively update the pointer and counter for #gpart after the
- * addition of a new particle.
- *
- * @param c The cell we are working on.
- * @param progeny_list The list of the progeny index at each level for the
- * leaf-cell where the particle was added.
- * @param main_branch Are we in a cell directly above the leaf where the new
- * particle was added?
- */
-void cell_recursively_shift_gparts(struct cell *c,
- const int progeny_list[space_cell_maxdepth],
- const int main_branch) {
- if (c->split) {
- /* No need to recurse in progenies located before the insestion point */
- const int first_progeny = main_branch ? progeny_list[(int)c->depth] : 0;
-
- for (int k = first_progeny; k < 8; ++k) {
- if (c->progeny[k] != NULL)
- cell_recursively_shift_gparts(c->progeny[k], progeny_list,
- main_branch && (k == first_progeny));
- }
- }
-
- /* When directly above the leaf with the new particle: increase the particle
- * count */
- /* When after the leaf with the new particle: shift by one position */
- if (main_branch) {
- c->grav.count++;
- } else {
- c->grav.parts++;
- }
-}
-
-/**
- * @brief "Add" a #spart in a given #cell.
- *
- * This function will add a #spart at the start of the current cell's array by
- * shifting all the #spart in the top-level cell by one position. All the
- * pointers and cell counts are updated accordingly.
- *
- * @param e The #engine.
- * @param c The leaf-cell in which to add the #spart.
- *
- * @return A pointer to the newly added #spart. The spart has a been zeroed
- * and given a position within the cell as well as set to the minimal active
- * time bin.
- */
-struct spart *cell_add_spart(struct engine *e, struct cell *const c) {
- /* Perform some basic consitency checks */
- if (c->nodeID != engine_rank) error("Adding spart on a foreign node");
- if (c->grav.ti_old_part != e->ti_current) error("Undrifted cell!");
- if (c->split) error("Addition of spart performed above the leaf level");
-
- /* Progeny number at each level */
- int progeny[space_cell_maxdepth];
-#ifdef SWIFT_DEBUG_CHECKS
- for (int i = 0; i < space_cell_maxdepth; ++i) progeny[i] = -1;
-#endif
-
- /* Get the top-level this leaf cell is in and compute the progeny indices at
- each level */
- struct cell *top = c;
- while (top->parent != NULL) {
- /* What is the progeny index of the cell? */
- for (int k = 0; k < 8; ++k) {
- if (top->parent->progeny[k] == top) {
- progeny[(int)top->parent->depth] = k;
- }
- }
-
- /* Check that the cell was indeed drifted to this point to avoid future
- * issues */
-#ifdef SWIFT_DEBUG_CHECKS
- if (top->hydro.super != NULL && top->stars.count > 0 &&
- top->stars.ti_old_part != e->ti_current) {
- error("Cell had not been correctly drifted before star formation");
- }
-#endif
-
- /* Climb up */
- top = top->parent;
- }
-
- /* Lock the top-level cell as we are going to operate on it */
- lock_lock(&top->stars.star_formation_lock);
-
- /* Are there any extra particles left? */
- if (top->stars.count == top->stars.count_total) {
-
- message("We ran out of free star particles!");
-
- /* Release the local lock before exiting. */
- if (lock_unlock(&top->stars.star_formation_lock) != 0)
- error("Failed to unlock the top-level cell.");
-
- atomic_inc(&e->forcerebuild);
- return NULL;
- }
-
- /* Number of particles to shift in order to get a free space. */
- const size_t n_copy = &top->stars.parts[top->stars.count] - c->stars.parts;
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (c->stars.parts + n_copy > top->stars.parts + top->stars.count)
- error("Copying beyond the allowed range");
-#endif
-
- if (n_copy > 0) {
- // MATTHIEU: This can be improved. We don't need to copy everything, just
- // need to swap a few particles.
- memmove(&c->stars.parts[1], &c->stars.parts[0],
- n_copy * sizeof(struct spart));
-
- /* Update the spart->gpart links (shift by 1) */
- for (size_t i = 0; i < n_copy; ++i) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (c->stars.parts[i + 1].gpart == NULL) {
- error("Incorrectly linked spart!");
- }
-#endif
- c->stars.parts[i + 1].gpart->id_or_neg_offset--;
- }
- }
-
- /* Recursively shift all the stars to get a free spot at the start of the
- * current cell*/
- cell_recursively_shift_sparts(top, progeny, /* main_branch=*/1);
-
- /* Make sure the gravity will be recomputed for this particle in the next
- * step
- */
- struct cell *top2 = c;
- while (top2->parent != NULL) {
- top2->stars.ti_old_part = e->ti_current;
- top2 = top2->parent;
- }
- top2->stars.ti_old_part = e->ti_current;
-
- /* Release the lock */
- if (lock_unlock(&top->stars.star_formation_lock) != 0)
- error("Failed to unlock the top-level cell.");
-
- /* We now have an empty spart as the first particle in that cell */
- struct spart *sp = &c->stars.parts[0];
- bzero(sp, sizeof(struct spart));
-
- /* Give it a decent position */
- sp->x[0] = c->loc[0] + 0.5 * c->width[0];
- sp->x[1] = c->loc[1] + 0.5 * c->width[1];
- sp->x[2] = c->loc[2] + 0.5 * c->width[2];
-
- /* Set it to the current time-bin */
- sp->time_bin = e->min_active_bin;
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Specify it was drifted to this point */
- sp->ti_drift = e->ti_current;
-#endif
-
- /* Register that we used one of the free slots. */
- const size_t one = 1;
- atomic_sub(&e->s->nr_extra_sparts, one);
-
- return sp;
-}
-
-/**
- * @brief "Add" a #sink in a given #cell.
- *
- * This function will add a #sink at the start of the current cell's array by
- * shifting all the #sink in the top-level cell by one position. All the
- * pointers and cell counts are updated accordingly.
- *
- * @param e The #engine.
- * @param c The leaf-cell in which to add the #sink.
- *
- * @return A pointer to the newly added #sink. The sink has a been zeroed
- * and given a position within the cell as well as set to the minimal active
- * time bin.
- */
-struct sink *cell_add_sink(struct engine *e, struct cell *const c) {
- /* Perform some basic consitency checks */
- if (c->nodeID != engine_rank) error("Adding sink on a foreign node");
- if (c->grav.ti_old_part != e->ti_current) error("Undrifted cell!");
- if (c->split) error("Addition of sink performed above the leaf level");
-
- /* Progeny number at each level */
- int progeny[space_cell_maxdepth];
-#ifdef SWIFT_DEBUG_CHECKS
- for (int i = 0; i < space_cell_maxdepth; ++i) progeny[i] = -1;
-#endif
-
- /* Get the top-level this leaf cell is in and compute the progeny indices at
- each level */
- struct cell *top = c;
- while (top->parent != NULL) {
- /* What is the progeny index of the cell? */
- for (int k = 0; k < 8; ++k) {
- if (top->parent->progeny[k] == top) {
- progeny[(int)top->parent->depth] = k;
- }
- }
-
- /* Check that the cell was indeed drifted to this point to avoid future
- * issues */
-#ifdef SWIFT_DEBUG_CHECKS
- if (top->hydro.super != NULL && top->sinks.count > 0 &&
- top->sinks.ti_old_part != e->ti_current) {
- error("Cell had not been correctly drifted before sink formation");
- }
-#endif
-
- /* Climb up */
- top = top->parent;
- }
-
- /* Lock the top-level cell as we are going to operate on it */
- lock_lock(&top->sinks.sink_formation_lock);
-
- /* Are there any extra particles left? */
- if (top->sinks.count == top->sinks.count_total) {
-
- error("We ran out of free sink particles!");
-
- /* Release the local lock before exiting. */
- if (lock_unlock(&top->sinks.sink_formation_lock) != 0)
- error("Failed to unlock the top-level cell.");
-
- atomic_inc(&e->forcerebuild);
- return NULL;
- }
-
- /* Number of particles to shift in order to get a free space. */
- const size_t n_copy = &top->sinks.parts[top->sinks.count] - c->sinks.parts;
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (c->sinks.parts + n_copy > top->sinks.parts + top->sinks.count)
- error("Copying beyond the allowed range");
-#endif
-
- if (n_copy > 0) {
- // MATTHIEU: This can be improved. We don't need to copy everything, just
- // need to swap a few particles.
- memmove(&c->sinks.parts[1], &c->sinks.parts[0],
- n_copy * sizeof(struct sink));
-
- /* Update the sink->gpart links (shift by 1) */
- for (size_t i = 0; i < n_copy; ++i) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (c->sinks.parts[i + 1].gpart == NULL) {
- error("Incorrectly linked sink!");
- }
-#endif
- c->sinks.parts[i + 1].gpart->id_or_neg_offset--;
- }
- }
-
- /* Recursively shift all the sinks to get a free spot at the start of the
- * current cell*/
- cell_recursively_shift_sinks(top, progeny, /* main_branch=*/1);
-
- /* Make sure the gravity will be recomputed for this particle in the next
- * step
- */
- struct cell *top2 = c;
- while (top2->parent != NULL) {
- top2->sinks.ti_old_part = e->ti_current;
- top2 = top2->parent;
- }
- top2->sinks.ti_old_part = e->ti_current;
-
- /* Release the lock */
- if (lock_unlock(&top->sinks.sink_formation_lock) != 0)
- error("Failed to unlock the top-level cell.");
-
- /* We now have an empty spart as the first particle in that cell */
- struct sink *sp = &c->sinks.parts[0];
- bzero(sp, sizeof(struct sink));
-
- /* Give it a decent position */
- sp->x[0] = c->loc[0] + 0.5 * c->width[0];
- sp->x[1] = c->loc[1] + 0.5 * c->width[1];
- sp->x[2] = c->loc[2] + 0.5 * c->width[2];
-
- /* Set it to the current time-bin */
- sp->time_bin = e->min_active_bin;
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Specify it was drifted to this point */
- sp->ti_drift = e->ti_current;
-#endif
-
- /* Register that we used one of the free slots. */
- const size_t one = 1;
- atomic_sub(&e->s->nr_extra_sinks, one);
-
- return sp;
-}
-
-/**
- * @brief "Add" a #gpart in a given #cell.
- *
- * This function will add a #gpart at the start of the current cell's array by
- * shifting all the #gpart in the top-level cell by one position. All the
- * pointers and cell counts are updated accordingly.
- *
- * @param e The #engine.
- * @param c The leaf-cell in which to add the #gpart.
- *
- * @return A pointer to the newly added #gpart. The gpart has a been zeroed
- * and given a position within the cell as well as set to the minimal active
- * time bin.
+ * @brief Clear the drift flags on the given cell.
*/
-struct gpart *cell_add_gpart(struct engine *e, struct cell *c) {
- /* Perform some basic consitency checks */
- if (c->nodeID != engine_rank) error("Adding gpart on a foreign node");
- if (c->grav.ti_old_part != e->ti_current) error("Undrifted cell!");
- if (c->split) error("Addition of gpart performed above the leaf level");
-
- struct space *s = e->s;
-
- /* Progeny number at each level */
- int progeny[space_cell_maxdepth];
-#ifdef SWIFT_DEBUG_CHECKS
- for (int i = 0; i < space_cell_maxdepth; ++i) progeny[i] = -1;
-#endif
-
- /* Get the top-level this leaf cell is in and compute the progeny indices at
- each level */
- struct cell *top = c;
- while (top->parent != NULL) {
- /* What is the progeny index of the cell? */
- for (int k = 0; k < 8; ++k) {
- if (top->parent->progeny[k] == top) {
- progeny[(int)top->parent->depth] = k;
- }
- }
-
- /* Check that the cell was indeed drifted to this point to avoid future
- * issues */
-#ifdef SWIFT_DEBUG_CHECKS
- if (top->grav.super != NULL && top->grav.count > 0 &&
- top->grav.ti_old_part != e->ti_current) {
- error("Cell had not been correctly drifted before adding a gpart");
- }
-#endif
-
- /* Climb up */
- top = top->parent;
- }
-
- /* Lock the top-level cell as we are going to operate on it */
- lock_lock(&top->grav.star_formation_lock);
-
- /* Are there any extra particles left? */
- if (top->grav.count == top->grav.count_total) {
-
- message("We ran out of free gravity particles!");
-
- /* Release the local lock before exiting. */
- if (lock_unlock(&top->grav.star_formation_lock) != 0)
- error("Failed to unlock the top-level cell.");
-
- atomic_inc(&e->forcerebuild);
- return NULL;
- }
-
- /* Number of particles to shift in order to get a free space. */
- const size_t n_copy = &top->grav.parts[top->grav.count] - c->grav.parts;
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (c->grav.parts + n_copy > top->grav.parts + top->grav.count)
- error("Copying beyond the allowed range");
-#endif
-
- if (n_copy > 0) {
- // MATTHIEU: This can be improved. We don't need to copy everything, just
- // need to swap a few particles.
- memmove(&c->grav.parts[1], &c->grav.parts[0],
- n_copy * sizeof(struct gpart));
-
- /* Update the gpart->spart links (shift by 1) */
- struct gpart *gparts = c->grav.parts;
- for (size_t i = 0; i < n_copy; ++i) {
- if (gparts[i + 1].type == swift_type_gas) {
- s->parts[-gparts[i + 1].id_or_neg_offset].gpart++;
- } else if (gparts[i + 1].type == swift_type_stars) {
- s->sparts[-gparts[i + 1].id_or_neg_offset].gpart++;
- } else if (gparts[i + 1].type == swift_type_black_hole) {
- s->bparts[-gparts[i + 1].id_or_neg_offset].gpart++;
- }
- }
- }
-
- /* Recursively shift all the gpart to get a free spot at the start of the
- * current cell*/
- cell_recursively_shift_gparts(top, progeny, /* main_branch=*/1);
-
- /* Make sure the gravity will be recomputed for this particle in the next
- * step
- */
- struct cell *top2 = c;
- while (top2->parent != NULL) {
- top2->grav.ti_old_part = e->ti_current;
- top2 = top2->parent;
- }
- top2->grav.ti_old_part = e->ti_current;
-
- /* Release the lock */
- if (lock_unlock(&top->grav.star_formation_lock) != 0)
- error("Failed to unlock the top-level cell.");
-
- /* We now have an empty gpart as the first particle in that cell */
- struct gpart *gp = &c->grav.parts[0];
- bzero(gp, sizeof(struct gpart));
-
- /* Give it a decent position */
- gp->x[0] = c->loc[0] + 0.5 * c->width[0];
- gp->x[1] = c->loc[1] + 0.5 * c->width[1];
- gp->x[2] = c->loc[2] + 0.5 * c->width[2];
-
- /* Set it to the current time-bin */
- gp->time_bin = e->min_active_bin;
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Specify it was drifted to this point */
- gp->ti_drift = e->ti_current;
-#endif
-
- /* Register that we used one of the free slots. */
- const size_t one = 1;
- atomic_sub(&e->s->nr_extra_gparts, one);
-
- return gp;
+void cell_clear_drift_flags(struct cell *c, void *data) {
+ cell_clear_flag(c, cell_flag_do_hydro_drift | cell_flag_do_hydro_sub_drift |
+ cell_flag_do_grav_drift | cell_flag_do_grav_sub_drift |
+ cell_flag_do_bh_drift | cell_flag_do_bh_sub_drift |
+ cell_flag_do_stars_drift |
+ cell_flag_do_stars_sub_drift |
+ cell_flag_do_sink_drift | cell_flag_do_sink_sub_drift);
}
/**
- * @brief "Remove" a gas particle from the calculation.
- *
- * The particle is inhibited and will officially be removed at the next
- * rebuild.
- *
- * @param e The #engine running on this node.
- * @param c The #cell from which to remove the particle.
- * @param p The #part to remove.
- * @param xp The extended data of the particle to remove.
+ * @brief Clear the limiter flags on the given cell.
*/
-void cell_remove_part(const struct engine *e, struct cell *c, struct part *p,
- struct xpart *xp) {
- /* Quick cross-check */
- if (c->nodeID != e->nodeID)
- error("Can't remove a particle in a foreign cell.");
-
- /* Don't remove a particle twice */
- if (p->time_bin == time_bin_inhibited) return;
-
- /* Mark the particle as inhibited */
- p->time_bin = time_bin_inhibited;
-
- /* Mark the gpart as inhibited and stand-alone */
- if (p->gpart) {
- p->gpart->time_bin = time_bin_inhibited;
- p->gpart->id_or_neg_offset = p->id;
- p->gpart->type = swift_type_dark_matter;
- }
-
- /* Update the space-wide counters */
- const size_t one = 1;
- atomic_add(&e->s->nr_inhibited_parts, one);
- if (p->gpart) {
- atomic_add(&e->s->nr_inhibited_gparts, one);
- }
-
- /* Un-link the part */
- p->gpart = NULL;
+void cell_clear_limiter_flags(struct cell *c, void *data) {
+ cell_clear_flag(c,
+ cell_flag_do_hydro_limiter | cell_flag_do_hydro_sub_limiter);
}
/**
- * @brief "Remove" a gravity particle from the calculation.
- *
- * The particle is inhibited and will officially be removed at the next
- * rebuild.
+ * @brief Set the super-cell pointers for all cells in a hierarchy.
*
- * @param e The #engine running on this node.
- * @param c The #cell from which to remove the particle.
- * @param gp The #gpart to remove.
+ * @param c The top-level #cell to play with.
+ * @param super Pointer to the deepest cell with tasks in this part of the
+ * tree.
+ * @param with_hydro Are we running with hydrodynamics on?
+ * @param with_grav Are we running with gravity on?
*/
-void cell_remove_gpart(const struct engine *e, struct cell *c,
- struct gpart *gp) {
-
- /* Quick cross-check */
- if (c->nodeID != e->nodeID)
- error("Can't remove a particle in a foreign cell.");
-
- /* Don't remove a particle twice */
- if (gp->time_bin == time_bin_inhibited) return;
-
- /* Quick cross-check */
- if (c->nodeID != e->nodeID)
- error("Can't remove a particle in a foreign cell.");
-
- if (gp->type == swift_type_dark_matter_background)
- error("Can't remove a DM background particle!");
+void cell_set_super(struct cell *c, struct cell *super, const int with_hydro,
+ const int with_grav) {
+ /* Are we in a cell which is either the hydro or gravity super? */
+ if (super == NULL && ((with_hydro && c->hydro.super != NULL) ||
+ (with_grav && c->grav.super != NULL)))
+ super = c;
- /* Mark the particle as inhibited */
- gp->time_bin = time_bin_inhibited;
+ /* Set the super-cell */
+ c->super = super;
- /* Update the space-wide counters */
- const size_t one = 1;
- atomic_add(&e->s->nr_inhibited_gparts, one);
+ /* Recurse */
+ if (c->split)
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL)
+ cell_set_super(c->progeny[k], super, with_hydro, with_grav);
}
/**
- * @brief "Remove" a star particle from the calculation.
- *
- * The particle is inhibited and will officially be removed at the next
- * rebuild.
+ * @brief Set the super-cell pointers for all cells in a hierarchy.
*
- * @param e The #engine running on this node.
- * @param c The #cell from which to remove the particle.
- * @param sp The #spart to remove.
+ * @param c The top-level #cell to play with.
+ * @param super_hydro Pointer to the deepest cell with tasks in this part of
+ * the tree.
*/
-void cell_remove_spart(const struct engine *e, struct cell *c,
- struct spart *sp) {
- /* Quick cross-check */
- if (c->nodeID != e->nodeID)
- error("Can't remove a particle in a foreign cell.");
-
- /* Don't remove a particle twice */
- if (sp->time_bin == time_bin_inhibited) return;
-
- /* Mark the particle as inhibited and stand-alone */
- sp->time_bin = time_bin_inhibited;
- if (sp->gpart) {
- sp->gpart->time_bin = time_bin_inhibited;
- sp->gpart->id_or_neg_offset = sp->id;
- sp->gpart->type = swift_type_dark_matter;
- }
+void cell_set_super_hydro(struct cell *c, struct cell *super_hydro) {
+ /* Are we in a cell with some kind of self/pair task ? */
+ if (super_hydro == NULL && c->hydro.density != NULL) super_hydro = c;
- /* Update the space-wide counters */
- const size_t one = 1;
- atomic_add(&e->s->nr_inhibited_sparts, one);
- if (sp->gpart) {
- atomic_add(&e->s->nr_inhibited_gparts, one);
- }
+ /* Set the super-cell */
+ c->hydro.super = super_hydro;
- /* Un-link the spart */
- sp->gpart = NULL;
+ /* Recurse */
+ if (c->split)
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL)
+ cell_set_super_hydro(c->progeny[k], super_hydro);
}
/**
- * @brief "Remove" a black hole particle from the calculation.
- *
- * The particle is inhibited and will officially be removed at the next
- * rebuild.
+ * @brief Set the super-cell pointers for all cells in a hierarchy.
*
- * @param e The #engine running on this node.
- * @param c The #cell from which to remove the particle.
- * @param bp The #bpart to remove.
+ * @param c The top-level #cell to play with.
+ * @param super_gravity Pointer to the deepest cell with tasks in this part of
+ * the tree.
*/
-void cell_remove_bpart(const struct engine *e, struct cell *c,
- struct bpart *bp) {
-
- /* Quick cross-check */
- if (c->nodeID != e->nodeID)
- error("Can't remove a particle in a foreign cell.");
-
- /* Don't remove a particle twice */
- if (bp->time_bin == time_bin_inhibited) return;
-
- /* Mark the particle as inhibited and stand-alone */
- bp->time_bin = time_bin_inhibited;
- if (bp->gpart) {
- bp->gpart->time_bin = time_bin_inhibited;
- bp->gpart->id_or_neg_offset = bp->id;
- bp->gpart->type = swift_type_dark_matter;
- }
+void cell_set_super_gravity(struct cell *c, struct cell *super_gravity) {
+ /* Are we in a cell with some kind of self/pair task ? */
+ if (super_gravity == NULL && (c->grav.grav != NULL || c->grav.mm != NULL))
+ super_gravity = c;
- /* Update the space-wide counters */
- const size_t one = 1;
- atomic_add(&e->s->nr_inhibited_bparts, one);
- if (bp->gpart) {
- atomic_add(&e->s->nr_inhibited_gparts, one);
- }
+ /* Set the super-cell */
+ c->grav.super = super_gravity;
- /* Un-link the bpart */
- bp->gpart = NULL;
+ /* Recurse */
+ if (c->split)
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL)
+ cell_set_super_gravity(c->progeny[k], super_gravity);
}
/**
- * @brief "Remove" a gas particle from the calculation and convert its gpart
- * friend to a dark matter particle.
- *
- * Note that the #part is not destroyed. The pointer is still valid
- * after this call and the properties of the #part are not altered
- * apart from the time-bin and #gpart pointer.
- * The particle is inhibited and will officially be removed at the next
- * rebuild.
- *
- * @param e The #engine running on this node.
- * @param c The #cell from which to remove the particle.
- * @param p The #part to remove.
- * @param xp The extended data of the particle to remove.
+ * @brief Mapper function to set the super pointer of the cells.
*
- * @return Pointer to the #gpart the #part has become. It carries the
- * ID of the #part and has a dark matter type.
+ * @param map_data The top-level cells.
+ * @param num_elements The number of top-level cells.
+ * @param extra_data Unused parameter.
*/
-struct gpart *cell_convert_part_to_gpart(const struct engine *e, struct cell *c,
- struct part *p, struct xpart *xp) {
- /* Quick cross-checks */
- if (c->nodeID != e->nodeID)
- error("Can't remove a particle in a foreign cell.");
-
- if (p->gpart == NULL)
- error("Trying to convert part without gpart friend to dark matter!");
-
- /* Get a handle */
- struct gpart *gp = p->gpart;
-
- /* Mark the particle as inhibited */
- p->time_bin = time_bin_inhibited;
+void cell_set_super_mapper(void *map_data, int num_elements, void *extra_data) {
+ const struct engine *e = (const struct engine *)extra_data;
- /* Un-link the part */
- p->gpart = NULL;
+ const int with_hydro = (e->policy & engine_policy_hydro);
+ const int with_grav = (e->policy & engine_policy_self_gravity) ||
+ (e->policy & engine_policy_external_gravity);
- /* Mark the gpart as dark matter */
- gp->type = swift_type_dark_matter;
- gp->id_or_neg_offset = p->id;
+ for (int ind = 0; ind < num_elements; ind++) {
+ struct cell *c = &((struct cell *)map_data)[ind];
-#ifdef SWIFT_DEBUG_CHECKS
- gp->ti_kick = p->ti_kick;
+ /* All top-level cells get an MPI tag. */
+#ifdef WITH_MPI
+ cell_ensure_tagged(c);
#endif
- /* Update the space-wide counters */
- atomic_inc(&e->s->nr_inhibited_parts);
+ /* Super-pointer for hydro */
+ if (with_hydro) cell_set_super_hydro(c, NULL);
+
+ /* Super-pointer for gravity */
+ if (with_grav) cell_set_super_gravity(c, NULL);
- return gp;
+ /* Super-pointer for common operations */
+ cell_set_super(c, NULL, with_hydro, with_grav);
+ }
}
/**
- * @brief "Remove" a spart particle from the calculation and convert its gpart
- * friend to a dark matter particle.
- *
- * Note that the #spart is not destroyed. The pointer is still valid
- * after this call and the properties of the #spart are not altered
- * apart from the time-bin and #gpart pointer.
- * The particle is inhibited and will officially be removed at the next
- * rebuild.
+ * @brief Does this cell or any of its children have any task ?
*
- * @param e The #engine running on this node.
- * @param c The #cell from which to remove the particle.
- * @param sp The #spart to remove.
+ * We use the timestep-related tasks to probe this as these always
+ * exist in a cell hierarchy that has any kind of task.
*
- * @return Pointer to the #gpart the #spart has become. It carries the
- * ID of the #spart and has a dark matter type.
+ * @param c The #cell to probe.
*/
-struct gpart *cell_convert_spart_to_gpart(const struct engine *e,
- struct cell *c, struct spart *sp) {
- /* Quick cross-check */
- if (c->nodeID != e->nodeID)
- error("Can't remove a particle in a foreign cell.");
-
- if (sp->gpart == NULL)
- error("Trying to convert spart without gpart friend to dark matter!");
-
- /* Get a handle */
- struct gpart *gp = sp->gpart;
-
- /* Mark the particle as inhibited */
- sp->time_bin = time_bin_inhibited;
-
- /* Un-link the spart */
- sp->gpart = NULL;
-
- /* Mark the gpart as dark matter */
- gp->type = swift_type_dark_matter;
- gp->id_or_neg_offset = sp->id;
-
-#ifdef SWIFT_DEBUG_CHECKS
- gp->ti_kick = sp->ti_kick;
+int cell_has_tasks(struct cell *c) {
+#ifdef WITH_MPI
+ if (c->timestep != NULL || c->mpi.recv != NULL) return 1;
+#else
+ if (c->timestep != NULL) return 1;
#endif
- /* Update the space-wide counters */
- atomic_inc(&e->s->nr_inhibited_sparts);
-
- return gp;
+ if (c->split) {
+ int count = 0;
+ for (int k = 0; k < 8; ++k)
+ if (c->progeny[k] != NULL) count += cell_has_tasks(c->progeny[k]);
+ return count;
+ } else {
+ return 0;
+ }
}
/**
- * @brief "Remove" a #part from a #cell and replace it with a #spart
- * connected to the same #gpart.
- *
- * Note that the #part is not destroyed. The pointer is still valid
- * after this call and the properties of the #part are not altered
- * apart from the time-bin and #gpart pointer.
- * The particle is inhibited and will officially be removed at the next
- * rebuild.
+ * @brief Resets all the sorting properties for the stars in a given cell
+ * hierarchy.
*
- * @param e The #engine.
- * @param c The #cell from which to remove the #part.
- * @param p The #part to remove (must be inside c).
- * @param xp The extended data of the #part.
+ * The clear_unused_flags argument can be used to additionally clean up all
+ * the flags demanding a sort for the given cell. This should be used with
+ * caution as it will prevent the sort tasks from doing anything on that cell
+ * until these flags are reset.
*
- * @return A fresh #spart with the same ID, position, velocity and
- * time-bin as the original #part.
+ * @param c The #cell to clean.
+ * @param clear_unused_flags Do we also clean the flags demanding a sort?
*/
-struct spart *cell_convert_part_to_spart(struct engine *e, struct cell *c,
- struct part *p, struct xpart *xp) {
- /* Quick cross-check */
- if (c->nodeID != e->nodeID)
- error("Can't remove a particle in a foreign cell.");
-
- if (p->gpart == NULL)
- error("Trying to convert part without gpart friend to star!");
-
- /* Create a fresh (empty) spart */
- struct spart *sp = cell_add_spart(e, c);
-
- /* Did we run out of free spart slots? */
- if (sp == NULL) return NULL;
-
- /* Copy over the distance since rebuild */
- sp->x_diff[0] = xp->x_diff[0];
- sp->x_diff[1] = xp->x_diff[1];
- sp->x_diff[2] = xp->x_diff[2];
-
- /* Destroy the gas particle and get it's gpart friend */
- struct gpart *gp = cell_convert_part_to_gpart(e, c, p, xp);
-
- /* Assign the ID back */
- sp->id = gp->id_or_neg_offset;
- gp->type = swift_type_stars;
-
- /* Re-link things */
- sp->gpart = gp;
- gp->id_or_neg_offset = -(sp - e->s->sparts);
-
- /* Synchronize clocks */
- gp->time_bin = sp->time_bin;
-
- /* Synchronize masses, positions and velocities */
- sp->mass = gp->mass;
- sp->x[0] = gp->x[0];
- sp->x[1] = gp->x[1];
- sp->x[2] = gp->x[2];
- sp->v[0] = gp->v_full[0];
- sp->v[1] = gp->v_full[1];
- sp->v[2] = gp->v_full[2];
+void cell_clear_stars_sort_flags(struct cell *c, const int clear_unused_flags) {
-#ifdef SWIFT_DEBUG_CHECKS
- sp->ti_kick = gp->ti_kick;
- gp->ti_drift = sp->ti_drift;
-#endif
+ /* Clear the flags that have not been reset by the sort task? */
+ if (clear_unused_flags) {
+ c->stars.requires_sorts = 0;
+ c->stars.do_sort = 0;
+ cell_clear_flag(c, cell_flag_do_stars_sub_sort);
+ }
- /* Set a smoothing length */
- sp->h = max(c->stars.h_max, c->hydro.h_max);
+ /* Indicate that the cell is not sorted and cancel the pointer sorting
+ * arrays.
+ */
+ c->stars.sorted = 0;
+ cell_free_stars_sorts(c);
- /* Here comes the Sun! */
- return sp;
+ /* Recurse if possible */
+ if (c->split) {
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL)
+ cell_clear_stars_sort_flags(c->progeny[k], clear_unused_flags);
+ }
}
/**
- * @brief Add a new #spart based on a #part and link it to a new #gpart.
- * The part and xpart are not changed.
+ * @brief Resets all the sorting properties for the hydro in a given cell
+ * hierarchy.
*
- * @param e The #engine.
- * @param c The #cell from which to remove the #part.
- * @param p The #part to remove (must be inside c).
- * @param xp The extended data of the #part.
+ * The clear_unused_flags argument can be used to additionally clean up all
+ * the flags demanding a sort for the given cell. This should be used with
+ * caution as it will prevent the sort tasks from doing anything on that cell
+ * until these flags are reset.
*
- * @return A fresh #spart with the same ID, position, velocity and
- * time-bin as the original #part.
+ * @param c The #cell to clean.
+ * @param clear_unused_flags Do we also clean the flags demanding a sort?
*/
-struct spart *cell_spawn_new_spart_from_part(struct engine *e, struct cell *c,
- const struct part *p,
- const struct xpart *xp) {
- /* Quick cross-check */
- if (c->nodeID != e->nodeID)
- error("Can't spawn a particle in a foreign cell.");
-
- if (p->gpart == NULL)
- error("Trying to create a new spart from a part without gpart friend!");
-
- /* Create a fresh (empty) spart */
- struct spart *sp = cell_add_spart(e, c);
-
- /* Did we run out of free spart slots? */
- if (sp == NULL) return NULL;
-
- /* Copy over the distance since rebuild */
- sp->x_diff[0] = xp->x_diff[0];
- sp->x_diff[1] = xp->x_diff[1];
- sp->x_diff[2] = xp->x_diff[2];
-
- /* Create a new gpart */
- struct gpart *gp = cell_add_gpart(e, c);
-
- /* Did we run out of free gpart slots? */
- if (gp == NULL) {
- /* Remove the particle created */
- cell_remove_spart(e, c, sp);
- return NULL;
- }
-
- /* Copy the gpart */
- *gp = *p->gpart;
-
- /* Assign the ID. */
- sp->id = space_get_new_unique_id(e->s);
- gp->type = swift_type_stars;
-
- /* Re-link things */
- sp->gpart = gp;
- gp->id_or_neg_offset = -(sp - e->s->sparts);
-
- /* Synchronize clocks */
- gp->time_bin = sp->time_bin;
-
- /* Synchronize masses, positions and velocities */
- sp->mass = hydro_get_mass(p);
- sp->x[0] = p->x[0];
- sp->x[1] = p->x[1];
- sp->x[2] = p->x[2];
- sp->v[0] = xp->v_full[0];
- sp->v[1] = xp->v_full[1];
- sp->v[2] = xp->v_full[2];
+void cell_clear_hydro_sort_flags(struct cell *c, const int clear_unused_flags) {
-#ifdef SWIFT_DEBUG_CHECKS
- sp->ti_kick = p->ti_kick;
- sp->ti_drift = p->ti_drift;
-#endif
+ /* Clear the flags that have not been reset by the sort task? */
+ if (clear_unused_flags) {
+ c->hydro.do_sort = 0;
+ c->hydro.requires_sorts = 0;
+ cell_clear_flag(c, cell_flag_do_hydro_sub_sort);
+ }
- /* Set a smoothing length */
- sp->h = p->h;
+ /* Indicate that the cell is not sorted and cancel the pointer sorting
+ * arrays.
+ */
+ c->hydro.sorted = 0;
+ cell_free_hydro_sorts(c);
- /* Here comes the Sun! */
- return sp;
+ /* Recurse if possible */
+ if (c->split) {
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL)
+ cell_clear_hydro_sort_flags(c->progeny[k], clear_unused_flags);
+ }
}
/**
- * @brief "Remove" a #part from a #cell and replace it with a #sink
- * connected to the same #gpart.
- *
- * Note that the #part is not destroyed. The pointer is still valid
- * after this call and the properties of the #part are not altered
- * apart from the time-bin and #gpart pointer.
- * The particle is inhibited and will officially be removed at the next
- * rebuild.
- *
- * @param e The #engine.
- * @param c The #cell from which to remove the #part.
- * @param p The #part to remove (must be inside c).
- * @param xp The extended data of the #part.
- *
- * @return A fresh #sink with the same ID, position, velocity and
- * time-bin as the original #part.
+ * @brief Recursively checks that all particles in a cell have a time-step
*/
-struct sink *cell_convert_part_to_sink(struct engine *e, struct cell *c,
- struct part *p, struct xpart *xp) {
- /* Quick cross-check */
- if (c->nodeID != e->nodeID)
- error("Can't remove a particle in a foreign cell.");
-
- if (p->gpart == NULL)
- error("Trying to convert part without gpart friend to sink!");
+void cell_check_timesteps(const struct cell *c, const integertime_t ti_current,
+ const timebin_t max_bin) {
+#ifdef SWIFT_DEBUG_CHECKS
- /* Create a fresh (empty) sink */
- struct sink *sp = cell_add_sink(e, c);
+ if (c->hydro.ti_end_min == 0 && c->grav.ti_end_min == 0 &&
+ c->stars.ti_end_min == 0 && c->black_holes.ti_end_min == 0 &&
+ c->sinks.ti_end_min == 0 && c->nr_tasks > 0)
+ error("Cell without assigned time-step");
- /* Did we run out of free sink slots? */
- if (sp == NULL) return NULL;
+ if (c->split) {
+ for (int k = 0; k < 8; ++k)
+ if (c->progeny[k] != NULL)
+ cell_check_timesteps(c->progeny[k], ti_current, max_bin);
+ } else {
+ if (c->nodeID == engine_rank)
+ for (int i = 0; i < c->hydro.count; ++i)
+ if (c->hydro.parts[i].time_bin == 0)
+ error("Particle without assigned time-bin");
+ }
- /* Copy over the distance since rebuild */
- sp->x_diff[0] = xp->x_diff[0];
- sp->x_diff[1] = xp->x_diff[1];
- sp->x_diff[2] = xp->x_diff[2];
+ /* Other checks not relevent when starting-up */
+ if (ti_current == 0) return;
- /* Destroy the gas particle and get it's gpart friend */
- struct gpart *gp = cell_convert_part_to_gpart(e, c, p, xp);
+ integertime_t ti_end_min = max_nr_timesteps;
+ integertime_t ti_end_max = 0;
+ integertime_t ti_beg_max = 0;
- /* Assign the ID back */
- sp->id = gp->id_or_neg_offset;
- gp->type = swift_type_sink;
+ int count = 0;
- /* Re-link things */
- sp->gpart = gp;
- gp->id_or_neg_offset = -(sp - e->s->sinks);
+ for (int i = 0; i < c->hydro.count; ++i) {
- /* Synchronize clocks */
- gp->time_bin = sp->time_bin;
+ const struct part *p = &c->hydro.parts[i];
+ if (p->time_bin == time_bin_inhibited) continue;
+ if (p->time_bin == time_bin_not_created) continue;
- /* Synchronize masses, positions and velocities */
- sp->mass = gp->mass;
- sp->x[0] = gp->x[0];
- sp->x[1] = gp->x[1];
- sp->x[2] = gp->x[2];
- sp->v[0] = gp->v_full[0];
- sp->v[1] = gp->v_full[1];
- sp->v[2] = gp->v_full[2];
+ ++count;
-#ifdef SWIFT_DEBUG_CHECKS
- sp->ti_kick = gp->ti_kick;
- gp->ti_drift = sp->ti_drift;
-#endif
+ integertime_t ti_end, ti_beg;
- /* Set a smoothing length */
- sp->r_cut = e->sink_properties->cut_off_radius;
+ if (p->time_bin <= max_bin) {
+ integertime_t time_step = get_integer_timestep(p->time_bin);
+ ti_end = get_integer_time_end(ti_current, p->time_bin) + time_step;
+ ti_beg = get_integer_time_begin(ti_current + 1, p->time_bin);
+ } else {
+ ti_end = get_integer_time_end(ti_current, p->time_bin);
+ ti_beg = get_integer_time_begin(ti_current + 1, p->time_bin);
+ }
- /* Here comes the Sink! */
- return sp;
-}
+ ti_end_min = min(ti_end, ti_end_min);
+ ti_end_max = max(ti_end, ti_end_max);
+ ti_beg_max = max(ti_beg, ti_beg_max);
+ }
-/**
- * @brief Re-arrange the #part in a top-level cell such that all the extra
- * ones for on-the-fly creation are located at the end of the array.
- *
- * @param c The #cell to sort.
- * @param parts_offset The offset between the first #part in the array and the
- * first #part in the global array in the space structure (for re-linking).
- */
-void cell_reorder_extra_parts(struct cell *c, const ptrdiff_t parts_offset) {
- struct part *parts = c->hydro.parts;
- struct xpart *xparts = c->hydro.xparts;
- const int count_real = c->hydro.count;
+ /* Only check cells that have at least one non-inhibited particle */
+ if (count > 0) {
- if (c->depth != 0 || c->nodeID != engine_rank)
- error("This function should only be called on local top-level cells!");
+ if (count != c->hydro.count) {
- int first_not_extra = count_real;
+ /* Note that we use a < as the particle with the smallest time-bin
+ might have been swallowed. This means we will run this cell with
+ 0 active particles but that's not wrong */
+ if (ti_end_min < c->hydro.ti_end_min)
+ error(
+ "Non-matching ti_end_min. Cell=%lld true=%lld ti_current=%lld "
+ "depth=%d",
+ c->hydro.ti_end_min, ti_end_min, ti_current, c->depth);
- /* Find extra particles */
- for (int i = 0; i < count_real; ++i) {
- if (parts[i].time_bin == time_bin_not_created) {
- /* Find the first non-extra particle after the end of the
- real particles */
- while (parts[first_not_extra].time_bin == time_bin_not_created) {
- ++first_not_extra;
- }
+ } else /* Normal case: nothing was swallowed/converted */ {
+ if (ti_end_min != c->hydro.ti_end_min)
+ error(
+ "Non-matching ti_end_min. Cell=%lld true=%lld ti_current=%lld "
+ "depth=%d",
+ c->hydro.ti_end_min, ti_end_min, ti_current, c->depth);
+ }
-#ifdef SWIFT_DEBUG_CHECKS
- if (first_not_extra >= count_real + space_extra_parts)
- error("Looking for extra particles beyond this cell's range!");
-#endif
+ if (ti_end_max > c->hydro.ti_end_max)
+ error(
+ "Non-matching ti_end_max. Cell=%lld true=%lld ti_current=%lld "
+ "depth=%d",
+ c->hydro.ti_end_max, ti_end_max, ti_current, c->depth);
- /* Swap everything, including g-part pointer */
- memswap(&parts[i], &parts[first_not_extra], sizeof(struct part));
- memswap(&xparts[i], &xparts[first_not_extra], sizeof(struct xpart));
- if (parts[i].gpart)
- parts[i].gpart->id_or_neg_offset = -(i + parts_offset);
- }
+ if (ti_beg_max != c->hydro.ti_beg_max)
+ error(
+ "Non-matching ti_beg_max. Cell=%lld true=%lld ti_current=%lld "
+ "depth=%d",
+ c->hydro.ti_beg_max, ti_beg_max, ti_current, c->depth);
}
-#ifdef SWIFT_DEBUG_CHECKS
- for (int i = 0; i < c->hydro.count_total; ++i) {
- if (parts[i].time_bin == time_bin_not_created && i < c->hydro.count) {
- error("Extra particle before the end of the regular array");
- }
- if (parts[i].time_bin != time_bin_not_created && i >= c->hydro.count) {
- error("Regular particle after the end of the regular array");
- }
- }
+#else
+ error("Calling debugging code without debugging flag activated.");
#endif
}
-/**
- * @brief Re-arrange the #spart in a top-level cell such that all the extra
- * ones for on-the-fly creation are located at the end of the array.
- *
- * @param c The #cell to sort.
- * @param sparts_offset The offset between the first #spart in the array and
- * the first #spart in the global array in the space structure (for
- * re-linking).
- */
-void cell_reorder_extra_sparts(struct cell *c, const ptrdiff_t sparts_offset) {
- struct spart *sparts = c->stars.parts;
- const int count_real = c->stars.count;
-
- if (c->depth != 0 || c->nodeID != engine_rank)
- error("This function should only be called on local top-level cells!");
-
- int first_not_extra = count_real;
-
- /* Find extra particles */
- for (int i = 0; i < count_real; ++i) {
- if (sparts[i].time_bin == time_bin_not_created) {
- /* Find the first non-extra particle after the end of the
- real particles */
- while (sparts[first_not_extra].time_bin == time_bin_not_created) {
- ++first_not_extra;
- }
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (first_not_extra >= count_real + space_extra_sparts)
- error("Looking for extra particles beyond this cell's range!");
-#endif
-
- /* Swap everything, including g-part pointer */
- memswap(&sparts[i], &sparts[first_not_extra], sizeof(struct spart));
- if (sparts[i].gpart)
- sparts[i].gpart->id_or_neg_offset = -(i + sparts_offset);
- sparts[first_not_extra].gpart = NULL;
+void cell_check_spart_pos(const struct cell *c,
+ const struct spart *global_sparts) {
#ifdef SWIFT_DEBUG_CHECKS
- if (sparts[first_not_extra].time_bin != time_bin_not_created)
- error("Incorrect swap occured!");
-#endif
- }
- }
-#ifdef SWIFT_DEBUG_CHECKS
- for (int i = 0; i < c->stars.count_total; ++i) {
- if (sparts[i].time_bin == time_bin_not_created && i < c->stars.count) {
- error("Extra particle before the end of the regular array");
- }
- if (sparts[i].time_bin != time_bin_not_created && i >= c->stars.count) {
- error("Regular particle after the end of the regular array");
- }
+ /* Recurse */
+ if (c->split) {
+ for (int k = 0; k < 8; ++k)
+ if (c->progeny[k] != NULL)
+ cell_check_spart_pos(c->progeny[k], global_sparts);
}
-#endif
-}
-/**
- * @brief Re-arrange the #sink in a top-level cell such that all the extra
- * ones for on-the-fly creation are located at the end of the array.
- *
- * @param c The #cell to sort.
- * @param sinks_offset The offset between the first #sink in the array and
- * the first #sink in the global array in the space structure (for
- * re-linking).
- */
-void cell_reorder_extra_sinks(struct cell *c, const ptrdiff_t sinks_offset) {
- struct sink *sinks = c->sinks.parts;
- const int count_real = c->sinks.count;
-
- if (c->depth != 0 || c->nodeID != engine_rank)
- error("This function should only be called on local top-level cells!");
-
- int first_not_extra = count_real;
-
- /* Find extra particles */
- for (int i = 0; i < count_real; ++i) {
- if (sinks[i].time_bin == time_bin_not_created) {
- /* Find the first non-extra particle after the end of the
- real particles */
- while (sinks[first_not_extra].time_bin == time_bin_not_created) {
- ++first_not_extra;
- }
+ struct spart *sparts = c->stars.parts;
+ const int count = c->stars.count;
+ for (int i = 0; i < count; ++i) {
+ const struct spart *sp = &sparts[i];
+ if ((sp->x[0] < c->loc[0] / space_stretch) ||
+ (sp->x[1] < c->loc[1] / space_stretch) ||
+ (sp->x[2] < c->loc[2] / space_stretch) ||
+ (sp->x[0] >= (c->loc[0] + c->width[0]) * space_stretch) ||
+ (sp->x[1] >= (c->loc[1] + c->width[1]) * space_stretch) ||
+ (sp->x[2] >= (c->loc[2] + c->width[2]) * space_stretch))
+ error("spart not in its cell!");
-#ifdef SWIFT_DEBUG_CHECKS
- if (first_not_extra >= count_real + space_extra_sinks)
- error("Looking for extra particles beyond this cell's range!");
-#endif
+ if (sp->time_bin != time_bin_not_created &&
+ sp->time_bin != time_bin_inhibited) {
+ const struct gpart *gp = sp->gpart;
+ if (gp == NULL && sp->time_bin != time_bin_not_created)
+ error("Unlinked spart!");
- /* Swap everything, including g-part pointer */
- memswap(&sinks[i], &sinks[first_not_extra], sizeof(struct sink));
- if (sinks[i].gpart)
- sinks[i].gpart->id_or_neg_offset = -(i + sinks_offset);
- sinks[first_not_extra].gpart = NULL;
-#ifdef SWIFT_DEBUG_CHECKS
- if (sinks[first_not_extra].time_bin != time_bin_not_created)
- error("Incorrect swap occured!");
-#endif
+ if (&global_sparts[-gp->id_or_neg_offset] != sp)
+ error("Incorrectly linked spart!");
}
}
-#ifdef SWIFT_DEBUG_CHECKS
- for (int i = 0; i < c->sinks.count_total; ++i) {
- if (sinks[i].time_bin == time_bin_not_created && i < c->sinks.count) {
- error("Extra particle before the end of the regular array");
- }
- if (sinks[i].time_bin != time_bin_not_created && i >= c->sinks.count) {
- error("Regular particle after the end of the regular array");
- }
- }
+#else
+ error("Calling a degugging function outside debugging mode.");
#endif
}
/**
- * @brief Re-arrange the #gpart in a top-level cell such that all the extra
- * ones for on-the-fly creation are located at the end of the array.
+ * @brief Checks that a cell and all its progenies have cleared their sort
+ * flags.
+ *
+ * Should only be used for debugging purposes.
*
- * @param c The #cell to sort.
- * @param parts The global array of #part (for re-linking).
- * @param sparts The global array of #spart (for re-linking).
+ * @param c The #cell to check.
*/
-void cell_reorder_extra_gparts(struct cell *c, struct part *parts,
- struct spart *sparts) {
- struct gpart *gparts = c->grav.parts;
- const int count_real = c->grav.count;
-
- if (c->depth != 0 || c->nodeID != engine_rank)
- error("This function should only be called on local top-level cells!");
-
- int first_not_extra = count_real;
-
- /* Find extra particles */
- for (int i = 0; i < count_real; ++i) {
- if (gparts[i].time_bin == time_bin_not_created) {
- /* Find the first non-extra particle after the end of the
- real particles */
- while (gparts[first_not_extra].time_bin == time_bin_not_created) {
- ++first_not_extra;
- }
+void cell_check_sort_flags(const struct cell *c) {
#ifdef SWIFT_DEBUG_CHECKS
- if (first_not_extra >= count_real + space_extra_gparts)
- error("Looking for extra particles beyond this cell's range!");
-#endif
+ const int do_hydro_sub_sort = cell_get_flag(c, cell_flag_do_hydro_sub_sort);
+ const int do_stars_sub_sort = cell_get_flag(c, cell_flag_do_stars_sub_sort);
- /* Swap everything (including pointers) */
- memswap_unaligned(&gparts[i], &gparts[first_not_extra],
- sizeof(struct gpart));
- if (gparts[i].type == swift_type_gas) {
- parts[-gparts[i].id_or_neg_offset].gpart = &gparts[i];
- } else if (gparts[i].type == swift_type_stars) {
- sparts[-gparts[i].id_or_neg_offset].gpart = &gparts[i];
- }
- }
- }
+ if (do_hydro_sub_sort)
+ error("cell %d has a hydro sub_sort flag set. Node=%d depth=%d maxdepth=%d",
+ c->cellID, c->nodeID, c->depth, c->maxdepth);
-#ifdef SWIFT_DEBUG_CHECKS
- for (int i = 0; i < c->grav.count_total; ++i) {
- if (gparts[i].time_bin == time_bin_not_created && i < c->grav.count) {
- error("Extra particle before the end of the regular array");
- }
- if (gparts[i].time_bin != time_bin_not_created && i >= c->grav.count) {
- error("Regular particle after the end of the regular array");
+ if (do_stars_sub_sort)
+ error("cell %d has a stars sub_sort flag set. Node=%d depth=%d maxdepth=%d",
+ c->cellID, c->nodeID, c->depth, c->maxdepth);
+
+ if (c->split) {
+ for (int k = 0; k < 8; ++k) {
+ if (c->progeny[k] != NULL) cell_check_sort_flags(c->progeny[k]);
}
}
#endif
diff --git a/src/cell.h b/src/cell.h
index 3a3f2aac43e1d0b0a9014f8515a8b80af0e468c1..d80d11b8229032795a64b2c862d3dc0bc310de70 100644
--- a/src/cell.h
+++ b/src/cell.h
@@ -33,14 +33,17 @@
/* Local includes. */
#include "align.h"
+#include "cell_black_holes.h"
+#include "cell_grav.h"
+#include "cell_hydro.h"
+#include "cell_sinks.h"
+#include "cell_stars.h"
#include "kernel_hydro.h"
-#include "lock.h"
#include "multipole_struct.h"
#include "part.h"
#include "periodic.h"
#include "sort_part.h"
#include "space.h"
-#include "star_formation_logger_struct.h"
#include "task.h"
#include "timeline.h"
@@ -364,492 +367,19 @@ struct cell {
volatile uint32_t flags;
/*! Hydro variables */
- struct {
-
- /*! Pointer to the #part data. */
- struct part *parts;
-
- /*! Pointer to the #xpart data. */
- struct xpart *xparts;
-
- /*! Pointer for the sorted indices. */
- struct sort_entry *sort;
-
- /*! Super cell, i.e. the highest-level parent cell that has a hydro
- * pair/self tasks */
- struct cell *super;
-
- /*! The task computing this cell's sorts. */
- struct task *sorts;
-
- /*! The drift task for parts */
- struct task *drift;
-
- /*! Linked list of the tasks computing this cell's hydro density. */
- struct link *density;
-
- /* Linked list of the tasks computing this cell's hydro gradients. */
- struct link *gradient;
-
- /*! Linked list of the tasks computing this cell's hydro forces. */
- struct link *force;
-
- /*! Linked list of the tasks computing this cell's limiter. */
- struct link *limiter;
-
- /*! Dependency implicit task for the ghost (in->ghost->out)*/
- struct task *ghost_in;
-
- /*! Dependency implicit task for the ghost (in->ghost->out)*/
- struct task *ghost_out;
-
- /*! The ghost task itself */
- struct task *ghost;
-
- /*! The extra ghost task for complex hydro schemes */
- struct task *extra_ghost;
-
- /*! The task to end the force calculation */
- struct task *end_force;
-
- /*! Dependency implicit task for cooling (in->cooling->out) */
- struct task *cooling_in;
-
- /*! Dependency implicit task for cooling (in->cooling->out) */
- struct task *cooling_out;
-
- /*! Task for cooling */
- struct task *cooling;
-
- /*! Task for star formation */
- struct task *star_formation;
-
- /*! Task for sink formation */
- struct task *sink_formation;
-
- /*! Task for sorting the stars again after a SF event */
- struct task *stars_resort;
-
- /*! Radiative transfer ghost in task */
- struct task *rt_in;
-
- /*! Radiative transfer ghost out task */
- struct task *rt_out;
-
- /*! Radiative transfer ghost1 task (finishes up injection) */
- struct task *rt_ghost1;
-
- /*! Task for self/pair injection step of radiative transfer */
- struct link *rt_inject;
-
- /*! Last (integer) time the cell's part were drifted forward in time. */
- integertime_t ti_old_part;
-
- /*! Minimum end of (integer) time step in this cell for hydro tasks. */
- integertime_t ti_end_min;
-
- /*! Maximum end of (integer) time step in this cell for hydro tasks. */
- integertime_t ti_end_max;
-
- /*! Maximum beginning of (integer) time step in this cell for hydro tasks.
- */
- integertime_t ti_beg_max;
-
- /*! Spin lock for various uses (#part case). */
- swift_lock_type lock;
-
- /*! Max smoothing length in this cell. */
- float h_max;
-
- /*! Maximum part movement in this cell since last construction. */
- float dx_max_part;
-
- /*! Maximum particle movement in this cell since the last sort. */
- float dx_max_sort;
-
- /*! Values of h_max before the drifts, used for sub-cell tasks. */
- float h_max_old;
-
- /*! Values of dx_max before the drifts, used for sub-cell tasks. */
- float dx_max_part_old;
-
- /*! Values of dx_max_sort before the drifts, used for sub-cell tasks. */
- float dx_max_sort_old;
-
- /*! Nr of #part in this cell. */
- int count;
-
- /*! Nr of #part this cell can hold after addition of new #part. */
- int count_total;
-
- /*! Number of #part updated in this cell. */
- int updated;
-
- /*! Is the #part data of this cell being used in a sub-cell? */
- int hold;
-
- /*! Bit mask of sort directions that will be needed in the next timestep. */
- uint16_t requires_sorts;
-
- /*! Bit mask of sorts that need to be computed for this cell. */
- uint16_t do_sort;
-
- /*! Bit-mask indicating the sorted directions */
- uint16_t sorted;
-
- /*! Bit-mask indicating the sorted directions */
- uint16_t sort_allocated;
-
-#ifdef SWIFT_DEBUG_CHECKS
-
- /*! Last (integer) time the cell's sort arrays were updated. */
- integertime_t ti_sort;
-
-#endif
-
- } hydro;
+ struct cell_hydro hydro;
/*! Grav variables */
- struct {
-
- /*! Pointer to the #gpart data. */
- struct gpart *parts;
-
- /*! Pointer to the #spart data at rebuild time. */
- struct gpart *parts_rebuild;
-
- /*! This cell's multipole. */
- struct gravity_tensors *multipole;
-
- /*! Super cell, i.e. the highest-level parent cell that has a grav pair/self
- * tasks */
- struct cell *super;
-
- /*! The drift task for gparts */
- struct task *drift;
-
- /*! Implicit task (going up- and down the tree) for the #gpart drifts */
- struct task *drift_out;
-
- /*! Linked list of the tasks computing this cell's gravity forces. */
- struct link *grav;
-
- /*! Linked list of the tasks computing this cell's gravity M-M forces. */
- struct link *mm;
-
- /*! The multipole initialistation task */
- struct task *init;
-
- /*! Implicit task for the gravity initialisation */
- struct task *init_out;
-
- /*! Task computing long range non-periodic gravity interactions */
- struct task *long_range;
-
- /*! Implicit task for the down propagation */
- struct task *down_in;
-
- /*! Task propagating the multipole to the particles */
- struct task *down;
-
- /*! The task to end the force calculation */
- struct task *end_force;
-
- /*! Minimum end of (integer) time step in this cell for gravity tasks. */
- integertime_t ti_end_min;
-
- /*! Maximum end of (integer) time step in this cell for gravity tasks. */
- integertime_t ti_end_max;
-
- /*! Maximum beginning of (integer) time step in this cell for gravity tasks.
- */
- integertime_t ti_beg_max;
-
- /*! Last (integer) time the cell's gpart were drifted forward in time. */
- integertime_t ti_old_part;
-
- /*! Last (integer) time the cell's multipole was drifted forward in time. */
- integertime_t ti_old_multipole;
-
- /*! Spin lock for various uses (#gpart case). */
- swift_lock_type plock;
-
- /*! Spin lock for various uses (#multipole case). */
- swift_lock_type mlock;
-
- /*! Spin lock for star formation use. */
- swift_lock_type star_formation_lock;
-
- /*! Nr of #gpart in this cell. */
- int count;
-
- /*! Nr of #gpart this cell can hold after addition of new #gpart. */
- int count_total;
-
- /*! Number of #gpart updated in this cell. */
- int updated;
-
- /*! Is the #gpart data of this cell being used in a sub-cell? */
- int phold;
-
- /*! Is the #multipole data of this cell being used in a sub-cell? */
- int mhold;
-
- /*! Number of M-M tasks that are associated with this cell. */
- short int nr_mm_tasks;
-
- } grav;
+ struct cell_grav grav;
/*! Stars variables */
- struct {
-
- /*! Pointer to the #spart data. */
- struct spart *parts;
-
- /*! Pointer to the #spart data at rebuild time. */
- struct spart *parts_rebuild;
-
- /*! The star ghost task itself */
- struct task *ghost;
-
- /*! Linked list of the tasks computing this cell's star density. */
- struct link *density;
-
- /*! Linked list of the tasks computing this cell's star feedback. */
- struct link *feedback;
-
- /*! The task computing this cell's sorts before the density. */
- struct task *sorts;
-
- /*! The drift task for sparts */
- struct task *drift;
-
- /*! Implicit tasks marking the entry of the stellar physics block of tasks
- */
- struct task *stars_in;
-
- /*! Implicit tasks marking the exit of the stellar physics block of tasks */
- struct task *stars_out;
-
- /*! Last (integer) time the cell's spart were drifted forward in time. */
- integertime_t ti_old_part;
-
- /*! Spin lock for various uses (#spart case). */
- swift_lock_type lock;
-
- /*! Spin lock for star formation use. */
- swift_lock_type star_formation_lock;
-
- /*! Nr of #spart in this cell. */
- int count;
-
- /*! Nr of #spart this cell can hold after addition of new #spart. */
- int count_total;
-
- /*! Max smoothing length in this cell. */
- float h_max;
-
- /*! Values of h_max before the drifts, used for sub-cell tasks. */
- float h_max_old;
-
- /*! Maximum part movement in this cell since last construction. */
- float dx_max_part;
-
- /*! Values of dx_max before the drifts, used for sub-cell tasks. */
- float dx_max_part_old;
-
- /*! Maximum particle movement in this cell since the last sort. */
- float dx_max_sort;
-
- /*! Values of dx_max_sort before the drifts, used for sub-cell tasks. */
- float dx_max_sort_old;
-
- /*! Pointer for the sorted indices. */
- struct sort_entry *sort;
-
- /*! Bit mask of sort directions that will be needed in the next timestep. */
- uint16_t requires_sorts;
-
- /*! Bit-mask indicating the sorted directions */
- uint16_t sorted;
-
- /*! Bit-mask indicating the sorted directions */
- uint16_t sort_allocated;
-
- /*! Bit mask of sorts that need to be computed for this cell. */
- uint16_t do_sort;
-
- /*! Maximum end of (integer) time step in this cell for star tasks. */
- integertime_t ti_end_min;
-
- /*! Maximum end of (integer) time step in this cell for star tasks. */
- integertime_t ti_end_max;
-
- /*! Maximum beginning of (integer) time step in this cell for star tasks.
- */
- integertime_t ti_beg_max;
-
- /*! Number of #spart updated in this cell. */
- int updated;
-
- /*! Is the #spart data of this cell being used in a sub-cell? */
- int hold;
-
- /*! Star formation history struct */
- struct star_formation_history sfh;
-
-#ifdef SWIFT_DEBUG_CHECKS
- /*! Last (integer) time the cell's sort arrays were updated. */
- integertime_t ti_sort;
-#endif
-
- } stars;
+ struct cell_stars stars;
/*! Black hole variables */
- struct {
-
- /*! Pointer to the #bpart data. */
- struct bpart *parts;
-
- /*! The drift task for bparts */
- struct task *drift;
-
- /*! Implicit tasks marking the entry of the BH physics block of tasks
- */
- struct task *black_holes_in;
-
- /*! Implicit tasks marking the exit of the BH physics block of tasks */
- struct task *black_holes_out;
-
- /*! The star ghost task itself */
- struct task *density_ghost;
-
- /*! The star ghost task itself */
- struct task *swallow_ghost[3];
-
- /*! Linked list of the tasks computing this cell's BH density. */
- struct link *density;
-
- /*! Linked list of the tasks computing this cell's BH swallowing and
- * merging. */
- struct link *swallow;
-
- /*! Linked list of the tasks processing the particles to swallow */
- struct link *do_gas_swallow;
-
- /*! Linked list of the tasks processing the particles to swallow */
- struct link *do_bh_swallow;
-
- /*! Linked list of the tasks computing this cell's BH feedback. */
- struct link *feedback;
-
- /*! Last (integer) time the cell's bpart were drifted forward in time. */
- integertime_t ti_old_part;
-
- /*! Spin lock for various uses (#bpart case). */
- swift_lock_type lock;
-
- /*! Nr of #bpart in this cell. */
- int count;
-
- /*! Nr of #bpart this cell can hold after addition of new #bpart. */
- int count_total;
-
- /*! Max smoothing length in this cell. */
- float h_max;
-
- /*! Values of h_max before the drifts, used for sub-cell tasks. */
- float h_max_old;
-
- /*! Maximum part movement in this cell since last construction. */
- float dx_max_part;
-
- /*! Values of dx_max before the drifts, used for sub-cell tasks. */
- float dx_max_part_old;
-
- /*! Maximum end of (integer) time step in this cell for black tasks. */
- integertime_t ti_end_min;
-
- /*! Maximum end of (integer) time step in this cell for black hole tasks. */
- integertime_t ti_end_max;
-
- /*! Maximum beginning of (integer) time step in this cell for black hole
- * tasks.
- */
- integertime_t ti_beg_max;
-
- /*! Number of #bpart updated in this cell. */
- int updated;
-
- /*! Is the #bpart data of this cell being used in a sub-cell? */
- int hold;
-
- } black_holes;
+ struct cell_black_holes black_holes;
/*! Sink particles variables */
- struct {
-
- /*! Pointer to the #sink data. */
- struct sink *parts;
-
- /*! Linked list of the tasks computing this cell's sink formation checks. */
- struct link *compute_formation;
-
- /*! Nr of #sink in this cell. */
- int count;
-
- /*! Nr of #sink this cell can hold after addition of new one. */
- int count_total;
-
- /*! Max cut off radius in this cell. */
- float r_cut_max;
-
- /*! Values of r_cut_max before the drifts, used for sub-cell tasks. */
- float r_cut_max_old;
-
- /*! Number of #sink updated in this cell. */
- int updated;
-
- /*! Is the #sink data of this cell being used in a sub-cell? */
- int hold;
-
- /*! Spin lock for various uses (#sink case). */
- swift_lock_type lock;
-
- /*! Spin lock for sink formation use. */
- swift_lock_type sink_formation_lock;
-
- /*! Maximum part movement in this cell since last construction. */
- float dx_max_part;
-
- /*! Values of dx_max before the drifts, used for sub-cell tasks. */
- float dx_max_part_old;
-
- /*! Last (integer) time the cell's sink were drifted forward in time. */
- integertime_t ti_old_part;
-
- /*! Minimum end of (integer) time step in this cell for sink tasks. */
- integertime_t ti_end_min;
-
- /*! Maximum end of (integer) time step in this cell for sink tasks. */
- integertime_t ti_end_max;
-
- /*! Maximum beginning of (integer) time step in this cell for sink
- * tasks.
- */
- integertime_t ti_beg_max;
-
- /*! The drift task for sinks */
- struct task *drift;
-
- /*! Implicit tasks marking the entry of the sink block of tasks
- */
- struct task *sink_in;
-
- /*! Implicit tasks marking the exit of the sink block of tasks */
- struct task *sink_out;
-
- } sinks;
+ struct cell_sinks sinks;
#ifdef WITH_MPI
/*! MPI variables */
@@ -986,7 +516,7 @@ int cell_pack_multipoles(struct cell *c, struct gravity_tensors *m);
int cell_unpack_multipoles(struct cell *c, struct gravity_tensors *m);
int cell_pack_sf_counts(struct cell *c, struct pcell_sf *pcell);
int cell_unpack_sf_counts(struct cell *c, struct pcell_sf *pcell);
-int cell_getsize(struct cell *c);
+int cell_get_tree_size(struct cell *c);
int cell_link_parts(struct cell *c, struct part *parts);
int cell_link_gparts(struct cell *c, struct gpart *gparts);
int cell_link_sparts(struct cell *c, struct spart *sparts);
diff --git a/src/cell_black_holes.h b/src/cell_black_holes.h
new file mode 100644
index 0000000000000000000000000000000000000000..c745942814fd2ddef87a32574010491683e14f07
--- /dev/null
+++ b/src/cell_black_holes.h
@@ -0,0 +1,113 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+#ifndef SWIFT_CELL_BLACK_HOLES_H
+#define SWIFT_CELL_BLACK_HOLES_H
+
+/* Config parameters. */
+#include "../config.h"
+
+/* Local includes. */
+#include "lock.h"
+#include "timeline.h"
+
+/**
+ * @brief BHs-related cell variables.
+ */
+struct cell_black_holes {
+
+ /*! Pointer to the #bpart data. */
+ struct bpart *parts;
+
+ /*! The drift task for bparts */
+ struct task *drift;
+
+ /*! Implicit tasks marking the entry of the BH physics block of tasks
+ */
+ struct task *black_holes_in;
+
+ /*! Implicit tasks marking the exit of the BH physics block of tasks */
+ struct task *black_holes_out;
+
+ /*! The star ghost task itself */
+ struct task *density_ghost;
+
+ /*! The star ghost task itself */
+ struct task *swallow_ghost[3];
+
+ /*! Linked list of the tasks computing this cell's BH density. */
+ struct link *density;
+
+ /*! Linked list of the tasks computing this cell's BH swallowing and
+ * merging. */
+ struct link *swallow;
+
+ /*! Linked list of the tasks processing the particles to swallow */
+ struct link *do_gas_swallow;
+
+ /*! Linked list of the tasks processing the particles to swallow */
+ struct link *do_bh_swallow;
+
+ /*! Linked list of the tasks computing this cell's BH feedback. */
+ struct link *feedback;
+
+ /*! Last (integer) time the cell's bpart were drifted forward in time. */
+ integertime_t ti_old_part;
+
+ /*! Spin lock for various uses (#bpart case). */
+ swift_lock_type lock;
+
+ /*! Nr of #bpart in this cell. */
+ int count;
+
+ /*! Nr of #bpart this cell can hold after addition of new #bpart. */
+ int count_total;
+
+ /*! Max smoothing length in this cell. */
+ float h_max;
+
+ /*! Values of h_max before the drifts, used for sub-cell tasks. */
+ float h_max_old;
+
+ /*! Maximum part movement in this cell since last construction. */
+ float dx_max_part;
+
+ /*! Values of dx_max before the drifts, used for sub-cell tasks. */
+ float dx_max_part_old;
+
+ /*! Maximum end of (integer) time step in this cell for black tasks. */
+ integertime_t ti_end_min;
+
+ /*! Maximum end of (integer) time step in this cell for black hole tasks. */
+ integertime_t ti_end_max;
+
+ /*! Maximum beginning of (integer) time step in this cell for black hole
+ * tasks.
+ */
+ integertime_t ti_beg_max;
+
+ /*! Number of #bpart updated in this cell. */
+ int updated;
+
+ /*! Is the #bpart data of this cell being used in a sub-cell? */
+ int hold;
+};
+
+#endif /* SWIFT_CELL_BLACK_HOLES_H */
diff --git a/src/cell_convert_part.c b/src/cell_convert_part.c
new file mode 100644
index 0000000000000000000000000000000000000000..c70f40aee8a525006365092f0b76de8d441f0853
--- /dev/null
+++ b/src/cell_convert_part.c
@@ -0,0 +1,1018 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* This object's header. */
+#include "cell.h"
+
+/* Local headers. */
+#include "engine.h"
+#include "hydro.h"
+#include "sink_properties.h"
+
+/**
+ * @brief Recursively update the pointer and counter for #spart after the
+ * addition of a new particle.
+ *
+ * @param c The cell we are working on.
+ * @param progeny_list The list of the progeny index at each level for the
+ * leaf-cell where the particle was added.
+ * @param main_branch Are we in a cell directly above the leaf where the new
+ * particle was added?
+ */
+void cell_recursively_shift_sparts(struct cell *c,
+ const int progeny_list[space_cell_maxdepth],
+ const int main_branch) {
+ if (c->split) {
+ /* No need to recurse in progenies located before the insestion point */
+ const int first_progeny = main_branch ? progeny_list[(int)c->depth] : 0;
+
+ for (int k = first_progeny; k < 8; ++k) {
+ if (c->progeny[k] != NULL)
+ cell_recursively_shift_sparts(c->progeny[k], progeny_list,
+ main_branch && (k == first_progeny));
+ }
+ }
+
+ /* When directly above the leaf with the new particle: increase the particle
+ * count */
+ /* When after the leaf with the new particle: shift by one position */
+ if (main_branch) {
+ c->stars.count++;
+
+ /* Indicate that the cell is not sorted and cancel the pointer sorting
+ * arrays. */
+ c->stars.sorted = 0;
+ cell_free_stars_sorts(c);
+
+ } else {
+ c->stars.parts++;
+ }
+}
+
+/**
+ * @brief Recursively update the pointer and counter for #sink after the
+ * addition of a new particle.
+ *
+ * @param c The cell we are working on.
+ * @param progeny_list The list of the progeny index at each level for the
+ * leaf-cell where the particle was added.
+ * @param main_branch Are we in a cell directly above the leaf where the new
+ * particle was added?
+ */
+void cell_recursively_shift_sinks(struct cell *c,
+ const int progeny_list[space_cell_maxdepth],
+ const int main_branch) {
+ if (c->split) {
+ /* No need to recurse in progenies located before the insestion point */
+ const int first_progeny = main_branch ? progeny_list[(int)c->depth] : 0;
+
+ for (int k = first_progeny; k < 8; ++k) {
+ if (c->progeny[k] != NULL)
+ cell_recursively_shift_sinks(c->progeny[k], progeny_list,
+ main_branch && (k == first_progeny));
+ }
+ }
+
+ /* When directly above the leaf with the new particle: increase the particle
+ * count */
+ /* When after the leaf with the new particle: shift by one position */
+ if (main_branch) {
+ c->sinks.count++;
+ } else {
+ c->sinks.parts++;
+ }
+}
+
+/**
+ * @brief Recursively update the pointer and counter for #gpart after the
+ * addition of a new particle.
+ *
+ * @param c The cell we are working on.
+ * @param progeny_list The list of the progeny index at each level for the
+ * leaf-cell where the particle was added.
+ * @param main_branch Are we in a cell directly above the leaf where the new
+ * particle was added?
+ */
+void cell_recursively_shift_gparts(struct cell *c,
+ const int progeny_list[space_cell_maxdepth],
+ const int main_branch) {
+ if (c->split) {
+ /* No need to recurse in progenies located before the insestion point */
+ const int first_progeny = main_branch ? progeny_list[(int)c->depth] : 0;
+
+ for (int k = first_progeny; k < 8; ++k) {
+ if (c->progeny[k] != NULL)
+ cell_recursively_shift_gparts(c->progeny[k], progeny_list,
+ main_branch && (k == first_progeny));
+ }
+ }
+
+ /* When directly above the leaf with the new particle: increase the particle
+ * count */
+ /* When after the leaf with the new particle: shift by one position */
+ if (main_branch) {
+ c->grav.count++;
+ } else {
+ c->grav.parts++;
+ }
+}
+
+/**
+ * @brief "Add" a #spart in a given #cell.
+ *
+ * This function will add a #spart at the start of the current cell's array by
+ * shifting all the #spart in the top-level cell by one position. All the
+ * pointers and cell counts are updated accordingly.
+ *
+ * @param e The #engine.
+ * @param c The leaf-cell in which to add the #spart.
+ *
+ * @return A pointer to the newly added #spart. The spart has a been zeroed
+ * and given a position within the cell as well as set to the minimal active
+ * time bin.
+ */
+struct spart *cell_add_spart(struct engine *e, struct cell *const c) {
+ /* Perform some basic consitency checks */
+ if (c->nodeID != engine_rank) error("Adding spart on a foreign node");
+ if (c->grav.ti_old_part != e->ti_current) error("Undrifted cell!");
+ if (c->split) error("Addition of spart performed above the leaf level");
+
+ /* Progeny number at each level */
+ int progeny[space_cell_maxdepth];
+#ifdef SWIFT_DEBUG_CHECKS
+ for (int i = 0; i < space_cell_maxdepth; ++i) progeny[i] = -1;
+#endif
+
+ /* Get the top-level this leaf cell is in and compute the progeny indices at
+ each level */
+ struct cell *top = c;
+ while (top->parent != NULL) {
+ /* What is the progeny index of the cell? */
+ for (int k = 0; k < 8; ++k) {
+ if (top->parent->progeny[k] == top) {
+ progeny[(int)top->parent->depth] = k;
+ }
+ }
+
+ /* Check that the cell was indeed drifted to this point to avoid future
+ * issues */
+#ifdef SWIFT_DEBUG_CHECKS
+ if (top->hydro.super != NULL && top->stars.count > 0 &&
+ top->stars.ti_old_part != e->ti_current) {
+ error("Cell had not been correctly drifted before star formation");
+ }
+#endif
+
+ /* Climb up */
+ top = top->parent;
+ }
+
+ /* Lock the top-level cell as we are going to operate on it */
+ lock_lock(&top->stars.star_formation_lock);
+
+ /* Are there any extra particles left? */
+ if (top->stars.count == top->stars.count_total) {
+
+ message("We ran out of free star particles!");
+
+ /* Release the local lock before exiting. */
+ if (lock_unlock(&top->stars.star_formation_lock) != 0)
+ error("Failed to unlock the top-level cell.");
+
+ atomic_inc(&e->forcerebuild);
+ return NULL;
+ }
+
+ /* Number of particles to shift in order to get a free space. */
+ const size_t n_copy = &top->stars.parts[top->stars.count] - c->stars.parts;
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->stars.parts + n_copy > top->stars.parts + top->stars.count)
+ error("Copying beyond the allowed range");
+#endif
+
+ if (n_copy > 0) {
+ // MATTHIEU: This can be improved. We don't need to copy everything, just
+ // need to swap a few particles.
+ memmove(&c->stars.parts[1], &c->stars.parts[0],
+ n_copy * sizeof(struct spart));
+
+ /* Update the spart->gpart links (shift by 1) */
+ for (size_t i = 0; i < n_copy; ++i) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->stars.parts[i + 1].gpart == NULL) {
+ error("Incorrectly linked spart!");
+ }
+#endif
+ c->stars.parts[i + 1].gpart->id_or_neg_offset--;
+ }
+ }
+
+ /* Recursively shift all the stars to get a free spot at the start of the
+ * current cell*/
+ cell_recursively_shift_sparts(top, progeny, /* main_branch=*/1);
+
+ /* Make sure the gravity will be recomputed for this particle in the next
+ * step
+ */
+ struct cell *top2 = c;
+ while (top2->parent != NULL) {
+ top2->stars.ti_old_part = e->ti_current;
+ top2 = top2->parent;
+ }
+ top2->stars.ti_old_part = e->ti_current;
+
+ /* Release the lock */
+ if (lock_unlock(&top->stars.star_formation_lock) != 0)
+ error("Failed to unlock the top-level cell.");
+
+ /* We now have an empty spart as the first particle in that cell */
+ struct spart *sp = &c->stars.parts[0];
+ bzero(sp, sizeof(struct spart));
+
+ /* Give it a decent position */
+ sp->x[0] = c->loc[0] + 0.5 * c->width[0];
+ sp->x[1] = c->loc[1] + 0.5 * c->width[1];
+ sp->x[2] = c->loc[2] + 0.5 * c->width[2];
+
+ /* Set it to the current time-bin */
+ sp->time_bin = e->min_active_bin;
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Specify it was drifted to this point */
+ sp->ti_drift = e->ti_current;
+#endif
+
+ /* Register that we used one of the free slots. */
+ const size_t one = 1;
+ atomic_sub(&e->s->nr_extra_sparts, one);
+
+ return sp;
+}
+
+/**
+ * @brief "Add" a #sink in a given #cell.
+ *
+ * This function will add a #sink at the start of the current cell's array by
+ * shifting all the #sink in the top-level cell by one position. All the
+ * pointers and cell counts are updated accordingly.
+ *
+ * @param e The #engine.
+ * @param c The leaf-cell in which to add the #sink.
+ *
+ * @return A pointer to the newly added #sink. The sink has a been zeroed
+ * and given a position within the cell as well as set to the minimal active
+ * time bin.
+ */
+struct sink *cell_add_sink(struct engine *e, struct cell *const c) {
+ /* Perform some basic consitency checks */
+ if (c->nodeID != engine_rank) error("Adding sink on a foreign node");
+ if (c->grav.ti_old_part != e->ti_current) error("Undrifted cell!");
+ if (c->split) error("Addition of sink performed above the leaf level");
+
+ /* Progeny number at each level */
+ int progeny[space_cell_maxdepth];
+#ifdef SWIFT_DEBUG_CHECKS
+ for (int i = 0; i < space_cell_maxdepth; ++i) progeny[i] = -1;
+#endif
+
+ /* Get the top-level this leaf cell is in and compute the progeny indices at
+ each level */
+ struct cell *top = c;
+ while (top->parent != NULL) {
+ /* What is the progeny index of the cell? */
+ for (int k = 0; k < 8; ++k) {
+ if (top->parent->progeny[k] == top) {
+ progeny[(int)top->parent->depth] = k;
+ }
+ }
+
+ /* Check that the cell was indeed drifted to this point to avoid future
+ * issues */
+#ifdef SWIFT_DEBUG_CHECKS
+ if (top->hydro.super != NULL && top->sinks.count > 0 &&
+ top->sinks.ti_old_part != e->ti_current) {
+ error("Cell had not been correctly drifted before sink formation");
+ }
+#endif
+
+ /* Climb up */
+ top = top->parent;
+ }
+
+ /* Lock the top-level cell as we are going to operate on it */
+ lock_lock(&top->sinks.sink_formation_lock);
+
+ /* Are there any extra particles left? */
+ if (top->sinks.count == top->sinks.count_total) {
+
+ error("We ran out of free sink particles!");
+
+ /* Release the local lock before exiting. */
+ if (lock_unlock(&top->sinks.sink_formation_lock) != 0)
+ error("Failed to unlock the top-level cell.");
+
+ atomic_inc(&e->forcerebuild);
+ return NULL;
+ }
+
+ /* Number of particles to shift in order to get a free space. */
+ const size_t n_copy = &top->sinks.parts[top->sinks.count] - c->sinks.parts;
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->sinks.parts + n_copy > top->sinks.parts + top->sinks.count)
+ error("Copying beyond the allowed range");
+#endif
+
+ if (n_copy > 0) {
+ // MATTHIEU: This can be improved. We don't need to copy everything, just
+ // need to swap a few particles.
+ memmove(&c->sinks.parts[1], &c->sinks.parts[0],
+ n_copy * sizeof(struct sink));
+
+ /* Update the sink->gpart links (shift by 1) */
+ for (size_t i = 0; i < n_copy; ++i) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->sinks.parts[i + 1].gpart == NULL) {
+ error("Incorrectly linked sink!");
+ }
+#endif
+ c->sinks.parts[i + 1].gpart->id_or_neg_offset--;
+ }
+ }
+
+ /* Recursively shift all the sinks to get a free spot at the start of the
+ * current cell*/
+ cell_recursively_shift_sinks(top, progeny, /* main_branch=*/1);
+
+ /* Make sure the gravity will be recomputed for this particle in the next
+ * step
+ */
+ struct cell *top2 = c;
+ while (top2->parent != NULL) {
+ top2->sinks.ti_old_part = e->ti_current;
+ top2 = top2->parent;
+ }
+ top2->sinks.ti_old_part = e->ti_current;
+
+ /* Release the lock */
+ if (lock_unlock(&top->sinks.sink_formation_lock) != 0)
+ error("Failed to unlock the top-level cell.");
+
+ /* We now have an empty spart as the first particle in that cell */
+ struct sink *sp = &c->sinks.parts[0];
+ bzero(sp, sizeof(struct sink));
+
+ /* Give it a decent position */
+ sp->x[0] = c->loc[0] + 0.5 * c->width[0];
+ sp->x[1] = c->loc[1] + 0.5 * c->width[1];
+ sp->x[2] = c->loc[2] + 0.5 * c->width[2];
+
+ /* Set it to the current time-bin */
+ sp->time_bin = e->min_active_bin;
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Specify it was drifted to this point */
+ sp->ti_drift = e->ti_current;
+#endif
+
+ /* Register that we used one of the free slots. */
+ const size_t one = 1;
+ atomic_sub(&e->s->nr_extra_sinks, one);
+
+ return sp;
+}
+
+/**
+ * @brief "Add" a #gpart in a given #cell.
+ *
+ * This function will add a #gpart at the start of the current cell's array by
+ * shifting all the #gpart in the top-level cell by one position. All the
+ * pointers and cell counts are updated accordingly.
+ *
+ * @param e The #engine.
+ * @param c The leaf-cell in which to add the #gpart.
+ *
+ * @return A pointer to the newly added #gpart. The gpart has a been zeroed
+ * and given a position within the cell as well as set to the minimal active
+ * time bin.
+ */
+struct gpart *cell_add_gpart(struct engine *e, struct cell *c) {
+ /* Perform some basic consitency checks */
+ if (c->nodeID != engine_rank) error("Adding gpart on a foreign node");
+ if (c->grav.ti_old_part != e->ti_current) error("Undrifted cell!");
+ if (c->split) error("Addition of gpart performed above the leaf level");
+
+ struct space *s = e->s;
+
+ /* Progeny number at each level */
+ int progeny[space_cell_maxdepth];
+#ifdef SWIFT_DEBUG_CHECKS
+ for (int i = 0; i < space_cell_maxdepth; ++i) progeny[i] = -1;
+#endif
+
+ /* Get the top-level this leaf cell is in and compute the progeny indices at
+ each level */
+ struct cell *top = c;
+ while (top->parent != NULL) {
+ /* What is the progeny index of the cell? */
+ for (int k = 0; k < 8; ++k) {
+ if (top->parent->progeny[k] == top) {
+ progeny[(int)top->parent->depth] = k;
+ }
+ }
+
+ /* Check that the cell was indeed drifted to this point to avoid future
+ * issues */
+#ifdef SWIFT_DEBUG_CHECKS
+ if (top->grav.super != NULL && top->grav.count > 0 &&
+ top->grav.ti_old_part != e->ti_current) {
+ error("Cell had not been correctly drifted before adding a gpart");
+ }
+#endif
+
+ /* Climb up */
+ top = top->parent;
+ }
+
+ /* Lock the top-level cell as we are going to operate on it */
+ lock_lock(&top->grav.star_formation_lock);
+
+ /* Are there any extra particles left? */
+ if (top->grav.count == top->grav.count_total) {
+
+ message("We ran out of free gravity particles!");
+
+ /* Release the local lock before exiting. */
+ if (lock_unlock(&top->grav.star_formation_lock) != 0)
+ error("Failed to unlock the top-level cell.");
+
+ atomic_inc(&e->forcerebuild);
+ return NULL;
+ }
+
+ /* Number of particles to shift in order to get a free space. */
+ const size_t n_copy = &top->grav.parts[top->grav.count] - c->grav.parts;
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->grav.parts + n_copy > top->grav.parts + top->grav.count)
+ error("Copying beyond the allowed range");
+#endif
+
+ if (n_copy > 0) {
+ // MATTHIEU: This can be improved. We don't need to copy everything, just
+ // need to swap a few particles.
+ memmove(&c->grav.parts[1], &c->grav.parts[0],
+ n_copy * sizeof(struct gpart));
+
+ /* Update the gpart->spart links (shift by 1) */
+ struct gpart *gparts = c->grav.parts;
+ for (size_t i = 0; i < n_copy; ++i) {
+ if (gparts[i + 1].type == swift_type_gas) {
+ s->parts[-gparts[i + 1].id_or_neg_offset].gpart++;
+ } else if (gparts[i + 1].type == swift_type_stars) {
+ s->sparts[-gparts[i + 1].id_or_neg_offset].gpart++;
+ } else if (gparts[i + 1].type == swift_type_black_hole) {
+ s->bparts[-gparts[i + 1].id_or_neg_offset].gpart++;
+ }
+ }
+ }
+
+ /* Recursively shift all the gpart to get a free spot at the start of the
+ * current cell*/
+ cell_recursively_shift_gparts(top, progeny, /* main_branch=*/1);
+
+ /* Make sure the gravity will be recomputed for this particle in the next
+ * step
+ */
+ struct cell *top2 = c;
+ while (top2->parent != NULL) {
+ top2->grav.ti_old_part = e->ti_current;
+ top2 = top2->parent;
+ }
+ top2->grav.ti_old_part = e->ti_current;
+
+ /* Release the lock */
+ if (lock_unlock(&top->grav.star_formation_lock) != 0)
+ error("Failed to unlock the top-level cell.");
+
+ /* We now have an empty gpart as the first particle in that cell */
+ struct gpart *gp = &c->grav.parts[0];
+ bzero(gp, sizeof(struct gpart));
+
+ /* Give it a decent position */
+ gp->x[0] = c->loc[0] + 0.5 * c->width[0];
+ gp->x[1] = c->loc[1] + 0.5 * c->width[1];
+ gp->x[2] = c->loc[2] + 0.5 * c->width[2];
+
+ /* Set it to the current time-bin */
+ gp->time_bin = e->min_active_bin;
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Specify it was drifted to this point */
+ gp->ti_drift = e->ti_current;
+#endif
+
+ /* Register that we used one of the free slots. */
+ const size_t one = 1;
+ atomic_sub(&e->s->nr_extra_gparts, one);
+
+ return gp;
+}
+
+/**
+ * @brief "Remove" a gas particle from the calculation.
+ *
+ * The particle is inhibited and will officially be removed at the next
+ * rebuild.
+ *
+ * @param e The #engine running on this node.
+ * @param c The #cell from which to remove the particle.
+ * @param p The #part to remove.
+ * @param xp The extended data of the particle to remove.
+ */
+void cell_remove_part(const struct engine *e, struct cell *c, struct part *p,
+ struct xpart *xp) {
+ /* Quick cross-check */
+ if (c->nodeID != e->nodeID)
+ error("Can't remove a particle in a foreign cell.");
+
+ /* Don't remove a particle twice */
+ if (p->time_bin == time_bin_inhibited) return;
+
+ /* Mark the particle as inhibited */
+ p->time_bin = time_bin_inhibited;
+
+ /* Mark the gpart as inhibited and stand-alone */
+ if (p->gpart) {
+ p->gpart->time_bin = time_bin_inhibited;
+ p->gpart->id_or_neg_offset = p->id;
+ p->gpart->type = swift_type_dark_matter;
+ }
+
+ /* Update the space-wide counters */
+ const size_t one = 1;
+ atomic_add(&e->s->nr_inhibited_parts, one);
+ if (p->gpart) {
+ atomic_add(&e->s->nr_inhibited_gparts, one);
+ }
+
+ /* Un-link the part */
+ p->gpart = NULL;
+}
+
+/**
+ * @brief "Remove" a gravity particle from the calculation.
+ *
+ * The particle is inhibited and will officially be removed at the next
+ * rebuild.
+ *
+ * @param e The #engine running on this node.
+ * @param c The #cell from which to remove the particle.
+ * @param gp The #gpart to remove.
+ */
+void cell_remove_gpart(const struct engine *e, struct cell *c,
+ struct gpart *gp) {
+
+ /* Quick cross-check */
+ if (c->nodeID != e->nodeID)
+ error("Can't remove a particle in a foreign cell.");
+
+ /* Don't remove a particle twice */
+ if (gp->time_bin == time_bin_inhibited) return;
+
+ /* Quick cross-check */
+ if (c->nodeID != e->nodeID)
+ error("Can't remove a particle in a foreign cell.");
+
+ if (gp->type == swift_type_dark_matter_background)
+ error("Can't remove a DM background particle!");
+
+ /* Mark the particle as inhibited */
+ gp->time_bin = time_bin_inhibited;
+
+ /* Update the space-wide counters */
+ const size_t one = 1;
+ atomic_add(&e->s->nr_inhibited_gparts, one);
+}
+
+/**
+ * @brief "Remove" a star particle from the calculation.
+ *
+ * The particle is inhibited and will officially be removed at the next
+ * rebuild.
+ *
+ * @param e The #engine running on this node.
+ * @param c The #cell from which to remove the particle.
+ * @param sp The #spart to remove.
+ */
+void cell_remove_spart(const struct engine *e, struct cell *c,
+ struct spart *sp) {
+ /* Quick cross-check */
+ if (c->nodeID != e->nodeID)
+ error("Can't remove a particle in a foreign cell.");
+
+ /* Don't remove a particle twice */
+ if (sp->time_bin == time_bin_inhibited) return;
+
+ /* Mark the particle as inhibited and stand-alone */
+ sp->time_bin = time_bin_inhibited;
+ if (sp->gpart) {
+ sp->gpart->time_bin = time_bin_inhibited;
+ sp->gpart->id_or_neg_offset = sp->id;
+ sp->gpart->type = swift_type_dark_matter;
+ }
+
+ /* Update the space-wide counters */
+ const size_t one = 1;
+ atomic_add(&e->s->nr_inhibited_sparts, one);
+ if (sp->gpart) {
+ atomic_add(&e->s->nr_inhibited_gparts, one);
+ }
+
+ /* Un-link the spart */
+ sp->gpart = NULL;
+}
+
+/**
+ * @brief "Remove" a black hole particle from the calculation.
+ *
+ * The particle is inhibited and will officially be removed at the next
+ * rebuild.
+ *
+ * @param e The #engine running on this node.
+ * @param c The #cell from which to remove the particle.
+ * @param bp The #bpart to remove.
+ */
+void cell_remove_bpart(const struct engine *e, struct cell *c,
+ struct bpart *bp) {
+
+ /* Quick cross-check */
+ if (c->nodeID != e->nodeID)
+ error("Can't remove a particle in a foreign cell.");
+
+ /* Don't remove a particle twice */
+ if (bp->time_bin == time_bin_inhibited) return;
+
+ /* Mark the particle as inhibited and stand-alone */
+ bp->time_bin = time_bin_inhibited;
+ if (bp->gpart) {
+ bp->gpart->time_bin = time_bin_inhibited;
+ bp->gpart->id_or_neg_offset = bp->id;
+ bp->gpart->type = swift_type_dark_matter;
+ }
+
+ /* Update the space-wide counters */
+ const size_t one = 1;
+ atomic_add(&e->s->nr_inhibited_bparts, one);
+ if (bp->gpart) {
+ atomic_add(&e->s->nr_inhibited_gparts, one);
+ }
+
+ /* Un-link the bpart */
+ bp->gpart = NULL;
+}
+
+/**
+ * @brief "Remove" a gas particle from the calculation and convert its gpart
+ * friend to a dark matter particle.
+ *
+ * Note that the #part is not destroyed. The pointer is still valid
+ * after this call and the properties of the #part are not altered
+ * apart from the time-bin and #gpart pointer.
+ * The particle is inhibited and will officially be removed at the next
+ * rebuild.
+ *
+ * @param e The #engine running on this node.
+ * @param c The #cell from which to remove the particle.
+ * @param p The #part to remove.
+ * @param xp The extended data of the particle to remove.
+ *
+ * @return Pointer to the #gpart the #part has become. It carries the
+ * ID of the #part and has a dark matter type.
+ */
+struct gpart *cell_convert_part_to_gpart(const struct engine *e, struct cell *c,
+ struct part *p, struct xpart *xp) {
+ /* Quick cross-checks */
+ if (c->nodeID != e->nodeID)
+ error("Can't remove a particle in a foreign cell.");
+
+ if (p->gpart == NULL)
+ error("Trying to convert part without gpart friend to dark matter!");
+
+ /* Get a handle */
+ struct gpart *gp = p->gpart;
+
+ /* Mark the particle as inhibited */
+ p->time_bin = time_bin_inhibited;
+
+ /* Un-link the part */
+ p->gpart = NULL;
+
+ /* Mark the gpart as dark matter */
+ gp->type = swift_type_dark_matter;
+ gp->id_or_neg_offset = p->id;
+
+#ifdef SWIFT_DEBUG_CHECKS
+ gp->ti_kick = p->ti_kick;
+#endif
+
+ /* Update the space-wide counters */
+ atomic_inc(&e->s->nr_inhibited_parts);
+
+ return gp;
+}
+
+/**
+ * @brief "Remove" a spart particle from the calculation and convert its gpart
+ * friend to a dark matter particle.
+ *
+ * Note that the #spart is not destroyed. The pointer is still valid
+ * after this call and the properties of the #spart are not altered
+ * apart from the time-bin and #gpart pointer.
+ * The particle is inhibited and will officially be removed at the next
+ * rebuild.
+ *
+ * @param e The #engine running on this node.
+ * @param c The #cell from which to remove the particle.
+ * @param sp The #spart to remove.
+ *
+ * @return Pointer to the #gpart the #spart has become. It carries the
+ * ID of the #spart and has a dark matter type.
+ */
+struct gpart *cell_convert_spart_to_gpart(const struct engine *e,
+ struct cell *c, struct spart *sp) {
+ /* Quick cross-check */
+ if (c->nodeID != e->nodeID)
+ error("Can't remove a particle in a foreign cell.");
+
+ if (sp->gpart == NULL)
+ error("Trying to convert spart without gpart friend to dark matter!");
+
+ /* Get a handle */
+ struct gpart *gp = sp->gpart;
+
+ /* Mark the particle as inhibited */
+ sp->time_bin = time_bin_inhibited;
+
+ /* Un-link the spart */
+ sp->gpart = NULL;
+
+ /* Mark the gpart as dark matter */
+ gp->type = swift_type_dark_matter;
+ gp->id_or_neg_offset = sp->id;
+
+#ifdef SWIFT_DEBUG_CHECKS
+ gp->ti_kick = sp->ti_kick;
+#endif
+
+ /* Update the space-wide counters */
+ atomic_inc(&e->s->nr_inhibited_sparts);
+
+ return gp;
+}
+
+/**
+ * @brief "Remove" a #part from a #cell and replace it with a #spart
+ * connected to the same #gpart.
+ *
+ * Note that the #part is not destroyed. The pointer is still valid
+ * after this call and the properties of the #part are not altered
+ * apart from the time-bin and #gpart pointer.
+ * The particle is inhibited and will officially be removed at the next
+ * rebuild.
+ *
+ * @param e The #engine.
+ * @param c The #cell from which to remove the #part.
+ * @param p The #part to remove (must be inside c).
+ * @param xp The extended data of the #part.
+ *
+ * @return A fresh #spart with the same ID, position, velocity and
+ * time-bin as the original #part.
+ */
+struct spart *cell_convert_part_to_spart(struct engine *e, struct cell *c,
+ struct part *p, struct xpart *xp) {
+ /* Quick cross-check */
+ if (c->nodeID != e->nodeID)
+ error("Can't remove a particle in a foreign cell.");
+
+ if (p->gpart == NULL)
+ error("Trying to convert part without gpart friend to star!");
+
+ /* Create a fresh (empty) spart */
+ struct spart *sp = cell_add_spart(e, c);
+
+ /* Did we run out of free spart slots? */
+ if (sp == NULL) return NULL;
+
+ /* Copy over the distance since rebuild */
+ sp->x_diff[0] = xp->x_diff[0];
+ sp->x_diff[1] = xp->x_diff[1];
+ sp->x_diff[2] = xp->x_diff[2];
+
+ /* Destroy the gas particle and get it's gpart friend */
+ struct gpart *gp = cell_convert_part_to_gpart(e, c, p, xp);
+
+ /* Assign the ID back */
+ sp->id = gp->id_or_neg_offset;
+ gp->type = swift_type_stars;
+
+ /* Re-link things */
+ sp->gpart = gp;
+ gp->id_or_neg_offset = -(sp - e->s->sparts);
+
+ /* Synchronize clocks */
+ gp->time_bin = sp->time_bin;
+
+ /* Synchronize masses, positions and velocities */
+ sp->mass = gp->mass;
+ sp->x[0] = gp->x[0];
+ sp->x[1] = gp->x[1];
+ sp->x[2] = gp->x[2];
+ sp->v[0] = gp->v_full[0];
+ sp->v[1] = gp->v_full[1];
+ sp->v[2] = gp->v_full[2];
+
+#ifdef SWIFT_DEBUG_CHECKS
+ sp->ti_kick = gp->ti_kick;
+ gp->ti_drift = sp->ti_drift;
+#endif
+
+ /* Set a smoothing length */
+ sp->h = max(c->stars.h_max, c->hydro.h_max);
+
+ /* Here comes the Sun! */
+ return sp;
+}
+
+/**
+ * @brief Add a new #spart based on a #part and link it to a new #gpart.
+ * The part and xpart are not changed.
+ *
+ * @param e The #engine.
+ * @param c The #cell from which to remove the #part.
+ * @param p The #part to remove (must be inside c).
+ * @param xp The extended data of the #part.
+ *
+ * @return A fresh #spart with the same ID, position, velocity and
+ * time-bin as the original #part.
+ */
+struct spart *cell_spawn_new_spart_from_part(struct engine *e, struct cell *c,
+ const struct part *p,
+ const struct xpart *xp) {
+ /* Quick cross-check */
+ if (c->nodeID != e->nodeID)
+ error("Can't spawn a particle in a foreign cell.");
+
+ if (p->gpart == NULL)
+ error("Trying to create a new spart from a part without gpart friend!");
+
+ /* Create a fresh (empty) spart */
+ struct spart *sp = cell_add_spart(e, c);
+
+ /* Did we run out of free spart slots? */
+ if (sp == NULL) return NULL;
+
+ /* Copy over the distance since rebuild */
+ sp->x_diff[0] = xp->x_diff[0];
+ sp->x_diff[1] = xp->x_diff[1];
+ sp->x_diff[2] = xp->x_diff[2];
+
+ /* Create a new gpart */
+ struct gpart *gp = cell_add_gpart(e, c);
+
+ /* Did we run out of free gpart slots? */
+ if (gp == NULL) {
+ /* Remove the particle created */
+ cell_remove_spart(e, c, sp);
+ return NULL;
+ }
+
+ /* Copy the gpart */
+ *gp = *p->gpart;
+
+ /* Assign the ID. */
+ sp->id = space_get_new_unique_id(e->s);
+ gp->type = swift_type_stars;
+
+ /* Re-link things */
+ sp->gpart = gp;
+ gp->id_or_neg_offset = -(sp - e->s->sparts);
+
+ /* Synchronize clocks */
+ gp->time_bin = sp->time_bin;
+
+ /* Synchronize masses, positions and velocities */
+ sp->mass = hydro_get_mass(p);
+ sp->x[0] = p->x[0];
+ sp->x[1] = p->x[1];
+ sp->x[2] = p->x[2];
+ sp->v[0] = xp->v_full[0];
+ sp->v[1] = xp->v_full[1];
+ sp->v[2] = xp->v_full[2];
+
+#ifdef SWIFT_DEBUG_CHECKS
+ sp->ti_kick = p->ti_kick;
+ sp->ti_drift = p->ti_drift;
+#endif
+
+ /* Set a smoothing length */
+ sp->h = p->h;
+
+ /* Here comes the Sun! */
+ return sp;
+}
+
+/**
+ * @brief "Remove" a #part from a #cell and replace it with a #sink
+ * connected to the same #gpart.
+ *
+ * Note that the #part is not destroyed. The pointer is still valid
+ * after this call and the properties of the #part are not altered
+ * apart from the time-bin and #gpart pointer.
+ * The particle is inhibited and will officially be removed at the next
+ * rebuild.
+ *
+ * @param e The #engine.
+ * @param c The #cell from which to remove the #part.
+ * @param p The #part to remove (must be inside c).
+ * @param xp The extended data of the #part.
+ *
+ * @return A fresh #sink with the same ID, position, velocity and
+ * time-bin as the original #part.
+ */
+struct sink *cell_convert_part_to_sink(struct engine *e, struct cell *c,
+ struct part *p, struct xpart *xp) {
+ /* Quick cross-check */
+ if (c->nodeID != e->nodeID)
+ error("Can't remove a particle in a foreign cell.");
+
+ if (p->gpart == NULL)
+ error("Trying to convert part without gpart friend to sink!");
+
+ /* Create a fresh (empty) sink */
+ struct sink *sp = cell_add_sink(e, c);
+
+ /* Did we run out of free sink slots? */
+ if (sp == NULL) return NULL;
+
+ /* Copy over the distance since rebuild */
+ sp->x_diff[0] = xp->x_diff[0];
+ sp->x_diff[1] = xp->x_diff[1];
+ sp->x_diff[2] = xp->x_diff[2];
+
+ /* Destroy the gas particle and get it's gpart friend */
+ struct gpart *gp = cell_convert_part_to_gpart(e, c, p, xp);
+
+ /* Assign the ID back */
+ sp->id = gp->id_or_neg_offset;
+ gp->type = swift_type_sink;
+
+ /* Re-link things */
+ sp->gpart = gp;
+ gp->id_or_neg_offset = -(sp - e->s->sinks);
+
+ /* Synchronize clocks */
+ gp->time_bin = sp->time_bin;
+
+ /* Synchronize masses, positions and velocities */
+ sp->mass = gp->mass;
+ sp->x[0] = gp->x[0];
+ sp->x[1] = gp->x[1];
+ sp->x[2] = gp->x[2];
+ sp->v[0] = gp->v_full[0];
+ sp->v[1] = gp->v_full[1];
+ sp->v[2] = gp->v_full[2];
+
+#ifdef SWIFT_DEBUG_CHECKS
+ sp->ti_kick = gp->ti_kick;
+ gp->ti_drift = sp->ti_drift;
+#endif
+
+ /* Set a smoothing length */
+ sp->r_cut = e->sink_properties->cut_off_radius;
+
+ /* Here comes the Sink! */
+ return sp;
+}
diff --git a/src/cell_drift.c b/src/cell_drift.c
new file mode 100644
index 0000000000000000000000000000000000000000..91292a8742924302b115931968b3ae4e86707b04
--- /dev/null
+++ b/src/cell_drift.c
@@ -0,0 +1,981 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* This object's header. */
+#include "cell.h"
+
+/* Local headers. */
+#include "active.h"
+#include "drift.h"
+#include "feedback.h"
+#include "gravity.h"
+#include "multipole.h"
+#include "pressure_floor.h"
+#include "rt.h"
+#include "star_formation.h"
+#include "tracers.h"
+
+/**
+ * @brief Recursively drifts the #part in a cell hierarchy.
+ *
+ * @param c The #cell.
+ * @param e The #engine (to get ti_current).
+ * @param force Drift the particles irrespective of the #cell flags.
+ */
+void cell_drift_part(struct cell *c, const struct engine *e, int force) {
+ const int periodic = e->s->periodic;
+ const double dim[3] = {e->s->dim[0], e->s->dim[1], e->s->dim[2]};
+ const int with_cosmology = (e->policy & engine_policy_cosmology);
+ const float hydro_h_max = e->hydro_properties->h_max;
+ const float hydro_h_min = e->hydro_properties->h_min;
+ const integertime_t ti_old_part = c->hydro.ti_old_part;
+ const integertime_t ti_current = e->ti_current;
+ struct part *const parts = c->hydro.parts;
+ struct xpart *const xparts = c->hydro.xparts;
+
+ float dx_max = 0.f, dx2_max = 0.f;
+ float dx_max_sort = 0.0f, dx2_max_sort = 0.f;
+ float cell_h_max = 0.f;
+
+ /* Drift irrespective of cell flags? */
+ force = (force || cell_get_flag(c, cell_flag_do_hydro_drift));
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Check that we only drift local cells. */
+ if (c->nodeID != engine_rank) error("Drifting a foreign cell is nope.");
+
+ /* Check that we are actually going to move forward. */
+ if (ti_current < ti_old_part) error("Attempt to drift to the past");
+#endif
+
+ /* Early abort? */
+ if (c->hydro.count == 0) {
+ /* Clear the drift flags. */
+ cell_clear_flag(c, cell_flag_do_hydro_drift | cell_flag_do_hydro_sub_drift);
+
+ /* Update the time of the last drift */
+ c->hydro.ti_old_part = ti_current;
+
+ return;
+ }
+
+ /* Ok, we have some particles somewhere in the hierarchy to drift */
+
+ /* Are we not in a leaf ? */
+ if (c->split && (force || cell_get_flag(c, cell_flag_do_hydro_sub_drift))) {
+
+ /* Loop over the progeny and collect their data. */
+ for (int k = 0; k < 8; k++) {
+ if (c->progeny[k] != NULL) {
+ struct cell *cp = c->progeny[k];
+
+ /* Collect */
+ cell_drift_part(cp, e, force);
+
+ /* Update */
+ dx_max = max(dx_max, cp->hydro.dx_max_part);
+ dx_max_sort = max(dx_max_sort, cp->hydro.dx_max_sort);
+ cell_h_max = max(cell_h_max, cp->hydro.h_max);
+ }
+ }
+
+ /* Store the values */
+ c->hydro.h_max = cell_h_max;
+ c->hydro.dx_max_part = dx_max;
+ c->hydro.dx_max_sort = dx_max_sort;
+
+ /* Update the time of the last drift */
+ c->hydro.ti_old_part = ti_current;
+
+ } else if (!c->split && force && ti_current > ti_old_part) {
+ /* Drift from the last time the cell was drifted to the current time */
+ double dt_drift, dt_kick_grav, dt_kick_hydro, dt_therm;
+ if (with_cosmology) {
+ dt_drift =
+ cosmology_get_drift_factor(e->cosmology, ti_old_part, ti_current);
+ dt_kick_grav =
+ cosmology_get_grav_kick_factor(e->cosmology, ti_old_part, ti_current);
+ dt_kick_hydro = cosmology_get_hydro_kick_factor(e->cosmology, ti_old_part,
+ ti_current);
+ dt_therm = cosmology_get_therm_kick_factor(e->cosmology, ti_old_part,
+ ti_current);
+ } else {
+ dt_drift = (ti_current - ti_old_part) * e->time_base;
+ dt_kick_grav = (ti_current - ti_old_part) * e->time_base;
+ dt_kick_hydro = (ti_current - ti_old_part) * e->time_base;
+ dt_therm = (ti_current - ti_old_part) * e->time_base;
+ }
+
+ /* Loop over all the gas particles in the cell */
+ const size_t nr_parts = c->hydro.count;
+ for (size_t k = 0; k < nr_parts; k++) {
+ /* Get a handle on the part. */
+ struct part *const p = &parts[k];
+ struct xpart *const xp = &xparts[k];
+
+ /* Ignore inhibited particles */
+ if (part_is_inhibited(p, e)) continue;
+
+ /* Apply the effects of feedback on this particle
+ * (Note: Only used in schemes that have a delayed feedback mechanism
+ * otherwise just an empty function) */
+ feedback_update_part(p, xp, e);
+
+ /* Drift... */
+ drift_part(p, xp, dt_drift, dt_kick_hydro, dt_kick_grav, dt_therm,
+ ti_old_part, ti_current, e->cosmology, e->hydro_properties,
+ e->entropy_floor);
+
+ /* Update the tracers properties */
+ tracers_after_drift(p, xp, e->internal_units, e->physical_constants,
+ with_cosmology, e->cosmology, e->hydro_properties,
+ e->cooling_func, e->time);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Make sure the particle does not drift by more than a box length. */
+ if (fabs(xp->v_full[0] * dt_drift) > e->s->dim[0] ||
+ fabs(xp->v_full[1] * dt_drift) > e->s->dim[1] ||
+ fabs(xp->v_full[2] * dt_drift) > e->s->dim[2]) {
+ error(
+ "Particle drifts by more than a box length! id %llu xp->v_full "
+ "%.5e %.5e %.5e p->v %.5e %.5e %.5e",
+ p->id, xp->v_full[0], xp->v_full[1], xp->v_full[2], p->v[0],
+ p->v[1], p->v[2]);
+ }
+#endif
+
+ /* In non-periodic BC runs, remove particles that crossed the border */
+ if (!periodic) {
+
+ /* Did the particle leave the box? */
+ if ((p->x[0] > dim[0]) || (p->x[0] < 0.) || // x
+ (p->x[1] > dim[1]) || (p->x[1] < 0.) || // y
+ (p->x[2] > dim[2]) || (p->x[2] < 0.)) { // z
+
+ lock_lock(&e->s->lock);
+
+ /* Re-check that the particle has not been removed
+ * by another thread before we do the deed. */
+ if (!part_is_inhibited(p, e)) {
+
+#ifdef WITH_LOGGER
+ if (e->policy & engine_policy_logger) {
+ /* Log the particle one last time. */
+ logger_log_part(
+ e->logger, p, xp, e, /* log_all */ 1,
+ logger_pack_flags_and_data(logger_flag_delete, 0));
+ }
+#endif
+
+ /* One last action before death? */
+ hydro_remove_part(p, xp);
+
+ /* Remove the particle entirely */
+ cell_remove_part(e, c, p, xp);
+ }
+
+ if (lock_unlock(&e->s->lock) != 0)
+ error("Failed to unlock the space!");
+
+ continue;
+ }
+ }
+
+ /* Limit h to within the allowed range */
+ p->h = min(p->h, hydro_h_max);
+ p->h = max(p->h, hydro_h_min);
+
+ /* Compute (square of) motion since last cell construction */
+ const float dx2 = xp->x_diff[0] * xp->x_diff[0] +
+ xp->x_diff[1] * xp->x_diff[1] +
+ xp->x_diff[2] * xp->x_diff[2];
+ dx2_max = max(dx2_max, dx2);
+ const float dx2_sort = xp->x_diff_sort[0] * xp->x_diff_sort[0] +
+ xp->x_diff_sort[1] * xp->x_diff_sort[1] +
+ xp->x_diff_sort[2] * xp->x_diff_sort[2];
+ dx2_max_sort = max(dx2_max_sort, dx2_sort);
+
+ /* Update the maximal smoothing length in the cell */
+ cell_h_max = max(cell_h_max, p->h);
+
+ /* Mark the particle has not being swallowed */
+ black_holes_mark_part_as_not_swallowed(&p->black_holes_data);
+
+ /* Get ready for a density calculation */
+ if (part_is_active(p, e)) {
+ hydro_init_part(p, &e->s->hs);
+ black_holes_init_potential(&p->black_holes_data);
+ chemistry_init_part(p, e->chemistry);
+ pressure_floor_init_part(p, xp);
+ star_formation_init_part(p, e->star_formation);
+ tracers_after_init(p, xp, e->internal_units, e->physical_constants,
+ with_cosmology, e->cosmology, e->hydro_properties,
+ e->cooling_func, e->time);
+ rt_init_part(p);
+ }
+ }
+
+ /* Now, get the maximal particle motion from its square */
+ dx_max = sqrtf(dx2_max);
+ dx_max_sort = sqrtf(dx2_max_sort);
+
+ /* Store the values */
+ c->hydro.h_max = cell_h_max;
+ c->hydro.dx_max_part = dx_max;
+ c->hydro.dx_max_sort = dx_max_sort;
+
+ /* Update the time of the last drift */
+ c->hydro.ti_old_part = ti_current;
+ }
+
+ /* Clear the drift flags. */
+ cell_clear_flag(c, cell_flag_do_hydro_drift | cell_flag_do_hydro_sub_drift);
+}
+
+/**
+ * @brief Recursively drifts the #gpart in a cell hierarchy.
+ *
+ * @param c The #cell.
+ * @param e The #engine (to get ti_current).
+ * @param force Drift the particles irrespective of the #cell flags.
+ */
+void cell_drift_gpart(struct cell *c, const struct engine *e, int force) {
+ const int periodic = e->s->periodic;
+ const double dim[3] = {e->s->dim[0], e->s->dim[1], e->s->dim[2]};
+ const int with_cosmology = (e->policy & engine_policy_cosmology);
+ const integertime_t ti_old_gpart = c->grav.ti_old_part;
+ const integertime_t ti_current = e->ti_current;
+ struct gpart *const gparts = c->grav.parts;
+ const struct gravity_props *grav_props = e->gravity_properties;
+
+ /* Drift irrespective of cell flags? */
+ force = (force || cell_get_flag(c, cell_flag_do_grav_drift));
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Check that we only drift local cells. */
+ if (c->nodeID != engine_rank) error("Drifting a foreign cell is nope.");
+
+ /* Check that we are actually going to move forward. */
+ if (ti_current < ti_old_gpart) error("Attempt to drift to the past");
+#endif
+
+ /* Early abort? */
+ if (c->grav.count == 0) {
+ /* Clear the drift flags. */
+ cell_clear_flag(c, cell_flag_do_grav_drift | cell_flag_do_grav_sub_drift);
+
+ /* Update the time of the last drift */
+ c->grav.ti_old_part = ti_current;
+
+ return;
+ }
+
+ /* Ok, we have some particles somewhere in the hierarchy to drift */
+
+ /* Are we not in a leaf ? */
+ if (c->split && (force || cell_get_flag(c, cell_flag_do_grav_sub_drift))) {
+
+ /* Loop over the progeny and collect their data. */
+ for (int k = 0; k < 8; k++) {
+ if (c->progeny[k] != NULL) {
+ struct cell *cp = c->progeny[k];
+
+ /* Recurse */
+ cell_drift_gpart(cp, e, force);
+ }
+ }
+
+ /* Update the time of the last drift */
+ c->grav.ti_old_part = ti_current;
+
+ } else if (!c->split && force && ti_current > ti_old_gpart) {
+ /* Drift from the last time the cell was drifted to the current time */
+ double dt_drift;
+ if (with_cosmology) {
+ dt_drift =
+ cosmology_get_drift_factor(e->cosmology, ti_old_gpart, ti_current);
+ } else {
+ dt_drift = (ti_current - ti_old_gpart) * e->time_base;
+ }
+
+ /* Loop over all the g-particles in the cell */
+ const size_t nr_gparts = c->grav.count;
+ for (size_t k = 0; k < nr_gparts; k++) {
+ /* Get a handle on the gpart. */
+ struct gpart *const gp = &gparts[k];
+
+ /* Ignore inhibited particles */
+ if (gpart_is_inhibited(gp, e)) continue;
+
+ /* Drift... */
+ drift_gpart(gp, dt_drift, ti_old_gpart, ti_current, grav_props, e);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Make sure the particle does not drift by more than a box length. */
+ if (fabs(gp->v_full[0] * dt_drift) > e->s->dim[0] ||
+ fabs(gp->v_full[1] * dt_drift) > e->s->dim[1] ||
+ fabs(gp->v_full[2] * dt_drift) > e->s->dim[2]) {
+ error(
+ "Particle drifts by more than a box length! gp->v_full %.5e %.5e "
+ "%.5e",
+ gp->v_full[0], gp->v_full[1], gp->v_full[2]);
+ }
+#endif
+
+ /* In non-periodic BC runs, remove particles that crossed the border */
+ if (!periodic) {
+
+ /* Did the particle leave the box? */
+ if ((gp->x[0] > dim[0]) || (gp->x[0] < 0.) || // x
+ (gp->x[1] > dim[1]) || (gp->x[1] < 0.) || // y
+ (gp->x[2] > dim[2]) || (gp->x[2] < 0.)) { // z
+
+ lock_lock(&e->s->lock);
+
+ /* Re-check that the particle has not been removed
+ * by another thread before we do the deed. */
+ if (!gpart_is_inhibited(gp, e)) {
+
+ /* Remove the particle entirely */
+ if (gp->type == swift_type_dark_matter) {
+
+#ifdef WITH_LOGGER
+ if (e->policy & engine_policy_logger) {
+ /* Log the particle one last time. */
+ logger_log_gpart(
+ e->logger, gp, e, /* log_all */ 1,
+ logger_pack_flags_and_data(logger_flag_delete, 0));
+ }
+#endif
+
+ /* Remove the particle */
+ cell_remove_gpart(e, c, gp);
+ }
+ }
+
+ if (lock_unlock(&e->s->lock) != 0)
+ error("Failed to unlock the space!");
+
+ continue;
+ }
+ }
+
+ /* Init gravity force fields. */
+ if (gpart_is_active(gp, e)) {
+ gravity_init_gpart(gp);
+ }
+ }
+
+ /* Update the time of the last drift */
+ c->grav.ti_old_part = ti_current;
+ }
+
+ /* Clear the drift flags. */
+ cell_clear_flag(c, cell_flag_do_grav_drift | cell_flag_do_grav_sub_drift);
+}
+
+/**
+ * @brief Recursively drifts the #spart in a cell hierarchy.
+ *
+ * @param c The #cell.
+ * @param e The #engine (to get ti_current).
+ * @param force Drift the particles irrespective of the #cell flags.
+ */
+void cell_drift_spart(struct cell *c, const struct engine *e, int force) {
+ const int periodic = e->s->periodic;
+ const double dim[3] = {e->s->dim[0], e->s->dim[1], e->s->dim[2]};
+ const int with_cosmology = (e->policy & engine_policy_cosmology);
+ const float stars_h_max = e->hydro_properties->h_max;
+ const float stars_h_min = e->hydro_properties->h_min;
+ const integertime_t ti_old_spart = c->stars.ti_old_part;
+ const integertime_t ti_current = e->ti_current;
+ struct spart *const sparts = c->stars.parts;
+
+ float dx_max = 0.f, dx2_max = 0.f;
+ float dx_max_sort = 0.0f, dx2_max_sort = 0.f;
+ float cell_h_max = 0.f;
+
+ /* Drift irrespective of cell flags? */
+ force = (force || cell_get_flag(c, cell_flag_do_stars_drift));
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Check that we only drift local cells. */
+ if (c->nodeID != engine_rank) error("Drifting a foreign cell is nope.");
+
+ /* Check that we are actually going to move forward. */
+ if (ti_current < ti_old_spart) error("Attempt to drift to the past");
+#endif
+
+ /* Early abort? */
+ if (c->stars.count == 0) {
+ /* Clear the drift flags. */
+ cell_clear_flag(c, cell_flag_do_stars_drift | cell_flag_do_stars_sub_drift);
+
+ /* Update the time of the last drift */
+ c->stars.ti_old_part = ti_current;
+
+ return;
+ }
+
+ /* Ok, we have some particles somewhere in the hierarchy to drift */
+
+ /* Are we not in a leaf ? */
+ if (c->split && (force || cell_get_flag(c, cell_flag_do_stars_sub_drift))) {
+
+ /* Loop over the progeny and collect their data. */
+ for (int k = 0; k < 8; k++) {
+ if (c->progeny[k] != NULL) {
+ struct cell *cp = c->progeny[k];
+
+ /* Recurse */
+ cell_drift_spart(cp, e, force);
+
+ /* Update */
+ dx_max = max(dx_max, cp->stars.dx_max_part);
+ dx_max_sort = max(dx_max_sort, cp->stars.dx_max_sort);
+ cell_h_max = max(cell_h_max, cp->stars.h_max);
+ }
+ }
+
+ /* Store the values */
+ c->stars.h_max = cell_h_max;
+ c->stars.dx_max_part = dx_max;
+ c->stars.dx_max_sort = dx_max_sort;
+
+ /* Update the time of the last drift */
+ c->stars.ti_old_part = ti_current;
+
+ } else if (!c->split && force && ti_current > ti_old_spart) {
+ /* Drift from the last time the cell was drifted to the current time */
+ double dt_drift;
+ if (with_cosmology) {
+ dt_drift =
+ cosmology_get_drift_factor(e->cosmology, ti_old_spart, ti_current);
+ } else {
+ dt_drift = (ti_current - ti_old_spart) * e->time_base;
+ }
+
+ /* Loop over all the star particles in the cell */
+ const size_t nr_sparts = c->stars.count;
+ for (size_t k = 0; k < nr_sparts; k++) {
+ /* Get a handle on the spart. */
+ struct spart *const sp = &sparts[k];
+
+ /* Ignore inhibited particles */
+ if (spart_is_inhibited(sp, e)) continue;
+
+ /* Drift... */
+ drift_spart(sp, dt_drift, ti_old_spart, ti_current);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Make sure the particle does not drift by more than a box length. */
+ if (fabs(sp->v[0] * dt_drift) > e->s->dim[0] ||
+ fabs(sp->v[1] * dt_drift) > e->s->dim[1] ||
+ fabs(sp->v[2] * dt_drift) > e->s->dim[2]) {
+ error("Particle drifts by more than a box length!");
+ }
+#endif
+
+ /* In non-periodic BC runs, remove particles that crossed the border */
+ if (!periodic) {
+
+ /* Did the particle leave the box? */
+ if ((sp->x[0] > dim[0]) || (sp->x[0] < 0.) || // x
+ (sp->x[1] > dim[1]) || (sp->x[1] < 0.) || // y
+ (sp->x[2] > dim[2]) || (sp->x[2] < 0.)) { // z
+
+ lock_lock(&e->s->lock);
+
+ /* Re-check that the particle has not been removed
+ * by another thread before we do the deed. */
+ if (!spart_is_inhibited(sp, e)) {
+
+#ifdef WITH_LOGGER
+ if (e->policy & engine_policy_logger) {
+ /* Log the particle one last time. */
+ logger_log_spart(
+ e->logger, sp, e, /* log_all */ 1,
+ logger_pack_flags_and_data(logger_flag_delete, 0));
+ }
+#endif
+
+ /* Remove the particle entirely */
+ cell_remove_spart(e, c, sp);
+ }
+
+ if (lock_unlock(&e->s->lock) != 0)
+ error("Failed to unlock the space!");
+
+ continue;
+ }
+ }
+
+ /* Limit h to within the allowed range */
+ sp->h = min(sp->h, stars_h_max);
+ sp->h = max(sp->h, stars_h_min);
+
+ /* Compute (square of) motion since last cell construction */
+ const float dx2 = sp->x_diff[0] * sp->x_diff[0] +
+ sp->x_diff[1] * sp->x_diff[1] +
+ sp->x_diff[2] * sp->x_diff[2];
+ dx2_max = max(dx2_max, dx2);
+
+ const float dx2_sort = sp->x_diff_sort[0] * sp->x_diff_sort[0] +
+ sp->x_diff_sort[1] * sp->x_diff_sort[1] +
+ sp->x_diff_sort[2] * sp->x_diff_sort[2];
+
+ dx2_max_sort = max(dx2_max_sort, dx2_sort);
+
+ /* Maximal smoothing length */
+ cell_h_max = max(cell_h_max, sp->h);
+
+ /* Get ready for a density calculation */
+ if (spart_is_active(sp, e)) {
+ stars_init_spart(sp);
+ feedback_init_spart(sp);
+ rt_init_spart(sp);
+ }
+ }
+
+ /* Now, get the maximal particle motion from its square */
+ dx_max = sqrtf(dx2_max);
+ dx_max_sort = sqrtf(dx2_max_sort);
+
+ /* Store the values */
+ c->stars.h_max = cell_h_max;
+ c->stars.dx_max_part = dx_max;
+ c->stars.dx_max_sort = dx_max_sort;
+
+ /* Update the time of the last drift */
+ c->stars.ti_old_part = ti_current;
+ }
+
+ /* Clear the drift flags. */
+ cell_clear_flag(c, cell_flag_do_stars_drift | cell_flag_do_stars_sub_drift);
+}
+
+/**
+ * @brief Recursively drifts the #bpart in a cell hierarchy.
+ *
+ * @param c The #cell.
+ * @param e The #engine (to get ti_current).
+ * @param force Drift the particles irrespective of the #cell flags.
+ */
+void cell_drift_bpart(struct cell *c, const struct engine *e, int force) {
+
+ const int periodic = e->s->periodic;
+ const double dim[3] = {e->s->dim[0], e->s->dim[1], e->s->dim[2]};
+ const int with_cosmology = (e->policy & engine_policy_cosmology);
+ const float black_holes_h_max = e->hydro_properties->h_max;
+ const float black_holes_h_min = e->hydro_properties->h_min;
+ const integertime_t ti_old_bpart = c->black_holes.ti_old_part;
+ const integertime_t ti_current = e->ti_current;
+ struct bpart *const bparts = c->black_holes.parts;
+
+ float dx_max = 0.f, dx2_max = 0.f;
+ float cell_h_max = 0.f;
+
+ /* Drift irrespective of cell flags? */
+ force = (force || cell_get_flag(c, cell_flag_do_bh_drift));
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Check that we only drift local cells. */
+ if (c->nodeID != engine_rank) error("Drifting a foreign cell is nope.");
+
+ /* Check that we are actually going to move forward. */
+ if (ti_current < ti_old_bpart) error("Attempt to drift to the past");
+#endif
+
+ /* Early abort? */
+ if (c->black_holes.count == 0) {
+
+ /* Clear the drift flags. */
+ cell_clear_flag(c, cell_flag_do_bh_drift | cell_flag_do_bh_sub_drift);
+
+ /* Update the time of the last drift */
+ c->black_holes.ti_old_part = ti_current;
+
+ return;
+ }
+
+ /* Ok, we have some particles somewhere in the hierarchy to drift */
+
+ /* Are we not in a leaf ? */
+ if (c->split && (force || cell_get_flag(c, cell_flag_do_bh_sub_drift))) {
+
+ /* Loop over the progeny and collect their data. */
+ for (int k = 0; k < 8; k++) {
+ if (c->progeny[k] != NULL) {
+ struct cell *cp = c->progeny[k];
+
+ /* Recurse */
+ cell_drift_bpart(cp, e, force);
+
+ /* Update */
+ dx_max = max(dx_max, cp->black_holes.dx_max_part);
+ cell_h_max = max(cell_h_max, cp->black_holes.h_max);
+ }
+ }
+
+ /* Store the values */
+ c->black_holes.h_max = cell_h_max;
+ c->black_holes.dx_max_part = dx_max;
+
+ /* Update the time of the last drift */
+ c->black_holes.ti_old_part = ti_current;
+
+ } else if (!c->split && force && ti_current > ti_old_bpart) {
+
+ /* Drift from the last time the cell was drifted to the current time */
+ double dt_drift;
+ if (with_cosmology) {
+ dt_drift =
+ cosmology_get_drift_factor(e->cosmology, ti_old_bpart, ti_current);
+ } else {
+ dt_drift = (ti_current - ti_old_bpart) * e->time_base;
+ }
+
+ /* Loop over all the star particles in the cell */
+ const size_t nr_bparts = c->black_holes.count;
+ for (size_t k = 0; k < nr_bparts; k++) {
+
+ /* Get a handle on the bpart. */
+ struct bpart *const bp = &bparts[k];
+
+ /* Ignore inhibited particles */
+ if (bpart_is_inhibited(bp, e)) continue;
+
+ /* Drift... */
+ drift_bpart(bp, dt_drift, ti_old_bpart, ti_current);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Make sure the particle does not drift by more than a box length. */
+ if (fabs(bp->v[0] * dt_drift) > e->s->dim[0] ||
+ fabs(bp->v[1] * dt_drift) > e->s->dim[1] ||
+ fabs(bp->v[2] * dt_drift) > e->s->dim[2]) {
+ error("Particle drifts by more than a box length!");
+ }
+#endif
+
+ /* In non-periodic BC runs, remove particles that crossed the border */
+ if (!periodic) {
+
+ /* Did the particle leave the box? */
+ if ((bp->x[0] > dim[0]) || (bp->x[0] < 0.) || // x
+ (bp->x[1] > dim[1]) || (bp->x[1] < 0.) || // y
+ (bp->x[2] > dim[2]) || (bp->x[2] < 0.)) { // z
+
+ lock_lock(&e->s->lock);
+
+ /* Re-check that the particle has not been removed
+ * by another thread before we do the deed. */
+ if (!bpart_is_inhibited(bp, e)) {
+
+#ifdef WITH_LOGGER
+ if (e->policy & engine_policy_logger) {
+ error("Logging of black hole particles is not yet implemented.");
+ }
+#endif
+
+ /* Remove the particle entirely */
+ cell_remove_bpart(e, c, bp);
+ }
+
+ if (lock_unlock(&e->s->lock) != 0)
+ error("Failed to unlock the space!");
+
+ continue;
+ }
+ }
+
+ /* Limit h to within the allowed range */
+ bp->h = min(bp->h, black_holes_h_max);
+ bp->h = max(bp->h, black_holes_h_min);
+
+ /* Compute (square of) motion since last cell construction */
+ const float dx2 = bp->x_diff[0] * bp->x_diff[0] +
+ bp->x_diff[1] * bp->x_diff[1] +
+ bp->x_diff[2] * bp->x_diff[2];
+ dx2_max = max(dx2_max, dx2);
+
+ /* Maximal smoothing length */
+ cell_h_max = max(cell_h_max, bp->h);
+
+ /* Mark the particle has not being swallowed */
+ black_holes_mark_bpart_as_not_swallowed(&bp->merger_data);
+
+ /* Get ready for a density calculation */
+ if (bpart_is_active(bp, e)) {
+ black_holes_init_bpart(bp);
+ }
+ }
+
+ /* Now, get the maximal particle motion from its square */
+ dx_max = sqrtf(dx2_max);
+
+ /* Store the values */
+ c->black_holes.h_max = cell_h_max;
+ c->black_holes.dx_max_part = dx_max;
+
+ /* Update the time of the last drift */
+ c->black_holes.ti_old_part = ti_current;
+ }
+
+ /* Clear the drift flags. */
+ cell_clear_flag(c, cell_flag_do_bh_drift | cell_flag_do_bh_sub_drift);
+}
+
+/**
+ * @brief Recursively drifts the #sink's in a cell hierarchy.
+ *
+ * @param c The #cell.
+ * @param e The #engine (to get ti_current).
+ * @param force Drift the particles irrespective of the #cell flags.
+ */
+void cell_drift_sink(struct cell *c, const struct engine *e, int force) {
+
+ const int periodic = e->s->periodic;
+ const double dim[3] = {e->s->dim[0], e->s->dim[1], e->s->dim[2]};
+ const int with_cosmology = (e->policy & engine_policy_cosmology);
+ const integertime_t ti_old_sink = c->sinks.ti_old_part;
+ const integertime_t ti_current = e->ti_current;
+ struct sink *const sinks = c->sinks.parts;
+
+ float dx_max = 0.f, dx2_max = 0.f;
+ float cell_r_max = 0.f;
+
+ /* Drift irrespective of cell flags? */
+ force = (force || cell_get_flag(c, cell_flag_do_sink_drift));
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Check that we only drift local cells. */
+ if (c->nodeID != engine_rank) error("Drifting a foreign cell is nope.");
+
+ /* Check that we are actually going to move forward. */
+ if (ti_current < ti_old_sink) error("Attempt to drift to the past");
+#endif
+
+ /* Early abort? */
+ if (c->sinks.count == 0) {
+
+ /* Clear the drift flags. */
+ cell_clear_flag(c, cell_flag_do_sink_drift | cell_flag_do_sink_sub_drift);
+
+ /* Update the time of the last drift */
+ c->sinks.ti_old_part = ti_current;
+
+ return;
+ }
+
+ /* Ok, we have some particles somewhere in the hierarchy to drift */
+
+ /* Are we not in a leaf ? */
+ if (c->split && (force || cell_get_flag(c, cell_flag_do_sink_sub_drift))) {
+
+ /* Loop over the progeny and collect their data. */
+ for (int k = 0; k < 8; k++) {
+ if (c->progeny[k] != NULL) {
+ struct cell *cp = c->progeny[k];
+
+ /* Recurse */
+ cell_drift_sink(cp, e, force);
+
+ /* Update */
+ dx_max = max(dx_max, cp->sinks.dx_max_part);
+ cell_r_max = max(cell_r_max, cp->sinks.r_cut_max);
+ }
+ }
+
+ /* Store the values */
+ c->sinks.r_cut_max = cell_r_max;
+ c->sinks.dx_max_part = dx_max;
+
+ /* Update the time of the last drift */
+ c->sinks.ti_old_part = ti_current;
+
+ } else if (!c->split && force && ti_current > ti_old_sink) {
+
+ /* Drift from the last time the cell was drifted to the current time */
+ double dt_drift;
+ if (with_cosmology) {
+ dt_drift =
+ cosmology_get_drift_factor(e->cosmology, ti_old_sink, ti_current);
+ } else {
+ dt_drift = (ti_current - ti_old_sink) * e->time_base;
+ }
+
+ /* Loop over all the star particles in the cell */
+ const size_t nr_sinks = c->sinks.count;
+ for (size_t k = 0; k < nr_sinks; k++) {
+
+ /* Get a handle on the sink. */
+ struct sink *const sink = &sinks[k];
+
+ /* Ignore inhibited particles */
+ if (sink_is_inhibited(sink, e)) continue;
+
+ /* Drift... */
+ drift_sink(sink, dt_drift, ti_old_sink, ti_current);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Make sure the particle does not drift by more than a box length. */
+ if (fabs(sink->v[0] * dt_drift) > e->s->dim[0] ||
+ fabs(sink->v[1] * dt_drift) > e->s->dim[1] ||
+ fabs(sink->v[2] * dt_drift) > e->s->dim[2]) {
+ error("Particle drifts by more than a box length!");
+ }
+#endif
+
+ /* In non-periodic BC runs, remove particles that crossed the border */
+ if (!periodic) {
+
+ /* Did the particle leave the box? */
+ if ((sink->x[0] > dim[0]) || (sink->x[0] < 0.) || // x
+ (sink->x[1] > dim[1]) || (sink->x[1] < 0.) || // y
+ (sink->x[2] > dim[2]) || (sink->x[2] < 0.)) { // z
+
+ lock_lock(&e->s->lock);
+
+ /* Re-check that the particle has not been removed
+ * by another thread before we do the deed. */
+ if (!sink_is_inhibited(sink, e)) {
+
+#ifdef WITH_LOGGER
+ if (e->policy & engine_policy_logger) {
+ error("Logging of sink particles is not yet implemented.");
+ }
+#endif
+
+ /* Remove the particle entirely */
+ // cell_remove_sink(e, c, bp);
+ error("TODO: loic implement cell_remove_sink");
+ }
+
+ if (lock_unlock(&e->s->lock) != 0)
+ error("Failed to unlock the space!");
+
+ continue;
+ }
+ }
+
+ /* sp->h does not need to be limited. */
+
+ /* Compute (square of) motion since last cell construction */
+ const float dx2 = sink->x_diff[0] * sink->x_diff[0] +
+ sink->x_diff[1] * sink->x_diff[1] +
+ sink->x_diff[2] * sink->x_diff[2];
+ dx2_max = max(dx2_max, dx2);
+
+ /* Maximal smoothing length */
+ cell_r_max = max(cell_r_max, sink->r_cut);
+
+ /* Get ready for a density calculation */
+ if (sink_is_active(sink, e)) {
+ sink_init_sink(sink);
+ }
+ }
+
+ /* Now, get the maximal particle motion from its square */
+ dx_max = sqrtf(dx2_max);
+
+ /* Store the values */
+ c->sinks.r_cut_max = cell_r_max;
+ c->sinks.dx_max_part = dx_max;
+
+ /* Update the time of the last drift */
+ c->sinks.ti_old_part = ti_current;
+ }
+
+ /* Clear the drift flags. */
+ cell_clear_flag(c, cell_flag_do_sink_drift | cell_flag_do_sink_sub_drift);
+}
+
+/**
+ * @brief Recursively drifts all multipoles in a cell hierarchy.
+ *
+ * @param c The #cell.
+ * @param e The #engine (to get ti_current).
+ */
+void cell_drift_all_multipoles(struct cell *c, const struct engine *e) {
+ const integertime_t ti_old_multipole = c->grav.ti_old_multipole;
+ const integertime_t ti_current = e->ti_current;
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Check that we are actually going to move forward. */
+ if (ti_current < ti_old_multipole) error("Attempt to drift to the past");
+#endif
+
+ /* Drift from the last time the cell was drifted to the current time */
+ double dt_drift;
+ if (e->policy & engine_policy_cosmology)
+ dt_drift =
+ cosmology_get_drift_factor(e->cosmology, ti_old_multipole, ti_current);
+ else
+ dt_drift = (ti_current - ti_old_multipole) * e->time_base;
+
+ /* Drift the multipole */
+ if (ti_current > ti_old_multipole) gravity_drift(c->grav.multipole, dt_drift);
+
+ /* Are we not in a leaf ? */
+ if (c->split) {
+ /* Loop over the progeny and recurse. */
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL) cell_drift_all_multipoles(c->progeny[k], e);
+ }
+
+ /* Update the time of the last drift */
+ c->grav.ti_old_multipole = ti_current;
+}
+
+/**
+ * @brief Drifts the multipole of a cell to the current time.
+ *
+ * Only drifts the multipole at this level. Multipoles deeper in the
+ * tree are not updated.
+ *
+ * @param c The #cell.
+ * @param e The #engine (to get ti_current).
+ */
+void cell_drift_multipole(struct cell *c, const struct engine *e) {
+ const integertime_t ti_old_multipole = c->grav.ti_old_multipole;
+ const integertime_t ti_current = e->ti_current;
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Check that we are actually going to move forward. */
+ if (ti_current < ti_old_multipole) error("Attempt to drift to the past");
+#endif
+
+ /* Drift from the last time the cell was drifted to the current time */
+ double dt_drift;
+ if (e->policy & engine_policy_cosmology)
+ dt_drift =
+ cosmology_get_drift_factor(e->cosmology, ti_old_multipole, ti_current);
+ else
+ dt_drift = (ti_current - ti_old_multipole) * e->time_base;
+
+ if (ti_current > ti_old_multipole) gravity_drift(c->grav.multipole, dt_drift);
+
+ /* Update the time of the last drift */
+ c->grav.ti_old_multipole = ti_current;
+}
diff --git a/src/cell_grav.h b/src/cell_grav.h
new file mode 100644
index 0000000000000000000000000000000000000000..4921021a02aec61c25a1431bc8ddd7fa7e022025
--- /dev/null
+++ b/src/cell_grav.h
@@ -0,0 +1,124 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+#ifndef SWIFT_CELL_GRAV_H
+#define SWIFT_CELL_GRAV_H
+
+/* Config parameters. */
+#include "../config.h"
+
+/* Local includes. */
+#include "lock.h"
+#include "timeline.h"
+
+/**
+ * @brief Gravity-related cell variables.
+ */
+
+struct cell_grav {
+
+ /*! Pointer to the #gpart data. */
+ struct gpart *parts;
+
+ /*! Pointer to the #spart data at rebuild time. */
+ struct gpart *parts_rebuild;
+
+ /*! This cell's multipole. */
+ struct gravity_tensors *multipole;
+
+ /*! Super cell, i.e. the highest-level parent cell that has a grav pair/self
+ * tasks */
+ struct cell *super;
+
+ /*! The drift task for gparts */
+ struct task *drift;
+
+ /*! Implicit task (going up- and down the tree) for the #gpart drifts */
+ struct task *drift_out;
+
+ /*! Linked list of the tasks computing this cell's gravity forces. */
+ struct link *grav;
+
+ /*! Linked list of the tasks computing this cell's gravity M-M forces. */
+ struct link *mm;
+
+ /*! The multipole initialistation task */
+ struct task *init;
+
+ /*! Implicit task for the gravity initialisation */
+ struct task *init_out;
+
+ /*! Task computing long range non-periodic gravity interactions */
+ struct task *long_range;
+
+ /*! Implicit task for the down propagation */
+ struct task *down_in;
+
+ /*! Task propagating the multipole to the particles */
+ struct task *down;
+
+ /*! The task to end the force calculation */
+ struct task *end_force;
+
+ /*! Minimum end of (integer) time step in this cell for gravity tasks. */
+ integertime_t ti_end_min;
+
+ /*! Maximum end of (integer) time step in this cell for gravity tasks. */
+ integertime_t ti_end_max;
+
+ /*! Maximum beginning of (integer) time step in this cell for gravity tasks.
+ */
+ integertime_t ti_beg_max;
+
+ /*! Last (integer) time the cell's gpart were drifted forward in time. */
+ integertime_t ti_old_part;
+
+ /*! Last (integer) time the cell's multipole was drifted forward in time. */
+ integertime_t ti_old_multipole;
+
+ /*! Spin lock for various uses (#gpart case). */
+ swift_lock_type plock;
+
+ /*! Spin lock for various uses (#multipole case). */
+ swift_lock_type mlock;
+
+ /*! Spin lock for star formation use. */
+ swift_lock_type star_formation_lock;
+
+ /*! Nr of #gpart in this cell. */
+ int count;
+
+ /*! Nr of #gpart this cell can hold after addition of new #gpart. */
+ int count_total;
+
+ /*! Number of #gpart updated in this cell. */
+ int updated;
+
+ /*! Is the #gpart data of this cell being used in a sub-cell? */
+ int phold;
+
+ /*! Is the #multipole data of this cell being used in a sub-cell? */
+ int mhold;
+
+ /*! Number of M-M tasks that are associated with this cell. */
+ short int nr_mm_tasks;
+};
+
+#endif /* SWIFT_CELL_GRAV_H */
diff --git a/src/cell_hydro.h b/src/cell_hydro.h
new file mode 100644
index 0000000000000000000000000000000000000000..42c36d9b917dc0575ea1f84ad61e335561c99f79
--- /dev/null
+++ b/src/cell_hydro.h
@@ -0,0 +1,178 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+#ifndef SWIFT_CELL_HYDRO_H
+#define SWIFT_CELL_HYDRO_H
+
+/* Config parameters. */
+#include "../config.h"
+
+/* Local includes. */
+#include "lock.h"
+#include "timeline.h"
+
+/**
+ * @brief Hydro-related cell variables.
+ */
+struct cell_hydro {
+
+ /*! Pointer to the #part data. */
+ struct part *parts;
+
+ /*! Pointer to the #xpart data. */
+ struct xpart *xparts;
+
+ /*! Pointer for the sorted indices. */
+ struct sort_entry *sort;
+
+ /*! Super cell, i.e. the highest-level parent cell that has a hydro
+ * pair/self tasks */
+ struct cell *super;
+
+ /*! The task computing this cell's sorts. */
+ struct task *sorts;
+
+ /*! The drift task for parts */
+ struct task *drift;
+
+ /*! Linked list of the tasks computing this cell's hydro density. */
+ struct link *density;
+
+ /* Linked list of the tasks computing this cell's hydro gradients. */
+ struct link *gradient;
+
+ /*! Linked list of the tasks computing this cell's hydro forces. */
+ struct link *force;
+
+ /*! Linked list of the tasks computing this cell's limiter. */
+ struct link *limiter;
+
+ /*! Dependency implicit task for the ghost (in->ghost->out)*/
+ struct task *ghost_in;
+
+ /*! Dependency implicit task for the ghost (in->ghost->out)*/
+ struct task *ghost_out;
+
+ /*! The ghost task itself */
+ struct task *ghost;
+
+ /*! The extra ghost task for complex hydro schemes */
+ struct task *extra_ghost;
+
+ /*! The task to end the force calculation */
+ struct task *end_force;
+
+ /*! Dependency implicit task for cooling (in->cooling->out) */
+ struct task *cooling_in;
+
+ /*! Dependency implicit task for cooling (in->cooling->out) */
+ struct task *cooling_out;
+
+ /*! Task for cooling */
+ struct task *cooling;
+
+ /*! Task for star formation */
+ struct task *star_formation;
+
+ /*! Task for sink formation */
+ struct task *sink_formation;
+
+ /*! Task for sorting the stars again after a SF event */
+ struct task *stars_resort;
+
+ /*! Radiative transfer ghost in task */
+ struct task *rt_in;
+
+ /*! Radiative transfer ghost out task */
+ struct task *rt_out;
+
+ /*! Radiative transfer ghost1 task (finishes up injection) */
+ struct task *rt_ghost1;
+
+ /*! Task for self/pair injection step of radiative transfer */
+ struct link *rt_inject;
+
+ /*! Last (integer) time the cell's part were drifted forward in time. */
+ integertime_t ti_old_part;
+
+ /*! Minimum end of (integer) time step in this cell for hydro tasks. */
+ integertime_t ti_end_min;
+
+ /*! Maximum end of (integer) time step in this cell for hydro tasks. */
+ integertime_t ti_end_max;
+
+ /*! Maximum beginning of (integer) time step in this cell for hydro tasks.
+ */
+ integertime_t ti_beg_max;
+
+ /*! Spin lock for various uses (#part case). */
+ swift_lock_type lock;
+
+ /*! Max smoothing length in this cell. */
+ float h_max;
+
+ /*! Maximum part movement in this cell since last construction. */
+ float dx_max_part;
+
+ /*! Maximum particle movement in this cell since the last sort. */
+ float dx_max_sort;
+
+ /*! Values of h_max before the drifts, used for sub-cell tasks. */
+ float h_max_old;
+
+ /*! Values of dx_max before the drifts, used for sub-cell tasks. */
+ float dx_max_part_old;
+
+ /*! Values of dx_max_sort before the drifts, used for sub-cell tasks. */
+ float dx_max_sort_old;
+
+ /*! Nr of #part in this cell. */
+ int count;
+
+ /*! Nr of #part this cell can hold after addition of new #part. */
+ int count_total;
+
+ /*! Number of #part updated in this cell. */
+ int updated;
+
+ /*! Is the #part data of this cell being used in a sub-cell? */
+ int hold;
+
+ /*! Bit mask of sort directions that will be needed in the next timestep. */
+ uint16_t requires_sorts;
+
+ /*! Bit mask of sorts that need to be computed for this cell. */
+ uint16_t do_sort;
+
+ /*! Bit-mask indicating the sorted directions */
+ uint16_t sorted;
+
+ /*! Bit-mask indicating the sorted directions */
+ uint16_t sort_allocated;
+
+#ifdef SWIFT_DEBUG_CHECKS
+
+ /*! Last (integer) time the cell's sort arrays were updated. */
+ integertime_t ti_sort;
+
+#endif
+};
+
+#endif /* SWIFT_CELL_HYDRO_H */
diff --git a/src/cell_lock.c b/src/cell_lock.c
new file mode 100644
index 0000000000000000000000000000000000000000..29b7ada3f87491c05fd7904994a3eb925a6f5a8c
--- /dev/null
+++ b/src/cell_lock.c
@@ -0,0 +1,498 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* This object's header. */
+#include "cell.h"
+
+/* Local headers. */
+#include "timers.h"
+
+/**
+ * @brief Lock a cell for access to its array of #part and hold its parents.
+ *
+ * @param c The #cell.
+ * @return 0 on success, 1 on failure
+ */
+int cell_locktree(struct cell *c) {
+ TIMER_TIC;
+
+ /* First of all, try to lock this cell. */
+ if (c->hydro.hold || lock_trylock(&c->hydro.lock) != 0) {
+ TIMER_TOC(timer_locktree);
+ return 1;
+ }
+
+ /* Did somebody hold this cell in the meantime? */
+ if (c->hydro.hold) {
+ /* Unlock this cell. */
+ if (lock_unlock(&c->hydro.lock) != 0) error("Failed to unlock cell.");
+
+ /* Admit defeat. */
+ TIMER_TOC(timer_locktree);
+ return 1;
+ }
+
+ /* Climb up the tree and lock/hold/unlock. */
+ struct cell *finger;
+ for (finger = c->parent; finger != NULL; finger = finger->parent) {
+ /* Lock this cell. */
+ if (lock_trylock(&finger->hydro.lock) != 0) break;
+
+ /* Increment the hold. */
+ atomic_inc(&finger->hydro.hold);
+
+ /* Unlock the cell. */
+ if (lock_unlock(&finger->hydro.lock) != 0) error("Failed to unlock cell.");
+ }
+
+ /* If we reached the top of the tree, we're done. */
+ if (finger == NULL) {
+ TIMER_TOC(timer_locktree);
+ return 0;
+ }
+
+ /* Otherwise, we hit a snag. */
+ else {
+ /* Undo the holds up to finger. */
+ for (struct cell *finger2 = c->parent; finger2 != finger;
+ finger2 = finger2->parent)
+ atomic_dec(&finger2->hydro.hold);
+
+ /* Unlock this cell. */
+ if (lock_unlock(&c->hydro.lock) != 0) error("Failed to unlock cell.");
+
+ /* Admit defeat. */
+ TIMER_TOC(timer_locktree);
+ return 1;
+ }
+}
+
+/**
+ * @brief Lock a cell for access to its array of #gpart and hold its parents.
+ *
+ * @param c The #cell.
+ * @return 0 on success, 1 on failure
+ */
+int cell_glocktree(struct cell *c) {
+ TIMER_TIC;
+
+ /* First of all, try to lock this cell. */
+ if (c->grav.phold || lock_trylock(&c->grav.plock) != 0) {
+ TIMER_TOC(timer_locktree);
+ return 1;
+ }
+
+ /* Did somebody hold this cell in the meantime? */
+ if (c->grav.phold) {
+ /* Unlock this cell. */
+ if (lock_unlock(&c->grav.plock) != 0) error("Failed to unlock cell.");
+
+ /* Admit defeat. */
+ TIMER_TOC(timer_locktree);
+ return 1;
+ }
+
+ /* Climb up the tree and lock/hold/unlock. */
+ struct cell *finger;
+ for (finger = c->parent; finger != NULL; finger = finger->parent) {
+ /* Lock this cell. */
+ if (lock_trylock(&finger->grav.plock) != 0) break;
+
+ /* Increment the hold. */
+ atomic_inc(&finger->grav.phold);
+
+ /* Unlock the cell. */
+ if (lock_unlock(&finger->grav.plock) != 0) error("Failed to unlock cell.");
+ }
+
+ /* If we reached the top of the tree, we're done. */
+ if (finger == NULL) {
+ TIMER_TOC(timer_locktree);
+ return 0;
+ }
+
+ /* Otherwise, we hit a snag. */
+ else {
+ /* Undo the holds up to finger. */
+ for (struct cell *finger2 = c->parent; finger2 != finger;
+ finger2 = finger2->parent)
+ atomic_dec(&finger2->grav.phold);
+
+ /* Unlock this cell. */
+ if (lock_unlock(&c->grav.plock) != 0) error("Failed to unlock cell.");
+
+ /* Admit defeat. */
+ TIMER_TOC(timer_locktree);
+ return 1;
+ }
+}
+
+/**
+ * @brief Lock a cell for access to its #multipole and hold its parents.
+ *
+ * @param c The #cell.
+ * @return 0 on success, 1 on failure
+ */
+int cell_mlocktree(struct cell *c) {
+ TIMER_TIC;
+
+ /* First of all, try to lock this cell. */
+ if (c->grav.mhold || lock_trylock(&c->grav.mlock) != 0) {
+ TIMER_TOC(timer_locktree);
+ return 1;
+ }
+
+ /* Did somebody hold this cell in the meantime? */
+ if (c->grav.mhold) {
+ /* Unlock this cell. */
+ if (lock_unlock(&c->grav.mlock) != 0) error("Failed to unlock cell.");
+
+ /* Admit defeat. */
+ TIMER_TOC(timer_locktree);
+ return 1;
+ }
+
+ /* Climb up the tree and lock/hold/unlock. */
+ struct cell *finger;
+ for (finger = c->parent; finger != NULL; finger = finger->parent) {
+ /* Lock this cell. */
+ if (lock_trylock(&finger->grav.mlock) != 0) break;
+
+ /* Increment the hold. */
+ atomic_inc(&finger->grav.mhold);
+
+ /* Unlock the cell. */
+ if (lock_unlock(&finger->grav.mlock) != 0) error("Failed to unlock cell.");
+ }
+
+ /* If we reached the top of the tree, we're done. */
+ if (finger == NULL) {
+ TIMER_TOC(timer_locktree);
+ return 0;
+ }
+
+ /* Otherwise, we hit a snag. */
+ else {
+ /* Undo the holds up to finger. */
+ for (struct cell *finger2 = c->parent; finger2 != finger;
+ finger2 = finger2->parent)
+ atomic_dec(&finger2->grav.mhold);
+
+ /* Unlock this cell. */
+ if (lock_unlock(&c->grav.mlock) != 0) error("Failed to unlock cell.");
+
+ /* Admit defeat. */
+ TIMER_TOC(timer_locktree);
+ return 1;
+ }
+}
+
+/**
+ * @brief Lock a cell for access to its array of #spart and hold its parents.
+ *
+ * @param c The #cell.
+ * @return 0 on success, 1 on failure
+ */
+int cell_slocktree(struct cell *c) {
+ TIMER_TIC;
+
+ /* First of all, try to lock this cell. */
+ if (c->stars.hold || lock_trylock(&c->stars.lock) != 0) {
+ TIMER_TOC(timer_locktree);
+ return 1;
+ }
+
+ /* Did somebody hold this cell in the meantime? */
+ if (c->stars.hold) {
+ /* Unlock this cell. */
+ if (lock_unlock(&c->stars.lock) != 0) error("Failed to unlock cell.");
+
+ /* Admit defeat. */
+ TIMER_TOC(timer_locktree);
+ return 1;
+ }
+
+ /* Climb up the tree and lock/hold/unlock. */
+ struct cell *finger;
+ for (finger = c->parent; finger != NULL; finger = finger->parent) {
+ /* Lock this cell. */
+ if (lock_trylock(&finger->stars.lock) != 0) break;
+
+ /* Increment the hold. */
+ atomic_inc(&finger->stars.hold);
+
+ /* Unlock the cell. */
+ if (lock_unlock(&finger->stars.lock) != 0) error("Failed to unlock cell.");
+ }
+
+ /* If we reached the top of the tree, we're done. */
+ if (finger == NULL) {
+ TIMER_TOC(timer_locktree);
+ return 0;
+ }
+
+ /* Otherwise, we hit a snag. */
+ else {
+ /* Undo the holds up to finger. */
+ for (struct cell *finger2 = c->parent; finger2 != finger;
+ finger2 = finger2->parent)
+ atomic_dec(&finger2->stars.hold);
+
+ /* Unlock this cell. */
+ if (lock_unlock(&c->stars.lock) != 0) error("Failed to unlock cell.");
+
+ /* Admit defeat. */
+ TIMER_TOC(timer_locktree);
+ return 1;
+ }
+}
+
+/**
+ * @brief Lock a cell for access to its array of #sink and hold its parents.
+ *
+ * @param c The #cell.
+ * @return 0 on success, 1 on failure
+ */
+int cell_sink_locktree(struct cell *c) {
+ TIMER_TIC;
+
+ /* First of all, try to lock this cell. */
+ if (c->sinks.hold || lock_trylock(&c->sinks.lock) != 0) {
+ TIMER_TOC(timer_locktree);
+ return 1;
+ }
+
+ /* Did somebody hold this cell in the meantime? */
+ if (c->sinks.hold) {
+ /* Unlock this cell. */
+ if (lock_unlock(&c->sinks.lock) != 0) error("Failed to unlock cell.");
+
+ /* Admit defeat. */
+ TIMER_TOC(timer_locktree);
+ return 1;
+ }
+
+ /* Climb up the tree and lock/hold/unlock. */
+ struct cell *finger;
+ for (finger = c->parent; finger != NULL; finger = finger->parent) {
+ /* Lock this cell. */
+ if (lock_trylock(&finger->sinks.lock) != 0) break;
+
+ /* Increment the hold. */
+ atomic_inc(&finger->sinks.hold);
+
+ /* Unlock the cell. */
+ if (lock_unlock(&finger->sinks.lock) != 0) error("Failed to unlock cell.");
+ }
+
+ /* If we reached the top of the tree, we're done. */
+ if (finger == NULL) {
+ TIMER_TOC(timer_locktree);
+ return 0;
+ }
+
+ /* Otherwise, we hit a snag. */
+ else {
+ /* Undo the holds up to finger. */
+ for (struct cell *finger2 = c->parent; finger2 != finger;
+ finger2 = finger2->parent)
+ atomic_dec(&finger2->sinks.hold);
+
+ /* Unlock this cell. */
+ if (lock_unlock(&c->sinks.lock) != 0) error("Failed to unlock cell.");
+
+ /* Admit defeat. */
+ TIMER_TOC(timer_locktree);
+ return 1;
+ }
+}
+
+/**
+ * @brief Lock a cell for access to its array of #bpart and hold its parents.
+ *
+ * @param c The #cell.
+ * @return 0 on success, 1 on failure
+ */
+int cell_blocktree(struct cell *c) {
+ TIMER_TIC;
+
+ /* First of all, try to lock this cell. */
+ if (c->black_holes.hold || lock_trylock(&c->black_holes.lock) != 0) {
+ TIMER_TOC(timer_locktree);
+ return 1;
+ }
+
+ /* Did somebody hold this cell in the meantime? */
+ if (c->black_holes.hold) {
+ /* Unlock this cell. */
+ if (lock_unlock(&c->black_holes.lock) != 0) error("Failed to unlock cell.");
+
+ /* Admit defeat. */
+ TIMER_TOC(timer_locktree);
+ return 1;
+ }
+
+ /* Climb up the tree and lock/hold/unlock. */
+ struct cell *finger;
+ for (finger = c->parent; finger != NULL; finger = finger->parent) {
+ /* Lock this cell. */
+ if (lock_trylock(&finger->black_holes.lock) != 0) break;
+
+ /* Increment the hold. */
+ atomic_inc(&finger->black_holes.hold);
+
+ /* Unlock the cell. */
+ if (lock_unlock(&finger->black_holes.lock) != 0)
+ error("Failed to unlock cell.");
+ }
+
+ /* If we reached the top of the tree, we're done. */
+ if (finger == NULL) {
+ TIMER_TOC(timer_locktree);
+ return 0;
+ }
+
+ /* Otherwise, we hit a snag. */
+ else {
+ /* Undo the holds up to finger. */
+ for (struct cell *finger2 = c->parent; finger2 != finger;
+ finger2 = finger2->parent)
+ atomic_dec(&finger2->black_holes.hold);
+
+ /* Unlock this cell. */
+ if (lock_unlock(&c->black_holes.lock) != 0) error("Failed to unlock cell.");
+
+ /* Admit defeat. */
+ TIMER_TOC(timer_locktree);
+ return 1;
+ }
+}
+
+/**
+ * @brief Unlock a cell's parents for access to #part array.
+ *
+ * @param c The #cell.
+ */
+void cell_unlocktree(struct cell *c) {
+ TIMER_TIC;
+
+ /* First of all, try to unlock this cell. */
+ if (lock_unlock(&c->hydro.lock) != 0) error("Failed to unlock cell.");
+
+ /* Climb up the tree and unhold the parents. */
+ for (struct cell *finger = c->parent; finger != NULL; finger = finger->parent)
+ atomic_dec(&finger->hydro.hold);
+
+ TIMER_TOC(timer_locktree);
+}
+
+/**
+ * @brief Unlock a cell's parents for access to #gpart array.
+ *
+ * @param c The #cell.
+ */
+void cell_gunlocktree(struct cell *c) {
+ TIMER_TIC;
+
+ /* First of all, try to unlock this cell. */
+ if (lock_unlock(&c->grav.plock) != 0) error("Failed to unlock cell.");
+
+ /* Climb up the tree and unhold the parents. */
+ for (struct cell *finger = c->parent; finger != NULL; finger = finger->parent)
+ atomic_dec(&finger->grav.phold);
+
+ TIMER_TOC(timer_locktree);
+}
+
+/**
+ * @brief Unlock a cell's parents for access to its #multipole.
+ *
+ * @param c The #cell.
+ */
+void cell_munlocktree(struct cell *c) {
+ TIMER_TIC;
+
+ /* First of all, try to unlock this cell. */
+ if (lock_unlock(&c->grav.mlock) != 0) error("Failed to unlock cell.");
+
+ /* Climb up the tree and unhold the parents. */
+ for (struct cell *finger = c->parent; finger != NULL; finger = finger->parent)
+ atomic_dec(&finger->grav.mhold);
+
+ TIMER_TOC(timer_locktree);
+}
+
+/**
+ * @brief Unlock a cell's parents for access to #spart array.
+ *
+ * @param c The #cell.
+ */
+void cell_sunlocktree(struct cell *c) {
+ TIMER_TIC;
+
+ /* First of all, try to unlock this cell. */
+ if (lock_unlock(&c->stars.lock) != 0) error("Failed to unlock cell.");
+
+ /* Climb up the tree and unhold the parents. */
+ for (struct cell *finger = c->parent; finger != NULL; finger = finger->parent)
+ atomic_dec(&finger->stars.hold);
+
+ TIMER_TOC(timer_locktree);
+}
+
+/**
+ * @brief Unlock a cell's parents for access to #sink array.
+ *
+ * @param c The #cell.
+ */
+void cell_sink_unlocktree(struct cell *c) {
+ TIMER_TIC;
+
+ /* First of all, try to unlock this cell. */
+ if (lock_unlock(&c->sinks.lock) != 0) error("Failed to unlock cell.");
+
+ /* Climb up the tree and unhold the parents. */
+ for (struct cell *finger = c->parent; finger != NULL; finger = finger->parent)
+ atomic_dec(&finger->sinks.hold);
+
+ TIMER_TOC(timer_locktree);
+}
+
+/**
+ * @brief Unlock a cell's parents for access to #bpart array.
+ *
+ * @param c The #cell.
+ */
+void cell_bunlocktree(struct cell *c) {
+ TIMER_TIC;
+
+ /* First of all, try to unlock this cell. */
+ if (lock_unlock(&c->black_holes.lock) != 0) error("Failed to unlock cell.");
+
+ /* Climb up the tree and unhold the parents. */
+ for (struct cell *finger = c->parent; finger != NULL; finger = finger->parent)
+ atomic_dec(&finger->black_holes.hold);
+
+ TIMER_TOC(timer_locktree);
+}
diff --git a/src/cell_pack.c b/src/cell_pack.c
new file mode 100644
index 0000000000000000000000000000000000000000..df45f4e3e752a7d219b3d8b0126c8650aa8c7cf7
--- /dev/null
+++ b/src/cell_pack.c
@@ -0,0 +1,768 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* This object's header. */
+#include "cell.h"
+
+/**
+ * @brief Pack the data of the given cell and all it's sub-cells.
+ *
+ * @param c The #cell.
+ * @param pc Pointer to an array of packed cells in which the
+ * cells will be packed.
+ * @param with_gravity Are we running with gravity and hence need
+ * to exchange multipoles?
+ *
+ * @return The number of packed cells.
+ */
+int cell_pack(struct cell *restrict c, struct pcell *restrict pc,
+ const int with_gravity) {
+#ifdef WITH_MPI
+
+ /* Start by packing the data of the current cell. */
+ pc->hydro.h_max = c->hydro.h_max;
+ pc->stars.h_max = c->stars.h_max;
+ pc->black_holes.h_max = c->black_holes.h_max;
+ pc->sinks.r_cut_max = c->sinks.r_cut_max;
+
+ pc->hydro.ti_end_min = c->hydro.ti_end_min;
+ pc->hydro.ti_end_max = c->hydro.ti_end_max;
+ pc->grav.ti_end_min = c->grav.ti_end_min;
+ pc->grav.ti_end_max = c->grav.ti_end_max;
+ pc->stars.ti_end_min = c->stars.ti_end_min;
+ pc->stars.ti_end_max = c->stars.ti_end_max;
+ pc->sinks.ti_end_min = c->sinks.ti_end_min;
+ pc->sinks.ti_end_max = c->sinks.ti_end_max;
+ pc->black_holes.ti_end_min = c->black_holes.ti_end_min;
+ pc->black_holes.ti_end_max = c->black_holes.ti_end_max;
+
+ pc->hydro.ti_old_part = c->hydro.ti_old_part;
+ pc->grav.ti_old_part = c->grav.ti_old_part;
+ pc->grav.ti_old_multipole = c->grav.ti_old_multipole;
+ pc->stars.ti_old_part = c->stars.ti_old_part;
+ pc->black_holes.ti_old_part = c->black_holes.ti_old_part;
+ pc->sinks.ti_old_part = c->sinks.ti_old_part;
+
+ pc->hydro.count = c->hydro.count;
+ pc->grav.count = c->grav.count;
+ pc->stars.count = c->stars.count;
+ pc->sinks.count = c->sinks.count;
+ pc->black_holes.count = c->black_holes.count;
+ pc->maxdepth = c->maxdepth;
+
+ /* Copy the Multipole related information */
+ if (with_gravity) {
+ const struct gravity_tensors *mp = c->grav.multipole;
+
+ pc->grav.m_pole = mp->m_pole;
+ pc->grav.CoM[0] = mp->CoM[0];
+ pc->grav.CoM[1] = mp->CoM[1];
+ pc->grav.CoM[2] = mp->CoM[2];
+ pc->grav.CoM_rebuild[0] = mp->CoM_rebuild[0];
+ pc->grav.CoM_rebuild[1] = mp->CoM_rebuild[1];
+ pc->grav.CoM_rebuild[2] = mp->CoM_rebuild[2];
+ pc->grav.r_max = mp->r_max;
+ pc->grav.r_max_rebuild = mp->r_max_rebuild;
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ pc->cellID = c->cellID;
+#endif
+
+ /* Fill in the progeny, depth-first recursion. */
+ int count = 1;
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL) {
+ pc->progeny[k] = count;
+ count += cell_pack(c->progeny[k], &pc[count], with_gravity);
+ } else {
+ pc->progeny[k] = -1;
+ }
+
+ /* Return the number of packed cells used. */
+ c->mpi.pcell_size = count;
+ return count;
+
+#else
+ error("SWIFT was not compiled with MPI support.");
+ return 0;
+#endif
+}
+
+/**
+ * @brief Pack the tag of the given cell and all it's sub-cells.
+ *
+ * @param c The #cell.
+ * @param tags Pointer to an array of packed tags.
+ *
+ * @return The number of packed tags.
+ */
+int cell_pack_tags(const struct cell *c, int *tags) {
+#ifdef WITH_MPI
+
+ /* Start by packing the data of the current cell. */
+ tags[0] = c->mpi.tag;
+
+ /* Fill in the progeny, depth-first recursion. */
+ int count = 1;
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL)
+ count += cell_pack_tags(c->progeny[k], &tags[count]);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->mpi.pcell_size != count) error("Inconsistent tag and pcell count!");
+#endif // SWIFT_DEBUG_CHECKS
+
+ /* Return the number of packed tags used. */
+ return count;
+
+#else
+ error("SWIFT was not compiled with MPI support.");
+ return 0;
+#endif
+}
+
+void cell_pack_part_swallow(const struct cell *c,
+ struct black_holes_part_data *data) {
+
+ const size_t count = c->hydro.count;
+ const struct part *parts = c->hydro.parts;
+
+ for (size_t i = 0; i < count; ++i) {
+ data[i] = parts[i].black_holes_data;
+ }
+}
+
+void cell_unpack_part_swallow(struct cell *c,
+ const struct black_holes_part_data *data) {
+
+ const size_t count = c->hydro.count;
+ struct part *parts = c->hydro.parts;
+
+ for (size_t i = 0; i < count; ++i) {
+ parts[i].black_holes_data = data[i];
+ }
+}
+
+void cell_pack_bpart_swallow(const struct cell *c,
+ struct black_holes_bpart_data *data) {
+
+ const size_t count = c->black_holes.count;
+ const struct bpart *bparts = c->black_holes.parts;
+
+ for (size_t i = 0; i < count; ++i) {
+ data[i] = bparts[i].merger_data;
+ }
+}
+
+void cell_unpack_bpart_swallow(struct cell *c,
+ const struct black_holes_bpart_data *data) {
+
+ const size_t count = c->black_holes.count;
+ struct bpart *bparts = c->black_holes.parts;
+
+ for (size_t i = 0; i < count; ++i) {
+ bparts[i].merger_data = data[i];
+ }
+}
+
+/**
+ * @brief Unpack the data of a given cell and its sub-cells.
+ *
+ * @param pc An array of packed #pcell.
+ * @param c The #cell in which to unpack the #pcell.
+ * @param s The #space in which the cells are created.
+ * @param with_gravity Are we running with gravity and hence need
+ * to exchange multipoles?
+ *
+ * @return The number of cells created.
+ */
+int cell_unpack(struct pcell *restrict pc, struct cell *restrict c,
+ struct space *restrict s, const int with_gravity) {
+#ifdef WITH_MPI
+
+ /* Unpack the current pcell. */
+ c->hydro.h_max = pc->hydro.h_max;
+ c->stars.h_max = pc->stars.h_max;
+ c->black_holes.h_max = pc->black_holes.h_max;
+ c->sinks.r_cut_max = pc->sinks.r_cut_max;
+
+ c->hydro.ti_end_min = pc->hydro.ti_end_min;
+ c->hydro.ti_end_max = pc->hydro.ti_end_max;
+ c->grav.ti_end_min = pc->grav.ti_end_min;
+ c->grav.ti_end_max = pc->grav.ti_end_max;
+ c->stars.ti_end_min = pc->stars.ti_end_min;
+ c->stars.ti_end_max = pc->stars.ti_end_max;
+ c->black_holes.ti_end_min = pc->black_holes.ti_end_min;
+ c->black_holes.ti_end_max = pc->black_holes.ti_end_max;
+ c->sinks.ti_end_min = pc->sinks.ti_end_min;
+ c->sinks.ti_end_max = pc->sinks.ti_end_max;
+
+ c->hydro.ti_old_part = pc->hydro.ti_old_part;
+ c->grav.ti_old_part = pc->grav.ti_old_part;
+ c->grav.ti_old_multipole = pc->grav.ti_old_multipole;
+ c->stars.ti_old_part = pc->stars.ti_old_part;
+ c->black_holes.ti_old_part = pc->black_holes.ti_old_part;
+ c->sinks.ti_old_part = pc->sinks.ti_old_part;
+
+ c->hydro.count = pc->hydro.count;
+ c->grav.count = pc->grav.count;
+ c->stars.count = pc->stars.count;
+ c->sinks.count = pc->sinks.count;
+ c->black_holes.count = pc->black_holes.count;
+ c->maxdepth = pc->maxdepth;
+
+#ifdef SWIFT_DEBUG_CHECKS
+ c->cellID = pc->cellID;
+#endif
+
+ /* Copy the Multipole related information */
+ if (with_gravity) {
+ struct gravity_tensors *mp = c->grav.multipole;
+
+ mp->m_pole = pc->grav.m_pole;
+ mp->CoM[0] = pc->grav.CoM[0];
+ mp->CoM[1] = pc->grav.CoM[1];
+ mp->CoM[2] = pc->grav.CoM[2];
+ mp->CoM_rebuild[0] = pc->grav.CoM_rebuild[0];
+ mp->CoM_rebuild[1] = pc->grav.CoM_rebuild[1];
+ mp->CoM_rebuild[2] = pc->grav.CoM_rebuild[2];
+ mp->r_max = pc->grav.r_max;
+ mp->r_max_rebuild = pc->grav.r_max_rebuild;
+ }
+
+ /* Number of new cells created. */
+ int count = 1;
+
+ /* Fill the progeny recursively, depth-first. */
+ c->split = 0;
+ for (int k = 0; k < 8; k++)
+ if (pc->progeny[k] >= 0) {
+ struct cell *temp;
+ space_getcells(s, 1, &temp);
+ temp->hydro.count = 0;
+ temp->grav.count = 0;
+ temp->stars.count = 0;
+ temp->loc[0] = c->loc[0];
+ temp->loc[1] = c->loc[1];
+ temp->loc[2] = c->loc[2];
+ temp->width[0] = c->width[0] / 2;
+ temp->width[1] = c->width[1] / 2;
+ temp->width[2] = c->width[2] / 2;
+ temp->dmin = c->dmin / 2;
+ if (k & 4) temp->loc[0] += temp->width[0];
+ if (k & 2) temp->loc[1] += temp->width[1];
+ if (k & 1) temp->loc[2] += temp->width[2];
+ temp->depth = c->depth + 1;
+ temp->split = 0;
+ temp->hydro.dx_max_part = 0.f;
+ temp->hydro.dx_max_sort = 0.f;
+ temp->stars.dx_max_part = 0.f;
+ temp->stars.dx_max_sort = 0.f;
+ temp->black_holes.dx_max_part = 0.f;
+ temp->nodeID = c->nodeID;
+ temp->parent = c;
+ c->progeny[k] = temp;
+ c->split = 1;
+ count += cell_unpack(&pc[pc->progeny[k]], temp, s, with_gravity);
+ }
+
+ /* Return the total number of unpacked cells. */
+ c->mpi.pcell_size = count;
+ return count;
+
+#else
+ error("SWIFT was not compiled with MPI support.");
+ return 0;
+#endif
+}
+
+/**
+ * @brief Unpack the tags of a given cell and its sub-cells.
+ *
+ * @param tags An array of tags.
+ * @param c The #cell in which to unpack the tags.
+ *
+ * @return The number of tags created.
+ */
+int cell_unpack_tags(const int *tags, struct cell *restrict c) {
+#ifdef WITH_MPI
+
+ /* Unpack the current pcell. */
+ c->mpi.tag = tags[0];
+
+ /* Number of new cells created. */
+ int count = 1;
+
+ /* Fill the progeny recursively, depth-first. */
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL) {
+ count += cell_unpack_tags(&tags[count], c->progeny[k]);
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->mpi.pcell_size != count) error("Inconsistent tag and pcell count!");
+#endif // SWIFT_DEBUG_CHECKS
+
+ /* Return the total number of unpacked tags. */
+ return count;
+
+#else
+ error("SWIFT was not compiled with MPI support.");
+ return 0;
+#endif
+}
+
+/**
+ * @brief Pack the time information of the given cell and all it's sub-cells.
+ *
+ * @param c The #cell.
+ * @param pcells (output) The end-of-timestep information we pack into
+ *
+ * @return The number of packed cells.
+ */
+int cell_pack_end_step_hydro(struct cell *restrict c,
+ struct pcell_step_hydro *restrict pcells) {
+#ifdef WITH_MPI
+
+ /* Pack this cell's data. */
+ pcells[0].ti_end_min = c->hydro.ti_end_min;
+ pcells[0].ti_end_max = c->hydro.ti_end_max;
+ pcells[0].dx_max_part = c->hydro.dx_max_part;
+
+ /* Fill in the progeny, depth-first recursion. */
+ int count = 1;
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL) {
+ count += cell_pack_end_step_hydro(c->progeny[k], &pcells[count]);
+ }
+
+ /* Return the number of packed values. */
+ return count;
+
+#else
+ error("SWIFT was not compiled with MPI support.");
+ return 0;
+#endif
+}
+
+/**
+ * @brief Unpack the time information of a given cell and its sub-cells.
+ *
+ * @param c The #cell
+ * @param pcells The end-of-timestep information to unpack
+ *
+ * @return The number of cells created.
+ */
+int cell_unpack_end_step_hydro(struct cell *restrict c,
+ struct pcell_step_hydro *restrict pcells) {
+#ifdef WITH_MPI
+
+ /* Unpack this cell's data. */
+ c->hydro.ti_end_min = pcells[0].ti_end_min;
+ c->hydro.ti_end_max = pcells[0].ti_end_max;
+ c->hydro.dx_max_part = pcells[0].dx_max_part;
+
+ /* Fill in the progeny, depth-first recursion. */
+ int count = 1;
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL) {
+ count += cell_unpack_end_step_hydro(c->progeny[k], &pcells[count]);
+ }
+
+ /* Return the number of packed values. */
+ return count;
+
+#else
+ error("SWIFT was not compiled with MPI support.");
+ return 0;
+#endif
+}
+
+/**
+ * @brief Pack the time information of the given cell and all it's sub-cells.
+ *
+ * @param c The #cell.
+ * @param pcells (output) The end-of-timestep information we pack into
+ *
+ * @return The number of packed cells.
+ */
+int cell_pack_end_step_grav(struct cell *restrict c,
+ struct pcell_step_grav *restrict pcells) {
+#ifdef WITH_MPI
+
+ /* Pack this cell's data. */
+ pcells[0].ti_end_min = c->grav.ti_end_min;
+ pcells[0].ti_end_max = c->grav.ti_end_max;
+
+ /* Fill in the progeny, depth-first recursion. */
+ int count = 1;
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL) {
+ count += cell_pack_end_step_grav(c->progeny[k], &pcells[count]);
+ }
+
+ /* Return the number of packed values. */
+ return count;
+
+#else
+ error("SWIFT was not compiled with MPI support.");
+ return 0;
+#endif
+}
+
+/**
+ * @brief Unpack the time information of a given cell and its sub-cells.
+ *
+ * @param c The #cell
+ * @param pcells The end-of-timestep information to unpack
+ *
+ * @return The number of cells created.
+ */
+int cell_unpack_end_step_grav(struct cell *restrict c,
+ struct pcell_step_grav *restrict pcells) {
+#ifdef WITH_MPI
+
+ /* Unpack this cell's data. */
+ c->grav.ti_end_min = pcells[0].ti_end_min;
+ c->grav.ti_end_max = pcells[0].ti_end_max;
+
+ /* Fill in the progeny, depth-first recursion. */
+ int count = 1;
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL) {
+ count += cell_unpack_end_step_grav(c->progeny[k], &pcells[count]);
+ }
+
+ /* Return the number of packed values. */
+ return count;
+
+#else
+ error("SWIFT was not compiled with MPI support.");
+ return 0;
+#endif
+}
+
+/**
+ * @brief Pack the time information of the given cell and all it's sub-cells.
+ *
+ * @param c The #cell.
+ * @param pcells (output) The end-of-timestep information we pack into
+ *
+ * @return The number of packed cells.
+ */
+int cell_pack_end_step_stars(struct cell *restrict c,
+ struct pcell_step_stars *restrict pcells) {
+#ifdef WITH_MPI
+
+ /* Pack this cell's data. */
+ pcells[0].ti_end_min = c->stars.ti_end_min;
+ pcells[0].ti_end_max = c->stars.ti_end_max;
+ pcells[0].dx_max_part = c->stars.dx_max_part;
+
+ /* Fill in the progeny, depth-first recursion. */
+ int count = 1;
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL) {
+ count += cell_pack_end_step_stars(c->progeny[k], &pcells[count]);
+ }
+
+ /* Return the number of packed values. */
+ return count;
+
+#else
+ error("SWIFT was not compiled with MPI support.");
+ return 0;
+#endif
+}
+
+/**
+ * @brief Unpack the time information of a given cell and its sub-cells.
+ *
+ * @param c The #cell
+ * @param pcells The end-of-timestep information to unpack
+ *
+ * @return The number of cells created.
+ */
+int cell_unpack_end_step_stars(struct cell *restrict c,
+ struct pcell_step_stars *restrict pcells) {
+#ifdef WITH_MPI
+
+ /* Unpack this cell's data. */
+ c->stars.ti_end_min = pcells[0].ti_end_min;
+ c->stars.ti_end_max = pcells[0].ti_end_max;
+ c->stars.dx_max_part = pcells[0].dx_max_part;
+
+ /* Fill in the progeny, depth-first recursion. */
+ int count = 1;
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL) {
+ count += cell_unpack_end_step_stars(c->progeny[k], &pcells[count]);
+ }
+
+ /* Return the number of packed values. */
+ return count;
+
+#else
+ error("SWIFT was not compiled with MPI support.");
+ return 0;
+#endif
+}
+
+/**
+ * @brief Pack the time information of the given cell and all it's sub-cells.
+ *
+ * @param c The #cell.
+ * @param pcells (output) The end-of-timestep information we pack into
+ *
+ * @return The number of packed cells.
+ */
+int cell_pack_end_step_black_holes(
+ struct cell *restrict c, struct pcell_step_black_holes *restrict pcells) {
+
+#ifdef WITH_MPI
+
+ /* Pack this cell's data. */
+ pcells[0].ti_end_min = c->black_holes.ti_end_min;
+ pcells[0].ti_end_max = c->black_holes.ti_end_max;
+ pcells[0].dx_max_part = c->black_holes.dx_max_part;
+
+ /* Fill in the progeny, depth-first recursion. */
+ int count = 1;
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL) {
+ count += cell_pack_end_step_black_holes(c->progeny[k], &pcells[count]);
+ }
+
+ /* Return the number of packed values. */
+ return count;
+
+#else
+ error("SWIFT was not compiled with MPI support.");
+ return 0;
+#endif
+}
+
+/**
+ * @brief Unpack the time information of a given cell and its sub-cells.
+ *
+ * @param c The #cell
+ * @param pcells The end-of-timestep information to unpack
+ *
+ * @return The number of cells created.
+ */
+int cell_unpack_end_step_black_holes(
+ struct cell *restrict c, struct pcell_step_black_holes *restrict pcells) {
+
+#ifdef WITH_MPI
+
+ /* Unpack this cell's data. */
+ c->black_holes.ti_end_min = pcells[0].ti_end_min;
+ c->black_holes.ti_end_max = pcells[0].ti_end_max;
+ c->black_holes.dx_max_part = pcells[0].dx_max_part;
+
+ /* Fill in the progeny, depth-first recursion. */
+ int count = 1;
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL) {
+ count += cell_unpack_end_step_black_holes(c->progeny[k], &pcells[count]);
+ }
+
+ /* Return the number of packed values. */
+ return count;
+
+#else
+ error("SWIFT was not compiled with MPI support.");
+ return 0;
+#endif
+}
+
+/**
+ * @brief Pack the multipole information of the given cell and all it's
+ * sub-cells.
+ *
+ * @param c The #cell.
+ * @param pcells (output) The multipole information we pack into
+ *
+ * @return The number of packed cells.
+ */
+int cell_pack_multipoles(struct cell *restrict c,
+ struct gravity_tensors *restrict pcells) {
+#ifdef WITH_MPI
+
+ /* Pack this cell's data. */
+ pcells[0] = *c->grav.multipole;
+
+ /* Fill in the progeny, depth-first recursion. */
+ int count = 1;
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL) {
+ count += cell_pack_multipoles(c->progeny[k], &pcells[count]);
+ }
+
+ /* Return the number of packed values. */
+ return count;
+
+#else
+ error("SWIFT was not compiled with MPI support.");
+ return 0;
+#endif
+}
+
+/**
+ * @brief Unpack the multipole information of a given cell and its sub-cells.
+ *
+ * @param c The #cell
+ * @param pcells The multipole information to unpack
+ *
+ * @return The number of cells created.
+ */
+int cell_unpack_multipoles(struct cell *restrict c,
+ struct gravity_tensors *restrict pcells) {
+#ifdef WITH_MPI
+
+ /* Unpack this cell's data. */
+ *c->grav.multipole = pcells[0];
+
+ /* Fill in the progeny, depth-first recursion. */
+ int count = 1;
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL) {
+ count += cell_unpack_multipoles(c->progeny[k], &pcells[count]);
+ }
+
+ /* Return the number of packed values. */
+ return count;
+
+#else
+ error("SWIFT was not compiled with MPI support.");
+ return 0;
+#endif
+}
+
+/**
+ * @brief Pack the counts for star formation of the given cell and all it's
+ * sub-cells.
+ *
+ * @param c The #cell.
+ * @param pcells (output) The multipole information we pack into
+ *
+ * @return The number of packed cells.
+ */
+int cell_pack_sf_counts(struct cell *restrict c,
+ struct pcell_sf *restrict pcells) {
+
+#ifdef WITH_MPI
+
+ /* Pack this cell's data. */
+ pcells[0].stars.delta_from_rebuild = c->stars.parts - c->stars.parts_rebuild;
+ pcells[0].stars.count = c->stars.count;
+ pcells[0].stars.dx_max_part = c->stars.dx_max_part;
+
+ /* Pack this cell's data. */
+ pcells[0].grav.delta_from_rebuild = c->grav.parts - c->grav.parts_rebuild;
+ pcells[0].grav.count = c->grav.count;
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Stars */
+ if (c->stars.parts_rebuild == NULL)
+ error("Star particles array at rebuild is NULL! c->depth=%d", c->depth);
+
+ if (pcells[0].stars.delta_from_rebuild < 0)
+ error("Stars part pointer moved in the wrong direction!");
+
+ if (pcells[0].stars.delta_from_rebuild > 0 && c->depth == 0)
+ error("Shifting the top-level pointer is not allowed!");
+
+ /* Grav */
+ if (c->grav.parts_rebuild == NULL)
+ error("Grav. particles array at rebuild is NULL! c->depth=%d", c->depth);
+
+ if (pcells[0].grav.delta_from_rebuild < 0)
+ error("Grav part pointer moved in the wrong direction!");
+
+ if (pcells[0].grav.delta_from_rebuild > 0 && c->depth == 0)
+ error("Shifting the top-level pointer is not allowed!");
+#endif
+
+ /* Fill in the progeny, depth-first recursion. */
+ int count = 1;
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL) {
+ count += cell_pack_sf_counts(c->progeny[k], &pcells[count]);
+ }
+
+ /* Return the number of packed values. */
+ return count;
+
+#else
+ error("SWIFT was not compiled with MPI support.");
+ return 0;
+#endif
+}
+
+/**
+ * @brief Unpack the counts for star formation of a given cell and its
+ * sub-cells.
+ *
+ * @param c The #cell
+ * @param pcells The multipole information to unpack
+ *
+ * @return The number of cells created.
+ */
+int cell_unpack_sf_counts(struct cell *restrict c,
+ struct pcell_sf *restrict pcells) {
+
+#ifdef WITH_MPI
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->stars.parts_rebuild == NULL)
+ error("Star particles array at rebuild is NULL!");
+ if (c->grav.parts_rebuild == NULL)
+ error("Grav particles array at rebuild is NULL!");
+#endif
+
+ /* Unpack this cell's data. */
+ c->stars.count = pcells[0].stars.count;
+ c->stars.parts = c->stars.parts_rebuild + pcells[0].stars.delta_from_rebuild;
+ c->stars.dx_max_part = pcells[0].stars.dx_max_part;
+
+ c->grav.count = pcells[0].grav.count;
+ c->grav.parts = c->grav.parts_rebuild + pcells[0].grav.delta_from_rebuild;
+
+ /* Fill in the progeny, depth-first recursion. */
+ int count = 1;
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL) {
+ count += cell_unpack_sf_counts(c->progeny[k], &pcells[count]);
+ }
+
+ /* Return the number of packed values. */
+ return count;
+
+#else
+ error("SWIFT was not compiled with MPI support.");
+ return 0;
+#endif
+}
diff --git a/src/cell_sinks.h b/src/cell_sinks.h
new file mode 100644
index 0000000000000000000000000000000000000000..2304cd1eefd71aca189b397f2bca446435b23dcd
--- /dev/null
+++ b/src/cell_sinks.h
@@ -0,0 +1,97 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+#ifndef SWIFT_CELL_SINKS_H
+#define SWIFT_CELL_SINKS_H
+
+/* Config parameters. */
+#include "../config.h"
+
+/* Local includes. */
+#include "lock.h"
+#include "timeline.h"
+
+/**
+ * @brief Sinks-related cell variables.
+ */
+struct cell_sinks {
+
+ /*! Pointer to the #sink data. */
+ struct sink *parts;
+
+ /*! Linked list of the tasks computing this cell's sink formation checks. */
+ struct link *compute_formation;
+
+ /*! Nr of #sink in this cell. */
+ int count;
+
+ /*! Nr of #sink this cell can hold after addition of new one. */
+ int count_total;
+
+ /*! Max cut off radius in this cell. */
+ float r_cut_max;
+
+ /*! Values of r_cut_max before the drifts, used for sub-cell tasks. */
+ float r_cut_max_old;
+
+ /*! Number of #sink updated in this cell. */
+ int updated;
+
+ /*! Is the #sink data of this cell being used in a sub-cell? */
+ int hold;
+
+ /*! Spin lock for various uses (#sink case). */
+ swift_lock_type lock;
+
+ /*! Spin lock for sink formation use. */
+ swift_lock_type sink_formation_lock;
+
+ /*! Maximum part movement in this cell since last construction. */
+ float dx_max_part;
+
+ /*! Values of dx_max before the drifts, used for sub-cell tasks. */
+ float dx_max_part_old;
+
+ /*! Last (integer) time the cell's sink were drifted forward in time. */
+ integertime_t ti_old_part;
+
+ /*! Minimum end of (integer) time step in this cell for sink tasks. */
+ integertime_t ti_end_min;
+
+ /*! Maximum end of (integer) time step in this cell for sink tasks. */
+ integertime_t ti_end_max;
+
+ /*! Maximum beginning of (integer) time step in this cell for sink
+ * tasks.
+ */
+ integertime_t ti_beg_max;
+
+ /*! The drift task for sinks */
+ struct task *drift;
+
+ /*! Implicit tasks marking the entry of the sink block of tasks
+ */
+ struct task *sink_in;
+
+ /*! Implicit tasks marking the exit of the sink block of tasks */
+ struct task *sink_out;
+};
+
+#endif /* SWIFT_CELL_SINKS_H */
diff --git a/src/cell_split.c b/src/cell_split.c
new file mode 100644
index 0000000000000000000000000000000000000000..64d4d0ad6ce7b6a763391d88cf8c4209d9b74f13
--- /dev/null
+++ b/src/cell_split.c
@@ -0,0 +1,682 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* This object's header. */
+#include "cell.h"
+
+/* Local headers. */
+#include "memswap.h"
+
+/**
+ * @brief Sort the parts into eight bins along the given pivots.
+ *
+ * @param c The #cell array to be sorted.
+ * @param parts_offset Offset of the cell parts array relative to the
+ * space's parts array, i.e. c->hydro.parts - s->parts.
+ * @param sparts_offset Offset of the cell sparts array relative to the
+ * space's sparts array, i.e. c->stars.parts - s->stars.parts.
+ * @param bparts_offset Offset of the cell bparts array relative to the
+ * space's bparts array, i.e. c->black_holes.parts -
+ * s->black_holes.parts.
+ * @param sinks_offset Offset of the cell sink array relative to the
+ * space's sink array, i.e. c->sinks.parts - s->sinks.parts.
+ * @param buff A buffer with at least max(c->hydro.count, c->grav.count)
+ * entries, used for sorting indices.
+ * @param sbuff A buffer with at least max(c->stars.count, c->grav.count)
+ * entries, used for sorting indices for the sparts.
+ * @param bbuff A buffer with at least max(c->black_holes.count, c->grav.count)
+ * entries, used for sorting indices for the sparts.
+ * @param gbuff A buffer with at least max(c->hydro.count, c->grav.count)
+ * entries, used for sorting indices for the gparts.
+ * @param sinkbuff A buffer with at least max(c->sinks.count, c->grav.count)
+ * entries, used for sorting indices for the sinks.
+ */
+void cell_split(struct cell *c, const ptrdiff_t parts_offset,
+ const ptrdiff_t sparts_offset, const ptrdiff_t bparts_offset,
+ const ptrdiff_t sinks_offset, struct cell_buff *restrict buff,
+ struct cell_buff *restrict sbuff,
+ struct cell_buff *restrict bbuff,
+ struct cell_buff *restrict gbuff,
+ struct cell_buff *restrict sinkbuff) {
+
+ const int count = c->hydro.count, gcount = c->grav.count,
+ scount = c->stars.count, bcount = c->black_holes.count,
+ sink_count = c->sinks.count;
+ struct part *parts = c->hydro.parts;
+ struct xpart *xparts = c->hydro.xparts;
+ struct gpart *gparts = c->grav.parts;
+ struct spart *sparts = c->stars.parts;
+ struct bpart *bparts = c->black_holes.parts;
+ struct sink *sinks = c->sinks.parts;
+ const double pivot[3] = {c->loc[0] + c->width[0] / 2,
+ c->loc[1] + c->width[1] / 2,
+ c->loc[2] + c->width[2] / 2};
+ int bucket_count[8] = {0, 0, 0, 0, 0, 0, 0, 0};
+ int bucket_offset[9];
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Check that the buffs are OK. */
+ for (int k = 0; k < count; k++) {
+ if (buff[k].x[0] != parts[k].x[0] || buff[k].x[1] != parts[k].x[1] ||
+ buff[k].x[2] != parts[k].x[2])
+ error("Inconsistent buff contents.");
+ }
+ for (int k = 0; k < gcount; k++) {
+ if (gbuff[k].x[0] != gparts[k].x[0] || gbuff[k].x[1] != gparts[k].x[1] ||
+ gbuff[k].x[2] != gparts[k].x[2])
+ error("Inconsistent gbuff contents.");
+ }
+ for (int k = 0; k < scount; k++) {
+ if (sbuff[k].x[0] != sparts[k].x[0] || sbuff[k].x[1] != sparts[k].x[1] ||
+ sbuff[k].x[2] != sparts[k].x[2])
+ error("Inconsistent sbuff contents.");
+ }
+ for (int k = 0; k < bcount; k++) {
+ if (bbuff[k].x[0] != bparts[k].x[0] || bbuff[k].x[1] != bparts[k].x[1] ||
+ bbuff[k].x[2] != bparts[k].x[2])
+ error("Inconsistent bbuff contents.");
+ }
+ for (int k = 0; k < sink_count; k++) {
+ if (sinkbuff[k].x[0] != sinks[k].x[0] ||
+ sinkbuff[k].x[1] != sinks[k].x[1] || sinkbuff[k].x[2] != sinks[k].x[2])
+ error("Inconsistent sinkbuff contents.");
+ }
+#endif /* SWIFT_DEBUG_CHECKS */
+
+ /* Fill the buffer with the indices. */
+ for (int k = 0; k < count; k++) {
+ const int bid = (buff[k].x[0] >= pivot[0]) * 4 +
+ (buff[k].x[1] >= pivot[1]) * 2 + (buff[k].x[2] >= pivot[2]);
+ bucket_count[bid]++;
+ buff[k].ind = bid;
+ }
+
+ /* Set the buffer offsets. */
+ bucket_offset[0] = 0;
+ for (int k = 1; k <= 8; k++) {
+ bucket_offset[k] = bucket_offset[k - 1] + bucket_count[k - 1];
+ bucket_count[k - 1] = 0;
+ }
+
+ /* Run through the buckets, and swap particles to their correct spot. */
+ for (int bucket = 0; bucket < 8; bucket++) {
+ for (int k = bucket_offset[bucket] + bucket_count[bucket];
+ k < bucket_offset[bucket + 1]; k++) {
+ int bid = buff[k].ind;
+ if (bid != bucket) {
+ struct part part = parts[k];
+ struct xpart xpart = xparts[k];
+ struct cell_buff temp_buff = buff[k];
+ while (bid != bucket) {
+ int j = bucket_offset[bid] + bucket_count[bid]++;
+ while (buff[j].ind == bid) {
+ j++;
+ bucket_count[bid]++;
+ }
+ memswap(&parts[j], &part, sizeof(struct part));
+ memswap(&xparts[j], &xpart, sizeof(struct xpart));
+ memswap(&buff[j], &temp_buff, sizeof(struct cell_buff));
+ if (parts[j].gpart)
+ parts[j].gpart->id_or_neg_offset = -(j + parts_offset);
+ bid = temp_buff.ind;
+ }
+ parts[k] = part;
+ xparts[k] = xpart;
+ buff[k] = temp_buff;
+ if (parts[k].gpart)
+ parts[k].gpart->id_or_neg_offset = -(k + parts_offset);
+ }
+ bucket_count[bid]++;
+ }
+ }
+
+ /* Store the counts and offsets. */
+ for (int k = 0; k < 8; k++) {
+ c->progeny[k]->hydro.count = bucket_count[k];
+ c->progeny[k]->hydro.count_total = c->progeny[k]->hydro.count;
+ c->progeny[k]->hydro.parts = &c->hydro.parts[bucket_offset[k]];
+ c->progeny[k]->hydro.xparts = &c->hydro.xparts[bucket_offset[k]];
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Check that the buffs are OK. */
+ for (int k = 1; k < count; k++) {
+ if (buff[k].ind < buff[k - 1].ind) error("Buff not sorted.");
+ if (buff[k].x[0] != parts[k].x[0] || buff[k].x[1] != parts[k].x[1] ||
+ buff[k].x[2] != parts[k].x[2])
+ error("Inconsistent buff contents (k=%i).", k);
+ }
+
+ /* Verify that _all_ the parts have been assigned to a cell. */
+ for (int k = 1; k < 8; k++)
+ if (&c->progeny[k - 1]->hydro.parts[c->progeny[k - 1]->hydro.count] !=
+ c->progeny[k]->hydro.parts)
+ error("Particle sorting failed (internal consistency).");
+ if (c->progeny[0]->hydro.parts != c->hydro.parts)
+ error("Particle sorting failed (left edge).");
+ if (&c->progeny[7]->hydro.parts[c->progeny[7]->hydro.count] !=
+ &c->hydro.parts[count])
+ error("Particle sorting failed (right edge).");
+
+ /* Verify a few sub-cells. */
+ for (int k = 0; k < c->progeny[0]->hydro.count; k++)
+ if (c->progeny[0]->hydro.parts[k].x[0] >= pivot[0] ||
+ c->progeny[0]->hydro.parts[k].x[1] >= pivot[1] ||
+ c->progeny[0]->hydro.parts[k].x[2] >= pivot[2])
+ error("Sorting failed (progeny=0).");
+ for (int k = 0; k < c->progeny[1]->hydro.count; k++)
+ if (c->progeny[1]->hydro.parts[k].x[0] >= pivot[0] ||
+ c->progeny[1]->hydro.parts[k].x[1] >= pivot[1] ||
+ c->progeny[1]->hydro.parts[k].x[2] < pivot[2])
+ error("Sorting failed (progeny=1).");
+ for (int k = 0; k < c->progeny[2]->hydro.count; k++)
+ if (c->progeny[2]->hydro.parts[k].x[0] >= pivot[0] ||
+ c->progeny[2]->hydro.parts[k].x[1] < pivot[1] ||
+ c->progeny[2]->hydro.parts[k].x[2] >= pivot[2])
+ error("Sorting failed (progeny=2).");
+ for (int k = 0; k < c->progeny[3]->hydro.count; k++)
+ if (c->progeny[3]->hydro.parts[k].x[0] >= pivot[0] ||
+ c->progeny[3]->hydro.parts[k].x[1] < pivot[1] ||
+ c->progeny[3]->hydro.parts[k].x[2] < pivot[2])
+ error("Sorting failed (progeny=3).");
+ for (int k = 0; k < c->progeny[4]->hydro.count; k++)
+ if (c->progeny[4]->hydro.parts[k].x[0] < pivot[0] ||
+ c->progeny[4]->hydro.parts[k].x[1] >= pivot[1] ||
+ c->progeny[4]->hydro.parts[k].x[2] >= pivot[2])
+ error("Sorting failed (progeny=4).");
+ for (int k = 0; k < c->progeny[5]->hydro.count; k++)
+ if (c->progeny[5]->hydro.parts[k].x[0] < pivot[0] ||
+ c->progeny[5]->hydro.parts[k].x[1] >= pivot[1] ||
+ c->progeny[5]->hydro.parts[k].x[2] < pivot[2])
+ error("Sorting failed (progeny=5).");
+ for (int k = 0; k < c->progeny[6]->hydro.count; k++)
+ if (c->progeny[6]->hydro.parts[k].x[0] < pivot[0] ||
+ c->progeny[6]->hydro.parts[k].x[1] < pivot[1] ||
+ c->progeny[6]->hydro.parts[k].x[2] >= pivot[2])
+ error("Sorting failed (progeny=6).");
+ for (int k = 0; k < c->progeny[7]->hydro.count; k++)
+ if (c->progeny[7]->hydro.parts[k].x[0] < pivot[0] ||
+ c->progeny[7]->hydro.parts[k].x[1] < pivot[1] ||
+ c->progeny[7]->hydro.parts[k].x[2] < pivot[2])
+ error("Sorting failed (progeny=7).");
+#endif
+
+ /* Now do the same song and dance for the sparts. */
+ for (int k = 0; k < 8; k++) bucket_count[k] = 0;
+
+ /* Fill the buffer with the indices. */
+ for (int k = 0; k < scount; k++) {
+ const int bid = (sbuff[k].x[0] > pivot[0]) * 4 +
+ (sbuff[k].x[1] > pivot[1]) * 2 + (sbuff[k].x[2] > pivot[2]);
+ bucket_count[bid]++;
+ sbuff[k].ind = bid;
+ }
+
+ /* Set the buffer offsets. */
+ bucket_offset[0] = 0;
+ for (int k = 1; k <= 8; k++) {
+ bucket_offset[k] = bucket_offset[k - 1] + bucket_count[k - 1];
+ bucket_count[k - 1] = 0;
+ }
+
+ /* Run through the buckets, and swap particles to their correct spot. */
+ for (int bucket = 0; bucket < 8; bucket++) {
+ for (int k = bucket_offset[bucket] + bucket_count[bucket];
+ k < bucket_offset[bucket + 1]; k++) {
+ int bid = sbuff[k].ind;
+ if (bid != bucket) {
+ struct spart spart = sparts[k];
+ struct cell_buff temp_buff = sbuff[k];
+ while (bid != bucket) {
+ int j = bucket_offset[bid] + bucket_count[bid]++;
+ while (sbuff[j].ind == bid) {
+ j++;
+ bucket_count[bid]++;
+ }
+ memswap(&sparts[j], &spart, sizeof(struct spart));
+ memswap(&sbuff[j], &temp_buff, sizeof(struct cell_buff));
+ if (sparts[j].gpart)
+ sparts[j].gpart->id_or_neg_offset = -(j + sparts_offset);
+ bid = temp_buff.ind;
+ }
+ sparts[k] = spart;
+ sbuff[k] = temp_buff;
+ if (sparts[k].gpart)
+ sparts[k].gpart->id_or_neg_offset = -(k + sparts_offset);
+ }
+ bucket_count[bid]++;
+ }
+ }
+
+ /* Store the counts and offsets. */
+ for (int k = 0; k < 8; k++) {
+ c->progeny[k]->stars.count = bucket_count[k];
+ c->progeny[k]->stars.count_total = c->progeny[k]->stars.count;
+ c->progeny[k]->stars.parts = &c->stars.parts[bucket_offset[k]];
+ c->progeny[k]->stars.parts_rebuild = c->progeny[k]->stars.parts;
+ }
+
+ /* Now do the same song and dance for the bparts. */
+ for (int k = 0; k < 8; k++) bucket_count[k] = 0;
+
+ /* Fill the buffer with the indices. */
+ for (int k = 0; k < bcount; k++) {
+ const int bid = (bbuff[k].x[0] > pivot[0]) * 4 +
+ (bbuff[k].x[1] > pivot[1]) * 2 + (bbuff[k].x[2] > pivot[2]);
+ bucket_count[bid]++;
+ bbuff[k].ind = bid;
+ }
+
+ /* Set the buffer offsets. */
+ bucket_offset[0] = 0;
+ for (int k = 1; k <= 8; k++) {
+ bucket_offset[k] = bucket_offset[k - 1] + bucket_count[k - 1];
+ bucket_count[k - 1] = 0;
+ }
+
+ /* Run through the buckets, and swap particles to their correct spot. */
+ for (int bucket = 0; bucket < 8; bucket++) {
+ for (int k = bucket_offset[bucket] + bucket_count[bucket];
+ k < bucket_offset[bucket + 1]; k++) {
+ int bid = bbuff[k].ind;
+ if (bid != bucket) {
+ struct bpart bpart = bparts[k];
+ struct cell_buff temp_buff = bbuff[k];
+ while (bid != bucket) {
+ int j = bucket_offset[bid] + bucket_count[bid]++;
+ while (bbuff[j].ind == bid) {
+ j++;
+ bucket_count[bid]++;
+ }
+ memswap(&bparts[j], &bpart, sizeof(struct bpart));
+ memswap(&bbuff[j], &temp_buff, sizeof(struct cell_buff));
+ if (bparts[j].gpart)
+ bparts[j].gpart->id_or_neg_offset = -(j + bparts_offset);
+ bid = temp_buff.ind;
+ }
+ bparts[k] = bpart;
+ bbuff[k] = temp_buff;
+ if (bparts[k].gpart)
+ bparts[k].gpart->id_or_neg_offset = -(k + bparts_offset);
+ }
+ bucket_count[bid]++;
+ }
+ }
+
+ /* Store the counts and offsets. */
+ for (int k = 0; k < 8; k++) {
+ c->progeny[k]->black_holes.count = bucket_count[k];
+ c->progeny[k]->black_holes.count_total = c->progeny[k]->black_holes.count;
+ c->progeny[k]->black_holes.parts = &c->black_holes.parts[bucket_offset[k]];
+ }
+
+ /* Now do the same song and dance for the sinks. */
+ for (int k = 0; k < 8; k++) bucket_count[k] = 0;
+
+ /* Fill the buffer with the indices. */
+ for (int k = 0; k < sink_count; k++) {
+ const int bid = (sinkbuff[k].x[0] > pivot[0]) * 4 +
+ (sinkbuff[k].x[1] > pivot[1]) * 2 +
+ (sinkbuff[k].x[2] > pivot[2]);
+ bucket_count[bid]++;
+ sinkbuff[k].ind = bid;
+ }
+
+ /* Set the buffer offsets. */
+ bucket_offset[0] = 0;
+ for (int k = 1; k <= 8; k++) {
+ bucket_offset[k] = bucket_offset[k - 1] + bucket_count[k - 1];
+ bucket_count[k - 1] = 0;
+ }
+
+ /* Run through the buckets, and swap particles to their correct spot. */
+ for (int bucket = 0; bucket < 8; bucket++) {
+ for (int k = bucket_offset[bucket] + bucket_count[bucket];
+ k < bucket_offset[bucket + 1]; k++) {
+ int bid = sinkbuff[k].ind;
+ if (bid != bucket) {
+ struct sink sink = sinks[k];
+ struct cell_buff temp_buff = sinkbuff[k];
+ while (bid != bucket) {
+ int j = bucket_offset[bid] + bucket_count[bid]++;
+ while (sinkbuff[j].ind == bid) {
+ j++;
+ bucket_count[bid]++;
+ }
+ memswap(&sinks[j], &sink, sizeof(struct sink));
+ memswap(&sinkbuff[j], &temp_buff, sizeof(struct cell_buff));
+ if (sinks[j].gpart)
+ sinks[j].gpart->id_or_neg_offset = -(j + sinks_offset);
+ bid = temp_buff.ind;
+ }
+ sinks[k] = sink;
+ sinkbuff[k] = temp_buff;
+ if (sinks[k].gpart)
+ sinks[k].gpart->id_or_neg_offset = -(k + sinks_offset);
+ }
+ bucket_count[bid]++;
+ }
+ }
+
+ /* Store the counts and offsets. */
+ for (int k = 0; k < 8; k++) {
+ c->progeny[k]->sinks.count = bucket_count[k];
+ c->progeny[k]->sinks.count_total = c->progeny[k]->sinks.count;
+ c->progeny[k]->sinks.parts = &c->sinks.parts[bucket_offset[k]];
+ }
+
+ /* Finally, do the same song and dance for the gparts. */
+ for (int k = 0; k < 8; k++) bucket_count[k] = 0;
+
+ /* Fill the buffer with the indices. */
+ for (int k = 0; k < gcount; k++) {
+ const int bid = (gbuff[k].x[0] > pivot[0]) * 4 +
+ (gbuff[k].x[1] > pivot[1]) * 2 + (gbuff[k].x[2] > pivot[2]);
+ bucket_count[bid]++;
+ gbuff[k].ind = bid;
+ }
+
+ /* Set the buffer offsets. */
+ bucket_offset[0] = 0;
+ for (int k = 1; k <= 8; k++) {
+ bucket_offset[k] = bucket_offset[k - 1] + bucket_count[k - 1];
+ bucket_count[k - 1] = 0;
+ }
+
+ /* Run through the buckets, and swap particles to their correct spot. */
+ for (int bucket = 0; bucket < 8; bucket++) {
+ for (int k = bucket_offset[bucket] + bucket_count[bucket];
+ k < bucket_offset[bucket + 1]; k++) {
+ int bid = gbuff[k].ind;
+ if (bid != bucket) {
+ struct gpart gpart = gparts[k];
+ struct cell_buff temp_buff = gbuff[k];
+ while (bid != bucket) {
+ int j = bucket_offset[bid] + bucket_count[bid]++;
+ while (gbuff[j].ind == bid) {
+ j++;
+ bucket_count[bid]++;
+ }
+ memswap_unaligned(&gparts[j], &gpart, sizeof(struct gpart));
+ memswap(&gbuff[j], &temp_buff, sizeof(struct cell_buff));
+ if (gparts[j].type == swift_type_gas) {
+ parts[-gparts[j].id_or_neg_offset - parts_offset].gpart =
+ &gparts[j];
+ } else if (gparts[j].type == swift_type_stars) {
+ sparts[-gparts[j].id_or_neg_offset - sparts_offset].gpart =
+ &gparts[j];
+ } else if (gparts[j].type == swift_type_sink) {
+ sinks[-gparts[j].id_or_neg_offset - sinks_offset].gpart =
+ &gparts[j];
+ } else if (gparts[j].type == swift_type_black_hole) {
+ bparts[-gparts[j].id_or_neg_offset - bparts_offset].gpart =
+ &gparts[j];
+ }
+ bid = temp_buff.ind;
+ }
+ gparts[k] = gpart;
+ gbuff[k] = temp_buff;
+ if (gparts[k].type == swift_type_gas) {
+ parts[-gparts[k].id_or_neg_offset - parts_offset].gpart = &gparts[k];
+ } else if (gparts[k].type == swift_type_stars) {
+ sparts[-gparts[k].id_or_neg_offset - sparts_offset].gpart =
+ &gparts[k];
+ } else if (gparts[k].type == swift_type_sink) {
+ sinks[-gparts[k].id_or_neg_offset - sinks_offset].gpart = &gparts[k];
+ } else if (gparts[k].type == swift_type_black_hole) {
+ bparts[-gparts[k].id_or_neg_offset - bparts_offset].gpart =
+ &gparts[k];
+ }
+ }
+ bucket_count[bid]++;
+ }
+ }
+
+ /* Store the counts and offsets. */
+ for (int k = 0; k < 8; k++) {
+ c->progeny[k]->grav.count = bucket_count[k];
+ c->progeny[k]->grav.count_total = c->progeny[k]->grav.count;
+ c->progeny[k]->grav.parts = &c->grav.parts[bucket_offset[k]];
+ c->progeny[k]->grav.parts_rebuild = c->progeny[k]->grav.parts;
+ }
+}
+
+/**
+ * @brief Re-arrange the #part in a top-level cell such that all the extra
+ * ones for on-the-fly creation are located at the end of the array.
+ *
+ * @param c The #cell to sort.
+ * @param parts_offset The offset between the first #part in the array and the
+ * first #part in the global array in the space structure (for re-linking).
+ */
+void cell_reorder_extra_parts(struct cell *c, const ptrdiff_t parts_offset) {
+ struct part *parts = c->hydro.parts;
+ struct xpart *xparts = c->hydro.xparts;
+ const int count_real = c->hydro.count;
+
+ if (c->depth != 0 || c->nodeID != engine_rank)
+ error("This function should only be called on local top-level cells!");
+
+ int first_not_extra = count_real;
+
+ /* Find extra particles */
+ for (int i = 0; i < count_real; ++i) {
+ if (parts[i].time_bin == time_bin_not_created) {
+ /* Find the first non-extra particle after the end of the
+ real particles */
+ while (parts[first_not_extra].time_bin == time_bin_not_created) {
+ ++first_not_extra;
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (first_not_extra >= count_real + space_extra_parts)
+ error("Looking for extra particles beyond this cell's range!");
+#endif
+
+ /* Swap everything, including g-part pointer */
+ memswap(&parts[i], &parts[first_not_extra], sizeof(struct part));
+ memswap(&xparts[i], &xparts[first_not_extra], sizeof(struct xpart));
+ if (parts[i].gpart)
+ parts[i].gpart->id_or_neg_offset = -(i + parts_offset);
+ }
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ for (int i = 0; i < c->hydro.count_total; ++i) {
+ if (parts[i].time_bin == time_bin_not_created && i < c->hydro.count) {
+ error("Extra particle before the end of the regular array");
+ }
+ if (parts[i].time_bin != time_bin_not_created && i >= c->hydro.count) {
+ error("Regular particle after the end of the regular array");
+ }
+ }
+#endif
+}
+
+/**
+ * @brief Re-arrange the #spart in a top-level cell such that all the extra
+ * ones for on-the-fly creation are located at the end of the array.
+ *
+ * @param c The #cell to sort.
+ * @param sparts_offset The offset between the first #spart in the array and
+ * the first #spart in the global array in the space structure (for
+ * re-linking).
+ */
+void cell_reorder_extra_sparts(struct cell *c, const ptrdiff_t sparts_offset) {
+ struct spart *sparts = c->stars.parts;
+ const int count_real = c->stars.count;
+
+ if (c->depth != 0 || c->nodeID != engine_rank)
+ error("This function should only be called on local top-level cells!");
+
+ int first_not_extra = count_real;
+
+ /* Find extra particles */
+ for (int i = 0; i < count_real; ++i) {
+ if (sparts[i].time_bin == time_bin_not_created) {
+ /* Find the first non-extra particle after the end of the
+ real particles */
+ while (sparts[first_not_extra].time_bin == time_bin_not_created) {
+ ++first_not_extra;
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (first_not_extra >= count_real + space_extra_sparts)
+ error("Looking for extra particles beyond this cell's range!");
+#endif
+
+ /* Swap everything, including g-part pointer */
+ memswap(&sparts[i], &sparts[first_not_extra], sizeof(struct spart));
+ if (sparts[i].gpart)
+ sparts[i].gpart->id_or_neg_offset = -(i + sparts_offset);
+ sparts[first_not_extra].gpart = NULL;
+#ifdef SWIFT_DEBUG_CHECKS
+ if (sparts[first_not_extra].time_bin != time_bin_not_created)
+ error("Incorrect swap occured!");
+#endif
+ }
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ for (int i = 0; i < c->stars.count_total; ++i) {
+ if (sparts[i].time_bin == time_bin_not_created && i < c->stars.count) {
+ error("Extra particle before the end of the regular array");
+ }
+ if (sparts[i].time_bin != time_bin_not_created && i >= c->stars.count) {
+ error("Regular particle after the end of the regular array");
+ }
+ }
+#endif
+}
+
+/**
+ * @brief Re-arrange the #sink in a top-level cell such that all the extra
+ * ones for on-the-fly creation are located at the end of the array.
+ *
+ * @param c The #cell to sort.
+ * @param sinks_offset The offset between the first #sink in the array and
+ * the first #sink in the global array in the space structure (for
+ * re-linking).
+ */
+void cell_reorder_extra_sinks(struct cell *c, const ptrdiff_t sinks_offset) {
+ struct sink *sinks = c->sinks.parts;
+ const int count_real = c->sinks.count;
+
+ if (c->depth != 0 || c->nodeID != engine_rank)
+ error("This function should only be called on local top-level cells!");
+
+ int first_not_extra = count_real;
+
+ /* Find extra particles */
+ for (int i = 0; i < count_real; ++i) {
+ if (sinks[i].time_bin == time_bin_not_created) {
+ /* Find the first non-extra particle after the end of the
+ real particles */
+ while (sinks[first_not_extra].time_bin == time_bin_not_created) {
+ ++first_not_extra;
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (first_not_extra >= count_real + space_extra_sinks)
+ error("Looking for extra particles beyond this cell's range!");
+#endif
+
+ /* Swap everything, including g-part pointer */
+ memswap(&sinks[i], &sinks[first_not_extra], sizeof(struct sink));
+ if (sinks[i].gpart)
+ sinks[i].gpart->id_or_neg_offset = -(i + sinks_offset);
+ sinks[first_not_extra].gpart = NULL;
+#ifdef SWIFT_DEBUG_CHECKS
+ if (sinks[first_not_extra].time_bin != time_bin_not_created)
+ error("Incorrect swap occured!");
+#endif
+ }
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ for (int i = 0; i < c->sinks.count_total; ++i) {
+ if (sinks[i].time_bin == time_bin_not_created && i < c->sinks.count) {
+ error("Extra particle before the end of the regular array");
+ }
+ if (sinks[i].time_bin != time_bin_not_created && i >= c->sinks.count) {
+ error("Regular particle after the end of the regular array");
+ }
+ }
+#endif
+}
+
+/**
+ * @brief Re-arrange the #gpart in a top-level cell such that all the extra
+ * ones for on-the-fly creation are located at the end of the array.
+ *
+ * @param c The #cell to sort.
+ * @param parts The global array of #part (for re-linking).
+ * @param sparts The global array of #spart (for re-linking).
+ */
+void cell_reorder_extra_gparts(struct cell *c, struct part *parts,
+ struct spart *sparts) {
+ struct gpart *gparts = c->grav.parts;
+ const int count_real = c->grav.count;
+
+ if (c->depth != 0 || c->nodeID != engine_rank)
+ error("This function should only be called on local top-level cells!");
+
+ int first_not_extra = count_real;
+
+ /* Find extra particles */
+ for (int i = 0; i < count_real; ++i) {
+ if (gparts[i].time_bin == time_bin_not_created) {
+ /* Find the first non-extra particle after the end of the
+ real particles */
+ while (gparts[first_not_extra].time_bin == time_bin_not_created) {
+ ++first_not_extra;
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (first_not_extra >= count_real + space_extra_gparts)
+ error("Looking for extra particles beyond this cell's range!");
+#endif
+
+ /* Swap everything (including pointers) */
+ memswap_unaligned(&gparts[i], &gparts[first_not_extra],
+ sizeof(struct gpart));
+ if (gparts[i].type == swift_type_gas) {
+ parts[-gparts[i].id_or_neg_offset].gpart = &gparts[i];
+ } else if (gparts[i].type == swift_type_stars) {
+ sparts[-gparts[i].id_or_neg_offset].gpart = &gparts[i];
+ }
+ }
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ for (int i = 0; i < c->grav.count_total; ++i) {
+ if (gparts[i].time_bin == time_bin_not_created && i < c->grav.count) {
+ error("Extra particle before the end of the regular array");
+ }
+ if (gparts[i].time_bin != time_bin_not_created && i >= c->grav.count) {
+ error("Regular particle after the end of the regular array");
+ }
+ }
+#endif
+}
diff --git a/src/cell_stars.h b/src/cell_stars.h
new file mode 100644
index 0000000000000000000000000000000000000000..eb1f480f791ac977de8f28214aeac7e8e74ef47c
--- /dev/null
+++ b/src/cell_stars.h
@@ -0,0 +1,138 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+#ifndef SWIFT_CELL_STARS_H
+#define SWIFT_CELL_STARS_H
+
+/* Config parameters. */
+#include "../config.h"
+
+/* Local includes. */
+#include "lock.h"
+#include "star_formation_logger_struct.h"
+#include "timeline.h"
+
+/**
+ * @brief Stars-related cell variables.
+ */
+struct cell_stars {
+
+ /*! Pointer to the #spart data. */
+ struct spart *parts;
+
+ /*! Pointer to the #spart data at rebuild time. */
+ struct spart *parts_rebuild;
+
+ /*! The star ghost task itself */
+ struct task *ghost;
+
+ /*! Linked list of the tasks computing this cell's star density. */
+ struct link *density;
+
+ /*! Linked list of the tasks computing this cell's star feedback. */
+ struct link *feedback;
+
+ /*! The task computing this cell's sorts before the density. */
+ struct task *sorts;
+
+ /*! The drift task for sparts */
+ struct task *drift;
+
+ /*! Implicit tasks marking the entry of the stellar physics block of tasks
+ */
+ struct task *stars_in;
+
+ /*! Implicit tasks marking the exit of the stellar physics block of tasks */
+ struct task *stars_out;
+
+ /*! Last (integer) time the cell's spart were drifted forward in time. */
+ integertime_t ti_old_part;
+
+ /*! Spin lock for various uses (#spart case). */
+ swift_lock_type lock;
+
+ /*! Spin lock for star formation use. */
+ swift_lock_type star_formation_lock;
+
+ /*! Nr of #spart in this cell. */
+ int count;
+
+ /*! Nr of #spart this cell can hold after addition of new #spart. */
+ int count_total;
+
+ /*! Max smoothing length in this cell. */
+ float h_max;
+
+ /*! Values of h_max before the drifts, used for sub-cell tasks. */
+ float h_max_old;
+
+ /*! Maximum part movement in this cell since last construction. */
+ float dx_max_part;
+
+ /*! Values of dx_max before the drifts, used for sub-cell tasks. */
+ float dx_max_part_old;
+
+ /*! Maximum particle movement in this cell since the last sort. */
+ float dx_max_sort;
+
+ /*! Values of dx_max_sort before the drifts, used for sub-cell tasks. */
+ float dx_max_sort_old;
+
+ /*! Pointer for the sorted indices. */
+ struct sort_entry *sort;
+
+ /*! Bit mask of sort directions that will be needed in the next timestep. */
+ uint16_t requires_sorts;
+
+ /*! Bit-mask indicating the sorted directions */
+ uint16_t sorted;
+
+ /*! Bit-mask indicating the sorted directions */
+ uint16_t sort_allocated;
+
+ /*! Bit mask of sorts that need to be computed for this cell. */
+ uint16_t do_sort;
+
+ /*! Maximum end of (integer) time step in this cell for star tasks. */
+ integertime_t ti_end_min;
+
+ /*! Maximum end of (integer) time step in this cell for star tasks. */
+ integertime_t ti_end_max;
+
+ /*! Maximum beginning of (integer) time step in this cell for star tasks.
+ */
+ integertime_t ti_beg_max;
+
+ /*! Number of #spart updated in this cell. */
+ int updated;
+
+ /*! Is the #spart data of this cell being used in a sub-cell? */
+ int hold;
+
+ /*! Star formation history struct */
+ struct star_formation_history sfh;
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /*! Last (integer) time the cell's sort arrays were updated. */
+ integertime_t ti_sort;
+#endif
+};
+
+#endif /* SWIFT_CELL_STARS_H */
diff --git a/src/cell_unskip.c b/src/cell_unskip.c
new file mode 100644
index 0000000000000000000000000000000000000000..76e45e54451c1903ad5fc8dcdb8c25ea22904347
--- /dev/null
+++ b/src/cell_unskip.c
@@ -0,0 +1,2398 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* This object's header. */
+#include "cell.h"
+
+/* Local headers. */
+#include "active.h"
+#include "engine.h"
+#include "space_getsid.h"
+
+extern int engine_star_resort_task_depth;
+
+/**
+ * @brief Recursively clear the stars_resort flag in a cell hierarchy.
+ *
+ * @param c The #cell to act on.
+ */
+void cell_set_star_resort_flag(struct cell *c) {
+
+ cell_set_flag(c, cell_flag_do_stars_resort);
+
+ /* Abort if we reched the level where the resorting task lives */
+ if (c->depth == engine_star_resort_task_depth || c->hydro.super == c) return;
+
+ if (c->split) {
+ for (int k = 0; k < 8; ++k)
+ if (c->progeny[k] != NULL) cell_set_star_resort_flag(c->progeny[k]);
+ }
+}
+
+/**
+ * @brief Recurses in a cell hierarchy down to the level where the
+ * star resort tasks are and activates them.
+ *
+ * The function will fail if called *below* the super-level
+ *
+ * @param c The #cell to recurse into.
+ * @param s The #scheduler.
+ */
+void cell_activate_star_resort_tasks(struct cell *c, struct scheduler *s) {
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->hydro.super != NULL && c->hydro.super != c)
+ error("Function called below the super level!");
+#endif
+
+ /* The resort tasks are at either the chosen depth or the super level,
+ * whichever comes first. */
+ if ((c->depth == engine_star_resort_task_depth || c->hydro.super == c) &&
+ c->hydro.count > 0) {
+ scheduler_activate(s, c->hydro.stars_resort);
+ } else {
+ for (int k = 0; k < 8; ++k) {
+ if (c->progeny[k] != NULL) {
+ cell_activate_star_resort_tasks(c->progeny[k], s);
+ }
+ }
+ }
+}
+
+/**
+ * @brief Activate the star formation task as well as the resorting of stars
+ *
+ * Must be called at the top-level in the tree (where the SF task is...)
+ *
+ * @param c The (top-level) #cell.
+ * @param s The #scheduler.
+ * @param with_feedback Are we running with feedback?
+ */
+void cell_activate_star_formation_tasks(struct cell *c, struct scheduler *s,
+ const int with_feedback) {
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->depth != 0) error("Function should be called at the top-level only");
+#endif
+
+ /* Have we already unskipped that task? */
+ if (c->hydro.star_formation->skip == 0) return;
+
+ /* Activate the star formation task */
+ scheduler_activate(s, c->hydro.star_formation);
+
+ /* Activate the star resort tasks at whatever level they are */
+ if (with_feedback) {
+ cell_activate_star_resort_tasks(c, s);
+ }
+}
+
+/**
+ * @brief Activate the sink formation task.
+ *
+ * Must be called at the top-level in the tree (where the SF task is...)
+ *
+ * @param c The (top-level) #cell.
+ * @param s The #scheduler.
+ */
+void cell_activate_sink_formation_tasks(struct cell *c, struct scheduler *s) {
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->depth != 0) error("Function should be called at the top-level only");
+#endif
+
+ /* Have we already unskipped that task? */
+ if (c->hydro.sink_formation->skip == 0) return;
+
+ /* Activate the star formation task */
+ scheduler_activate(s, c->hydro.sink_formation);
+}
+
+/**
+ * @brief Recursively activate the hydro ghosts (and implicit links) in a cell
+ * hierarchy.
+ *
+ * @param c The #cell.
+ * @param s The #scheduler.
+ * @param e The #engine.
+ */
+void cell_recursively_activate_hydro_ghosts(struct cell *c, struct scheduler *s,
+ const struct engine *e) {
+ /* Early abort? */
+ if ((c->hydro.count == 0) || !cell_is_active_hydro(c, e)) return;
+
+ /* Is the ghost at this level? */
+ if (c->hydro.ghost != NULL) {
+ scheduler_activate(s, c->hydro.ghost);
+ } else {
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (!c->split)
+ error("Reached the leaf level without finding a hydro ghost!");
+#endif
+
+ /* Keep recursing */
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL)
+ cell_recursively_activate_hydro_ghosts(c->progeny[k], s, e);
+ }
+}
+
+/**
+ * @brief Activate the hydro ghosts (and implicit links) in a cell hierarchy.
+ *
+ * @param c The #cell.
+ * @param s The #scheduler.
+ * @param e The #engine.
+ */
+void cell_activate_hydro_ghosts(struct cell *c, struct scheduler *s,
+ const struct engine *e) {
+ scheduler_activate(s, c->hydro.ghost_in);
+ scheduler_activate(s, c->hydro.ghost_out);
+ cell_recursively_activate_hydro_ghosts(c, s, e);
+}
+
+/**
+ * @brief Recursively activate the cooling (and implicit links) in a cell
+ * hierarchy.
+ *
+ * @param c The #cell.
+ * @param s The #scheduler.
+ * @param e The #engine.
+ */
+void cell_recursively_activate_cooling(struct cell *c, struct scheduler *s,
+ const struct engine *e) {
+ /* Early abort? */
+ if ((c->hydro.count == 0) || !cell_is_active_hydro(c, e)) return;
+
+ /* Is the ghost at this level? */
+ if (c->hydro.cooling != NULL) {
+ scheduler_activate(s, c->hydro.cooling);
+ } else {
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (!c->split)
+ error("Reached the leaf level without finding a cooling task!");
+#endif
+
+ /* Keep recursing */
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL)
+ cell_recursively_activate_cooling(c->progeny[k], s, e);
+ }
+}
+
+/**
+ * @brief Activate the cooling tasks (and implicit links) in a cell hierarchy.
+ *
+ * @param c The #cell.
+ * @param s The #scheduler.
+ * @param e The #engine.
+ */
+void cell_activate_cooling(struct cell *c, struct scheduler *s,
+ const struct engine *e) {
+ scheduler_activate(s, c->hydro.cooling_in);
+ scheduler_activate(s, c->hydro.cooling_out);
+ cell_recursively_activate_cooling(c, s, e);
+}
+
+/**
+ * @brief Recurse down in a cell hierarchy until the hydro.super level is
+ * reached and activate the spart drift at that level.
+ *
+ * @param c The #cell to recurse into.
+ * @param s The #scheduler.
+ */
+void cell_activate_super_spart_drifts(struct cell *c, struct scheduler *s) {
+
+ /* Early abort? */
+ if (c->hydro.count == 0) return;
+
+ if (c == c->hydro.super) {
+ cell_activate_drift_spart(c, s);
+ } else {
+ if (c->split) {
+ for (int k = 0; k < 8; ++k) {
+ if (c->progeny[k] != NULL) {
+ cell_activate_super_spart_drifts(c->progeny[k], s);
+ }
+ }
+ } else {
+#ifdef SWIFT_DEBUG_CHECKS
+ error("Reached a leaf cell without finding a hydro.super!!");
+#endif
+ }
+ }
+}
+
+/**
+ * @brief Recurse down in a cell hierarchy until the hydro.super level is
+ * reached and activate the sink drift at that level.
+ *
+ * @param c The #cell to recurse into.
+ * @param s The #scheduler.
+ */
+void cell_activate_super_sink_drifts(struct cell *c, struct scheduler *s) {
+
+ /* Early abort? */
+ if (c->hydro.count == 0) return;
+
+ if (c == c->hydro.super) {
+ cell_activate_drift_sink(c, s);
+ } else {
+ if (c->split) {
+ for (int k = 0; k < 8; ++k) {
+ if (c->progeny[k] != NULL) {
+ cell_activate_super_sink_drifts(c->progeny[k], s);
+ }
+ }
+ } else {
+#ifdef SWIFT_DEBUG_CHECKS
+ error("Reached a leaf cell without finding a hydro.super!!");
+#endif
+ }
+ }
+}
+
+/**
+ * @brief Activate the #part drifts on the given cell.
+ */
+void cell_activate_drift_part(struct cell *c, struct scheduler *s) {
+ /* If this cell is already marked for drift, quit early. */
+ if (cell_get_flag(c, cell_flag_do_hydro_drift)) return;
+
+ /* Mark this cell for drifting. */
+ cell_set_flag(c, cell_flag_do_hydro_drift);
+
+ /* Set the do_sub_drifts all the way up and activate the super drift
+ if this has not yet been done. */
+ if (c == c->hydro.super) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->hydro.drift == NULL)
+ error("Trying to activate un-existing c->hydro.drift");
+#endif
+ scheduler_activate(s, c->hydro.drift);
+ } else {
+ for (struct cell *parent = c->parent;
+ parent != NULL && !cell_get_flag(parent, cell_flag_do_hydro_sub_drift);
+ parent = parent->parent) {
+ /* Mark this cell for drifting */
+ cell_set_flag(parent, cell_flag_do_hydro_sub_drift);
+
+ if (parent == c->hydro.super) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (parent->hydro.drift == NULL)
+ error("Trying to activate un-existing parent->hydro.drift");
+#endif
+ scheduler_activate(s, parent->hydro.drift);
+ break;
+ }
+ }
+ }
+}
+
+void cell_activate_sync_part(struct cell *c, struct scheduler *s) {
+ /* If this cell is already marked for drift, quit early. */
+ if (cell_get_flag(c, cell_flag_do_hydro_sync)) return;
+
+ /* Mark this cell for synchronization. */
+ cell_set_flag(c, cell_flag_do_hydro_sync);
+
+ /* Set the do_sub_sync all the way up and activate the super sync
+ if this has not yet been done. */
+ if (c == c->super) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->timestep_sync == NULL)
+ error("Trying to activate un-existing c->timestep_sync");
+#endif
+ scheduler_activate(s, c->timestep_sync);
+ scheduler_activate(s, c->kick1);
+ } else {
+ for (struct cell *parent = c->parent;
+ parent != NULL && !cell_get_flag(parent, cell_flag_do_hydro_sub_sync);
+ parent = parent->parent) {
+ /* Mark this cell for drifting */
+ cell_set_flag(parent, cell_flag_do_hydro_sub_sync);
+
+ if (parent == c->super) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (parent->timestep_sync == NULL)
+ error("Trying to activate un-existing parent->timestep_sync");
+#endif
+ scheduler_activate(s, parent->timestep_sync);
+ scheduler_activate(s, parent->kick1);
+ break;
+ }
+ }
+ }
+}
+
+/**
+ * @brief Activate the #gpart drifts on the given cell.
+ */
+void cell_activate_drift_gpart(struct cell *c, struct scheduler *s) {
+ /* If this cell is already marked for drift, quit early. */
+ if (cell_get_flag(c, cell_flag_do_grav_drift)) return;
+
+ /* Mark this cell for drifting. */
+ cell_set_flag(c, cell_flag_do_grav_drift);
+
+ if (c->grav.drift_out != NULL) scheduler_activate(s, c->grav.drift_out);
+
+ /* Set the do_grav_sub_drifts all the way up and activate the super drift
+ if this has not yet been done. */
+ if (c == c->grav.super) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->grav.drift == NULL)
+ error("Trying to activate un-existing c->grav.drift");
+#endif
+ scheduler_activate(s, c->grav.drift);
+ } else {
+ for (struct cell *parent = c->parent;
+ parent != NULL && !cell_get_flag(parent, cell_flag_do_grav_sub_drift);
+ parent = parent->parent) {
+ cell_set_flag(parent, cell_flag_do_grav_sub_drift);
+
+ if (parent->grav.drift_out) {
+ scheduler_activate(s, parent->grav.drift_out);
+ }
+
+ if (parent == c->grav.super) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (parent->grav.drift == NULL)
+ error("Trying to activate un-existing parent->grav.drift");
+#endif
+ scheduler_activate(s, parent->grav.drift);
+ break;
+ }
+ }
+ }
+}
+
+/**
+ * @brief Activate the #spart drifts on the given cell.
+ */
+void cell_activate_drift_spart(struct cell *c, struct scheduler *s) {
+ /* If this cell is already marked for drift, quit early. */
+ if (cell_get_flag(c, cell_flag_do_stars_drift)) return;
+
+ /* Mark this cell for drifting. */
+ cell_set_flag(c, cell_flag_do_stars_drift);
+
+ /* Set the do_stars_sub_drifts all the way up and activate the super drift
+ if this has not yet been done. */
+ if (c == c->hydro.super) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->stars.drift == NULL)
+ error("Trying to activate un-existing c->stars.drift");
+#endif
+ scheduler_activate(s, c->stars.drift);
+ } else {
+ for (struct cell *parent = c->parent;
+ parent != NULL && !cell_get_flag(parent, cell_flag_do_stars_sub_drift);
+ parent = parent->parent) {
+ /* Mark this cell for drifting */
+ cell_set_flag(parent, cell_flag_do_stars_sub_drift);
+
+ if (parent == c->hydro.super) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (parent->stars.drift == NULL)
+ error("Trying to activate un-existing parent->stars.drift");
+#endif
+ scheduler_activate(s, parent->stars.drift);
+ break;
+ }
+ }
+ }
+}
+
+/**
+ * @brief Activate the #bpart drifts on the given cell.
+ */
+void cell_activate_drift_bpart(struct cell *c, struct scheduler *s) {
+
+ /* If this cell is already marked for drift, quit early. */
+ if (cell_get_flag(c, cell_flag_do_bh_drift)) return;
+
+ /* Mark this cell for drifting. */
+ cell_set_flag(c, cell_flag_do_bh_drift);
+
+ /* Set the do_black_holes_sub_drifts all the way up and activate the super
+ drift if this has not yet been done. */
+ if (c == c->hydro.super) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->black_holes.drift == NULL)
+ error("Trying to activate un-existing c->black_holes.drift");
+#endif
+ scheduler_activate(s, c->black_holes.drift);
+ } else {
+ for (struct cell *parent = c->parent;
+ parent != NULL && !cell_get_flag(parent, cell_flag_do_bh_sub_drift);
+ parent = parent->parent) {
+ /* Mark this cell for drifting */
+ cell_set_flag(parent, cell_flag_do_bh_sub_drift);
+
+ if (parent == c->hydro.super) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (parent->black_holes.drift == NULL)
+ error("Trying to activate un-existing parent->black_holes.drift");
+#endif
+ scheduler_activate(s, parent->black_holes.drift);
+ break;
+ }
+ }
+ }
+}
+
+/**
+ * @brief Activate the #sink drifts on the given cell.
+ */
+void cell_activate_drift_sink(struct cell *c, struct scheduler *s) {
+
+ /* If this cell is already marked for drift, quit early. */
+ if (cell_get_flag(c, cell_flag_do_sink_drift)) return;
+
+ /* Mark this cell for drifting. */
+ cell_set_flag(c, cell_flag_do_sink_drift);
+
+ /* Set the do_sink_sub_drifts all the way up and activate the super
+ drift if this has not yet been done. */
+ if (c == c->hydro.super) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->sinks.drift == NULL)
+ error("Trying to activate un-existing c->sinks.drift");
+#endif
+ scheduler_activate(s, c->sinks.drift);
+ } else {
+ for (struct cell *parent = c->parent;
+ parent != NULL && !cell_get_flag(parent, cell_flag_do_sink_sub_drift);
+ parent = parent->parent) {
+ /* Mark this cell for drifting */
+ cell_set_flag(parent, cell_flag_do_sink_sub_drift);
+
+ if (parent == c->hydro.super) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (parent->sinks.drift == NULL)
+ error("Trying to activate un-existing parent->sinks.drift");
+#endif
+ scheduler_activate(s, parent->sinks.drift);
+ break;
+ }
+ }
+ }
+}
+
+/**
+ * @brief Activate the drifts on the given cell.
+ */
+void cell_activate_limiter(struct cell *c, struct scheduler *s) {
+ /* If this cell is already marked for limiting, quit early. */
+ if (cell_get_flag(c, cell_flag_do_hydro_limiter)) return;
+
+ /* Mark this cell for limiting. */
+ cell_set_flag(c, cell_flag_do_hydro_limiter);
+
+ /* Set the do_sub_limiter all the way up and activate the super limiter
+ if this has not yet been done. */
+ if (c == c->super) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->timestep_limiter == NULL)
+ error("Trying to activate un-existing c->timestep_limiter");
+#endif
+ scheduler_activate(s, c->timestep_limiter);
+ scheduler_activate(s, c->kick1);
+ } else {
+ for (struct cell *parent = c->parent;
+ parent != NULL &&
+ !cell_get_flag(parent, cell_flag_do_hydro_sub_limiter);
+ parent = parent->parent) {
+ /* Mark this cell for limiting */
+ cell_set_flag(parent, cell_flag_do_hydro_sub_limiter);
+
+ if (parent == c->super) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (parent->timestep_limiter == NULL)
+ error("Trying to activate un-existing parent->timestep_limiter");
+#endif
+ scheduler_activate(s, parent->timestep_limiter);
+ scheduler_activate(s, parent->kick1);
+ break;
+ }
+ }
+ }
+}
+
+/**
+ * @brief Activate the sorts up a cell hierarchy.
+ */
+void cell_activate_hydro_sorts_up(struct cell *c, struct scheduler *s) {
+ if (c == c->hydro.super) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->hydro.sorts == NULL)
+ error("Trying to activate un-existing c->hydro.sorts");
+#endif
+ scheduler_activate(s, c->hydro.sorts);
+ if (c->nodeID == engine_rank) cell_activate_drift_part(c, s);
+ } else {
+ for (struct cell *parent = c->parent;
+ parent != NULL && !cell_get_flag(parent, cell_flag_do_hydro_sub_sort);
+ parent = parent->parent) {
+ cell_set_flag(parent, cell_flag_do_hydro_sub_sort);
+ if (parent == c->hydro.super) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (parent->hydro.sorts == NULL)
+ error("Trying to activate un-existing parents->hydro.sorts");
+#endif
+ scheduler_activate(s, parent->hydro.sorts);
+ if (parent->nodeID == engine_rank) cell_activate_drift_part(parent, s);
+ break;
+ }
+ }
+ }
+}
+
+/**
+ * @brief Activate the sorts on a given cell, if needed.
+ */
+void cell_activate_hydro_sorts(struct cell *c, int sid, struct scheduler *s) {
+ /* Do we need to re-sort? */
+ if (c->hydro.dx_max_sort > space_maxreldx * c->dmin) {
+ /* Climb up the tree to active the sorts in that direction */
+ for (struct cell *finger = c; finger != NULL; finger = finger->parent) {
+ if (finger->hydro.requires_sorts) {
+ atomic_or(&finger->hydro.do_sort, finger->hydro.requires_sorts);
+ cell_activate_hydro_sorts_up(finger, s);
+ }
+ finger->hydro.sorted = 0;
+ }
+ }
+
+ /* Has this cell been sorted at all for the given sid? */
+ if (!(c->hydro.sorted & (1 << sid)) || c->nodeID != engine_rank) {
+ atomic_or(&c->hydro.do_sort, (1 << sid));
+ cell_activate_hydro_sorts_up(c, s);
+ }
+}
+
+/**
+ * @brief Activate the sorts up a cell hierarchy.
+ */
+void cell_activate_stars_sorts_up(struct cell *c, struct scheduler *s) {
+
+ if (c == c->hydro.super) {
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (c->stars.sorts == NULL)
+ error("Trying to activate un-existing c->stars.sorts");
+#endif
+ scheduler_activate(s, c->stars.sorts);
+ if (c->nodeID == engine_rank) {
+ cell_activate_drift_spart(c, s);
+ }
+ } else {
+
+ /* Climb up the tree and set the flags */
+ for (struct cell *parent = c->parent;
+ parent != NULL && !cell_get_flag(parent, cell_flag_do_stars_sub_sort);
+ parent = parent->parent) {
+
+ cell_set_flag(parent, cell_flag_do_stars_sub_sort);
+
+ /* Reached the super-level? Activate the task and abort */
+ if (parent == c->hydro.super) {
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (parent->stars.sorts == NULL)
+ error("Trying to activate un-existing parents->stars.sorts");
+#endif
+ scheduler_activate(s, parent->stars.sorts);
+ if (parent->nodeID == engine_rank) cell_activate_drift_spart(parent, s);
+ break;
+ }
+ }
+ }
+}
+
+/**
+ * @brief Activate the sorts on a given cell, if needed.
+ */
+void cell_activate_stars_sorts(struct cell *c, int sid, struct scheduler *s) {
+
+ /* Do we need to re-sort? */
+ if (c->stars.dx_max_sort > space_maxreldx * c->dmin) {
+
+ /* Climb up the tree to active the sorts in that direction */
+ for (struct cell *finger = c; finger != NULL; finger = finger->parent) {
+ if (finger->stars.requires_sorts) {
+ atomic_or(&finger->stars.do_sort, finger->stars.requires_sorts);
+ cell_activate_stars_sorts_up(finger, s);
+ }
+ finger->stars.sorted = 0;
+ }
+ }
+
+ /* Has this cell been sorted at all for the given sid? */
+ if (!(c->stars.sorted & (1 << sid)) || c->nodeID != engine_rank) {
+ atomic_or(&c->stars.do_sort, (1 << sid));
+ cell_activate_stars_sorts_up(c, s);
+ }
+}
+
+/**
+ * @brief Traverse a sub-cell task and activate the hydro drift tasks that are
+ * required by a hydro task
+ *
+ * @param ci The first #cell we recurse in.
+ * @param cj The second #cell we recurse in.
+ * @param s The task #scheduler.
+ * @param with_timestep_limiter Are we running with time-step limiting on?
+ */
+void cell_activate_subcell_hydro_tasks(struct cell *ci, struct cell *cj,
+ struct scheduler *s,
+ const int with_timestep_limiter) {
+ const struct engine *e = s->space->e;
+
+ /* Store the current dx_max and h_max values. */
+ ci->hydro.dx_max_part_old = ci->hydro.dx_max_part;
+ ci->hydro.h_max_old = ci->hydro.h_max;
+
+ if (cj != NULL) {
+ cj->hydro.dx_max_part_old = cj->hydro.dx_max_part;
+ cj->hydro.h_max_old = cj->hydro.h_max;
+ }
+
+ /* Self interaction? */
+ if (cj == NULL) {
+ /* Do anything? */
+ if (ci->hydro.count == 0 || !cell_is_active_hydro(ci, e)) return;
+
+ /* Recurse? */
+ if (cell_can_recurse_in_self_hydro_task(ci)) {
+ /* Loop over all progenies and pairs of progenies */
+ for (int j = 0; j < 8; j++) {
+ if (ci->progeny[j] != NULL) {
+ cell_activate_subcell_hydro_tasks(ci->progeny[j], NULL, s,
+ with_timestep_limiter);
+ for (int k = j + 1; k < 8; k++)
+ if (ci->progeny[k] != NULL)
+ cell_activate_subcell_hydro_tasks(ci->progeny[j], ci->progeny[k],
+ s, with_timestep_limiter);
+ }
+ }
+ } else {
+ /* We have reached the bottom of the tree: activate drift */
+ cell_activate_drift_part(ci, s);
+ if (with_timestep_limiter) cell_activate_limiter(ci, s);
+ }
+ }
+
+ /* Otherwise, pair interation */
+ else {
+ /* Should we even bother? */
+ if (!cell_is_active_hydro(ci, e) && !cell_is_active_hydro(cj, e)) return;
+ if (ci->hydro.count == 0 || cj->hydro.count == 0) return;
+
+ /* Get the orientation of the pair. */
+ double shift[3];
+ const int sid = space_getsid(s->space, &ci, &cj, shift);
+
+ /* recurse? */
+ if (cell_can_recurse_in_pair_hydro_task(ci) &&
+ cell_can_recurse_in_pair_hydro_task(cj)) {
+ const struct cell_split_pair *csp = &cell_split_pairs[sid];
+ for (int k = 0; k < csp->count; k++) {
+ const int pid = csp->pairs[k].pid;
+ const int pjd = csp->pairs[k].pjd;
+ if (ci->progeny[pid] != NULL && cj->progeny[pjd] != NULL)
+ cell_activate_subcell_hydro_tasks(ci->progeny[pid], cj->progeny[pjd],
+ s, with_timestep_limiter);
+ }
+ }
+
+ /* Otherwise, activate the sorts and drifts. */
+ else if (cell_is_active_hydro(ci, e) || cell_is_active_hydro(cj, e)) {
+ /* We are going to interact this pair, so store some values. */
+ atomic_or(&ci->hydro.requires_sorts, 1 << sid);
+ atomic_or(&cj->hydro.requires_sorts, 1 << sid);
+ ci->hydro.dx_max_sort_old = ci->hydro.dx_max_sort;
+ cj->hydro.dx_max_sort_old = cj->hydro.dx_max_sort;
+
+ /* Activate the drifts if the cells are local. */
+ if (ci->nodeID == engine_rank) cell_activate_drift_part(ci, s);
+ if (cj->nodeID == engine_rank) cell_activate_drift_part(cj, s);
+
+ /* Also activate the time-step limiter */
+ if (ci->nodeID == engine_rank && with_timestep_limiter)
+ cell_activate_limiter(ci, s);
+ if (cj->nodeID == engine_rank && with_timestep_limiter)
+ cell_activate_limiter(cj, s);
+
+ /* Do we need to sort the cells? */
+ cell_activate_hydro_sorts(ci, sid, s);
+ cell_activate_hydro_sorts(cj, sid, s);
+ }
+ } /* Otherwise, pair interation */
+}
+
+/**
+ * @brief Traverse a sub-cell task and activate the stars drift tasks that are
+ * required by a stars task
+ *
+ * @param ci The first #cell we recurse in.
+ * @param cj The second #cell we recurse in.
+ * @param s The task #scheduler.
+ * @param with_star_formation Are we running with star formation switched on?
+ * @param with_timestep_sync Are we running with time-step synchronization on?
+ */
+void cell_activate_subcell_stars_tasks(struct cell *ci, struct cell *cj,
+ struct scheduler *s,
+ const int with_star_formation,
+ const int with_timestep_sync) {
+ const struct engine *e = s->space->e;
+
+ /* Store the current dx_max and h_max values. */
+ ci->stars.dx_max_part_old = ci->stars.dx_max_part;
+ ci->stars.h_max_old = ci->stars.h_max;
+ ci->hydro.dx_max_part_old = ci->hydro.dx_max_part;
+ ci->hydro.h_max_old = ci->hydro.h_max;
+
+ if (cj != NULL) {
+ cj->stars.dx_max_part_old = cj->stars.dx_max_part;
+ cj->stars.h_max_old = cj->stars.h_max;
+ cj->hydro.dx_max_part_old = cj->hydro.dx_max_part;
+ cj->hydro.h_max_old = cj->hydro.h_max;
+ }
+
+ /* Self interaction? */
+ if (cj == NULL) {
+
+ const int ci_active = cell_is_active_stars(ci, e) ||
+ (with_star_formation && cell_is_active_hydro(ci, e));
+
+ /* Do anything? */
+ if (!ci_active || ci->hydro.count == 0 ||
+ (!with_star_formation && ci->stars.count == 0))
+ return;
+
+ /* Recurse? */
+ if (cell_can_recurse_in_self_stars_task(ci)) {
+ /* Loop over all progenies and pairs of progenies */
+ for (int j = 0; j < 8; j++) {
+ if (ci->progeny[j] != NULL) {
+ cell_activate_subcell_stars_tasks(
+ ci->progeny[j], NULL, s, with_star_formation, with_timestep_sync);
+ for (int k = j + 1; k < 8; k++)
+ if (ci->progeny[k] != NULL)
+ cell_activate_subcell_stars_tasks(ci->progeny[j], ci->progeny[k],
+ s, with_star_formation,
+ with_timestep_sync);
+ }
+ }
+ } else {
+ /* We have reached the bottom of the tree: activate drift */
+ cell_activate_drift_spart(ci, s);
+ cell_activate_drift_part(ci, s);
+ if (with_timestep_sync) cell_activate_sync_part(ci, s);
+ }
+ }
+
+ /* Otherwise, pair interation */
+ else {
+
+ /* Get the orientation of the pair. */
+ double shift[3];
+ const int sid = space_getsid(s->space, &ci, &cj, shift);
+
+ const int ci_active = cell_is_active_stars(ci, e) ||
+ (with_star_formation && cell_is_active_hydro(ci, e));
+ const int cj_active = cell_is_active_stars(cj, e) ||
+ (with_star_formation && cell_is_active_hydro(cj, e));
+
+ /* Should we even bother? */
+ if (!ci_active && !cj_active) return;
+
+ /* recurse? */
+ if (cell_can_recurse_in_pair_stars_task(ci, cj) &&
+ cell_can_recurse_in_pair_stars_task(cj, ci)) {
+
+ const struct cell_split_pair *csp = &cell_split_pairs[sid];
+ for (int k = 0; k < csp->count; k++) {
+ const int pid = csp->pairs[k].pid;
+ const int pjd = csp->pairs[k].pjd;
+ if (ci->progeny[pid] != NULL && cj->progeny[pjd] != NULL)
+ cell_activate_subcell_stars_tasks(ci->progeny[pid], cj->progeny[pjd],
+ s, with_star_formation,
+ with_timestep_sync);
+ }
+ }
+
+ /* Otherwise, activate the sorts and drifts. */
+ else {
+
+ if (ci_active) {
+
+ /* We are going to interact this pair, so store some values. */
+ atomic_or(&cj->hydro.requires_sorts, 1 << sid);
+ atomic_or(&ci->stars.requires_sorts, 1 << sid);
+
+ cj->hydro.dx_max_sort_old = cj->hydro.dx_max_sort;
+ ci->stars.dx_max_sort_old = ci->stars.dx_max_sort;
+
+ /* Activate the drifts if the cells are local. */
+ if (ci->nodeID == engine_rank) cell_activate_drift_spart(ci, s);
+ if (cj->nodeID == engine_rank) cell_activate_drift_part(cj, s);
+ if (cj->nodeID == engine_rank && with_timestep_sync)
+ cell_activate_sync_part(cj, s);
+
+ /* Do we need to sort the cells? */
+ cell_activate_hydro_sorts(cj, sid, s);
+ cell_activate_stars_sorts(ci, sid, s);
+ }
+
+ if (cj_active) {
+
+ /* We are going to interact this pair, so store some values. */
+ atomic_or(&cj->stars.requires_sorts, 1 << sid);
+ atomic_or(&ci->hydro.requires_sorts, 1 << sid);
+
+ ci->hydro.dx_max_sort_old = ci->hydro.dx_max_sort;
+ cj->stars.dx_max_sort_old = cj->stars.dx_max_sort;
+
+ /* Activate the drifts if the cells are local. */
+ if (ci->nodeID == engine_rank) cell_activate_drift_part(ci, s);
+ if (cj->nodeID == engine_rank) cell_activate_drift_spart(cj, s);
+ if (ci->nodeID == engine_rank && with_timestep_sync)
+ cell_activate_sync_part(ci, s);
+
+ /* Do we need to sort the cells? */
+ cell_activate_hydro_sorts(ci, sid, s);
+ cell_activate_stars_sorts(cj, sid, s);
+ }
+ }
+ } /* Otherwise, pair interation */
+}
+
+/**
+ * @brief Traverse a sub-cell task and activate the black_holes drift tasks that
+ * are required by a black_holes task
+ *
+ * @param ci The first #cell we recurse in.
+ * @param cj The second #cell we recurse in.
+ * @param s The task #scheduler.
+ * @param with_timestep_sync Are we running with time-step synchronization on?
+ */
+void cell_activate_subcell_black_holes_tasks(struct cell *ci, struct cell *cj,
+ struct scheduler *s,
+ const int with_timestep_sync) {
+ const struct engine *e = s->space->e;
+
+ /* Store the current dx_max and h_max values. */
+ ci->black_holes.dx_max_part_old = ci->black_holes.dx_max_part;
+ ci->black_holes.h_max_old = ci->black_holes.h_max;
+ ci->hydro.dx_max_part_old = ci->hydro.dx_max_part;
+ ci->hydro.h_max_old = ci->hydro.h_max;
+
+ if (cj != NULL) {
+ cj->black_holes.dx_max_part_old = cj->black_holes.dx_max_part;
+ cj->black_holes.h_max_old = cj->black_holes.h_max;
+ cj->hydro.dx_max_part_old = cj->hydro.dx_max_part;
+ cj->hydro.h_max_old = cj->hydro.h_max;
+ }
+
+ /* Self interaction? */
+ if (cj == NULL) {
+ /* Do anything? */
+ if (!cell_is_active_black_holes(ci, e) || ci->hydro.count == 0 ||
+ ci->black_holes.count == 0)
+ return;
+
+ /* Recurse? */
+ if (cell_can_recurse_in_self_black_holes_task(ci)) {
+ /* Loop over all progenies and pairs of progenies */
+ for (int j = 0; j < 8; j++) {
+ if (ci->progeny[j] != NULL) {
+ cell_activate_subcell_black_holes_tasks(ci->progeny[j], NULL, s,
+ with_timestep_sync);
+ for (int k = j + 1; k < 8; k++)
+ if (ci->progeny[k] != NULL)
+ cell_activate_subcell_black_holes_tasks(
+ ci->progeny[j], ci->progeny[k], s, with_timestep_sync);
+ }
+ }
+ } else {
+ /* We have reached the bottom of the tree: activate drift */
+ cell_activate_drift_bpart(ci, s);
+ cell_activate_drift_part(ci, s);
+ }
+ }
+
+ /* Otherwise, pair interation */
+ else {
+ /* Should we even bother? */
+ if (!cell_is_active_black_holes(ci, e) &&
+ !cell_is_active_black_holes(cj, e))
+ return;
+
+ /* Get the orientation of the pair. */
+ double shift[3];
+ const int sid = space_getsid(s->space, &ci, &cj, shift);
+
+ /* recurse? */
+ if (cell_can_recurse_in_pair_black_holes_task(ci, cj) &&
+ cell_can_recurse_in_pair_black_holes_task(cj, ci)) {
+ const struct cell_split_pair *csp = &cell_split_pairs[sid];
+ for (int k = 0; k < csp->count; k++) {
+ const int pid = csp->pairs[k].pid;
+ const int pjd = csp->pairs[k].pjd;
+ if (ci->progeny[pid] != NULL && cj->progeny[pjd] != NULL)
+ cell_activate_subcell_black_holes_tasks(
+ ci->progeny[pid], cj->progeny[pjd], s, with_timestep_sync);
+ }
+ }
+
+ /* Otherwise, activate the drifts. */
+ else if (cell_is_active_black_holes(ci, e) ||
+ cell_is_active_black_holes(cj, e)) {
+
+ /* Activate the drifts if the cells are local. */
+ if (ci->nodeID == engine_rank) cell_activate_drift_bpart(ci, s);
+ if (cj->nodeID == engine_rank) cell_activate_drift_part(cj, s);
+ if (cj->nodeID == engine_rank && with_timestep_sync)
+ cell_activate_sync_part(cj, s);
+
+ /* Activate the drifts if the cells are local. */
+ if (ci->nodeID == engine_rank) cell_activate_drift_part(ci, s);
+ if (cj->nodeID == engine_rank) cell_activate_drift_bpart(cj, s);
+ if (ci->nodeID == engine_rank && with_timestep_sync)
+ cell_activate_sync_part(ci, s);
+ }
+ } /* Otherwise, pair interation */
+}
+
+/**
+ * @brief Traverse a sub-cell task and activate the sinks drift tasks that
+ * are required by a sinks task
+ *
+ * @param ci The first #cell we recurse in.
+ * @param cj The second #cell we recurse in.
+ * @param s The task #scheduler.
+ * @param with_timestep_sync Are we running with time-step synchronization on?
+ */
+void cell_activate_subcell_sinks_tasks(struct cell *ci, struct cell *cj,
+ struct scheduler *s,
+ const int with_timestep_sync) {
+ const struct engine *e = s->space->e;
+
+ /* Store the current dx_max and h_max values. */
+ ci->sinks.dx_max_part_old = ci->sinks.dx_max_part;
+ ci->sinks.r_cut_max_old = ci->sinks.r_cut_max;
+ ci->hydro.dx_max_part_old = ci->hydro.dx_max_part;
+ ci->hydro.h_max_old = ci->hydro.h_max;
+
+ if (cj != NULL) {
+ cj->sinks.dx_max_part_old = cj->sinks.dx_max_part;
+ cj->sinks.r_cut_max_old = cj->sinks.r_cut_max;
+ cj->hydro.dx_max_part_old = cj->hydro.dx_max_part;
+ cj->hydro.h_max_old = cj->hydro.h_max;
+ }
+
+ /* Self interaction? */
+ if (cj == NULL) {
+
+ const int ci_active =
+ cell_is_active_sinks(ci, e) || cell_is_active_hydro(ci, e);
+
+ /* Do anything? */
+ if (!ci_active || ci->hydro.count == 0 || ci->sinks.count == 0) return;
+
+ /* Recurse? */
+ if (cell_can_recurse_in_self_sinks_task(ci)) {
+ /* Loop over all progenies and pairs of progenies */
+ for (int j = 0; j < 8; j++) {
+ if (ci->progeny[j] != NULL) {
+ cell_activate_subcell_sinks_tasks(ci->progeny[j], NULL, s,
+ with_timestep_sync);
+ for (int k = j + 1; k < 8; k++)
+ if (ci->progeny[k] != NULL)
+ cell_activate_subcell_sinks_tasks(ci->progeny[j], ci->progeny[k],
+ s, with_timestep_sync);
+ }
+ }
+ } else {
+ /* We have reached the bottom of the tree: activate drift */
+ cell_activate_drift_sink(ci, s);
+ cell_activate_drift_part(ci, s);
+ if (with_timestep_sync) cell_activate_sync_part(ci, s);
+ }
+ }
+
+ /* Otherwise, pair interation */
+ else {
+ /* Get the orientation of the pair. */
+ double shift[3];
+ const int sid = space_getsid(s->space, &ci, &cj, shift);
+
+ const int ci_active =
+ cell_is_active_sinks(ci, e) || cell_is_active_hydro(ci, e);
+ const int cj_active =
+ cell_is_active_sinks(cj, e) || cell_is_active_hydro(cj, e);
+
+ /* Should we even bother? */
+ if (!ci_active && !cj_active) return;
+
+ /* recurse? */
+ if (cell_can_recurse_in_pair_sinks_task(ci, cj) &&
+ cell_can_recurse_in_pair_sinks_task(cj, ci)) {
+
+ const struct cell_split_pair *csp = &cell_split_pairs[sid];
+ for (int k = 0; k < csp->count; k++) {
+ const int pid = csp->pairs[k].pid;
+ const int pjd = csp->pairs[k].pjd;
+ if (ci->progeny[pid] != NULL && cj->progeny[pjd] != NULL)
+ cell_activate_subcell_sinks_tasks(ci->progeny[pid], cj->progeny[pjd],
+ s, with_timestep_sync);
+ }
+ }
+
+ /* Otherwise, activate the sorts and drifts. */
+ else {
+
+ if (ci_active) {
+
+ /* We are going to interact this pair, so store some values. */
+ atomic_or(&cj->hydro.requires_sorts, 1 << sid);
+ cj->hydro.dx_max_sort_old = cj->hydro.dx_max_sort;
+
+ /* Activate the drifts if the cells are local. */
+ if (ci->nodeID == engine_rank) cell_activate_drift_sink(ci, s);
+ if (cj->nodeID == engine_rank) cell_activate_drift_part(cj, s);
+ if (cj->nodeID == engine_rank && with_timestep_sync)
+ cell_activate_sync_part(cj, s);
+
+ /* Do we need to sort the cells? */
+ cell_activate_hydro_sorts(cj, sid, s);
+ }
+
+ if (cj_active) {
+
+ /* We are going to interact this pair, so store some values. */
+ atomic_or(&ci->hydro.requires_sorts, 1 << sid);
+ ci->hydro.dx_max_sort_old = ci->hydro.dx_max_sort;
+
+ /* Activate the drifts if the cells are local. */
+ if (ci->nodeID == engine_rank) cell_activate_drift_part(ci, s);
+ if (cj->nodeID == engine_rank) cell_activate_drift_sink(cj, s);
+ if (ci->nodeID == engine_rank && with_timestep_sync)
+ cell_activate_sync_part(ci, s);
+
+ /* Do we need to sort the cells? */
+ cell_activate_hydro_sorts(ci, sid, s);
+ }
+ }
+ } /* Otherwise, pair interation */
+}
+
+/**
+ * @brief Traverse a sub-cell task and activate the gravity drift tasks that
+ * are required by a self gravity task.
+ *
+ * @param ci The first #cell we recurse in.
+ * @param cj The second #cell we recurse in.
+ * @param s The task #scheduler.
+ */
+void cell_activate_subcell_grav_tasks(struct cell *ci, struct cell *cj,
+ struct scheduler *s) {
+ /* Some constants */
+ const struct space *sp = s->space;
+ const struct engine *e = sp->e;
+
+ /* Self interaction? */
+ if (cj == NULL) {
+ /* Do anything? */
+ if (ci->grav.count == 0 || !cell_is_active_gravity(ci, e)) return;
+
+ /* Recurse? */
+ if (ci->split) {
+ /* Loop over all progenies and pairs of progenies */
+ for (int j = 0; j < 8; j++) {
+ if (ci->progeny[j] != NULL) {
+ cell_activate_subcell_grav_tasks(ci->progeny[j], NULL, s);
+ for (int k = j + 1; k < 8; k++)
+ if (ci->progeny[k] != NULL)
+ cell_activate_subcell_grav_tasks(ci->progeny[j], ci->progeny[k],
+ s);
+ }
+ }
+ } else {
+ /* We have reached the bottom of the tree: activate gpart drift */
+ cell_activate_drift_gpart(ci, s);
+ }
+ }
+
+ /* Pair interaction */
+ else {
+ /* Anything to do here? */
+ if (!cell_is_active_gravity(ci, e) && !cell_is_active_gravity(cj, e))
+ return;
+ if (ci->grav.count == 0 || cj->grav.count == 0) return;
+
+ /* Atomically drift the multipole in ci */
+ lock_lock(&ci->grav.mlock);
+ if (ci->grav.ti_old_multipole < e->ti_current) cell_drift_multipole(ci, e);
+ if (lock_unlock(&ci->grav.mlock) != 0) error("Impossible to unlock m-pole");
+
+ /* Atomically drift the multipole in cj */
+ lock_lock(&cj->grav.mlock);
+ if (cj->grav.ti_old_multipole < e->ti_current) cell_drift_multipole(cj, e);
+ if (lock_unlock(&cj->grav.mlock) != 0) error("Impossible to unlock m-pole");
+
+ /* Can we use multipoles ? */
+ if (cell_can_use_pair_mm(ci, cj, e, sp, /*use_rebuild_data=*/0,
+ /*is_tree_walk=*/1)) {
+
+ /* Ok, no need to drift anything */
+ return;
+ }
+ /* Otherwise, activate the gpart drifts if we are at the bottom. */
+ else if (!ci->split && !cj->split) {
+ /* Activate the drifts if the cells are local. */
+ if (cell_is_active_gravity(ci, e) || cell_is_active_gravity(cj, e)) {
+ if (ci->nodeID == engine_rank) cell_activate_drift_gpart(ci, s);
+ if (cj->nodeID == engine_rank) cell_activate_drift_gpart(cj, s);
+ }
+ }
+ /* Ok, we can still recurse */
+ else {
+ /* Recover the multipole information */
+ const struct gravity_tensors *const multi_i = ci->grav.multipole;
+ const struct gravity_tensors *const multi_j = cj->grav.multipole;
+ const double ri_max = multi_i->r_max;
+ const double rj_max = multi_j->r_max;
+
+ if (ri_max > rj_max) {
+ if (ci->split) {
+ /* Loop over ci's children */
+ for (int k = 0; k < 8; k++) {
+ if (ci->progeny[k] != NULL)
+ cell_activate_subcell_grav_tasks(ci->progeny[k], cj, s);
+ }
+
+ } else if (cj->split) {
+ /* Loop over cj's children */
+ for (int k = 0; k < 8; k++) {
+ if (cj->progeny[k] != NULL)
+ cell_activate_subcell_grav_tasks(ci, cj->progeny[k], s);
+ }
+
+ } else {
+ error("Fundamental error in the logic");
+ }
+ } else if (rj_max >= ri_max) {
+ if (cj->split) {
+ /* Loop over cj's children */
+ for (int k = 0; k < 8; k++) {
+ if (cj->progeny[k] != NULL)
+ cell_activate_subcell_grav_tasks(ci, cj->progeny[k], s);
+ }
+
+ } else if (ci->split) {
+ /* Loop over ci's children */
+ for (int k = 0; k < 8; k++) {
+ if (ci->progeny[k] != NULL)
+ cell_activate_subcell_grav_tasks(ci->progeny[k], cj, s);
+ }
+
+ } else {
+ error("Fundamental error in the logic");
+ }
+ }
+ }
+ }
+}
+
+/**
+ * @brief Traverse a sub-cell task and activate the gravity drift tasks that
+ * are required by an external gravity task.
+ *
+ * @param ci The #cell we recurse in.
+ * @param s The task #scheduler.
+ */
+void cell_activate_subcell_external_grav_tasks(struct cell *ci,
+ struct scheduler *s) {
+ /* Some constants */
+ const struct space *sp = s->space;
+ const struct engine *e = sp->e;
+
+ /* Do anything? */
+ if (!cell_is_active_gravity(ci, e)) return;
+
+ /* Recurse? */
+ if (ci->split) {
+ /* Loop over all progenies (no need for pairs for self-gravity) */
+ for (int j = 0; j < 8; j++) {
+ if (ci->progeny[j] != NULL) {
+ cell_activate_subcell_external_grav_tasks(ci->progeny[j], s);
+ }
+ }
+ } else {
+ /* We have reached the bottom of the tree: activate gpart drift */
+ cell_activate_drift_gpart(ci, s);
+ }
+}
+
+/**
+ * @brief Traverse a sub-cell task and activate the radiative transfer tasks
+ *
+ * @param ci The first #cell we recurse in.
+ * @param cj The second #cell we recurse in.
+ * @param s The task #scheduler.
+ */
+void cell_activate_subcell_rt_tasks(struct cell *ci, struct cell *cj,
+ struct scheduler *s) {
+ const struct engine *e = s->space->e;
+
+ /* Self interaction? */
+ if (cj == NULL) {
+ /* Do anything? */
+ if (ci->hydro.count == 0 || !cell_is_active_hydro(ci, e)) return;
+
+ /* Recurse? */
+ if (cell_can_recurse_in_self_hydro_task(ci)) {
+ /* Loop over all progenies and pairs of progenies */
+ for (int j = 0; j < 8; j++) {
+ if (ci->progeny[j] != NULL) {
+ cell_activate_subcell_rt_tasks(ci->progeny[j], NULL, s);
+ for (int k = j + 1; k < 8; k++)
+ if (ci->progeny[k] != NULL)
+ cell_activate_subcell_rt_tasks(ci->progeny[j], ci->progeny[k], s);
+ }
+ }
+ } else {
+ /* We have reached the bottom of the tree: activate tasks */
+ for (struct link *l = ci->hydro.rt_inject; l != NULL; l = l->next) {
+ struct task *t = l->t;
+ const int ci_active = cell_is_active_hydro(ci, e);
+#ifdef WITH_MPI
+ const int ci_nodeID = ci->nodeID;
+#else
+ const int ci_nodeID = e->nodeID;
+#endif
+ /* Only activate tasks that involve a local active cell. */
+ if (ci_active && ci_nodeID == e->nodeID) {
+ scheduler_activate(s, t);
+ }
+ }
+ }
+ }
+
+ /* Otherwise, pair interaction */
+ else {
+ /* Should we even bother? */
+ if (!cell_is_active_hydro(ci, e) && !cell_is_active_hydro(cj, e)) return;
+ if (ci->hydro.count == 0 || cj->hydro.count == 0) return;
+
+ /* Get the orientation of the pair. */
+ double shift[3];
+ const int sid = space_getsid(s->space, &ci, &cj, shift);
+
+ /* recurse? */
+ if (cell_can_recurse_in_pair_hydro_task(ci) &&
+ cell_can_recurse_in_pair_hydro_task(cj)) {
+ const struct cell_split_pair *csp = &cell_split_pairs[sid];
+ for (int k = 0; k < csp->count; k++) {
+ const int pid = csp->pairs[k].pid;
+ const int pjd = csp->pairs[k].pjd;
+ if (ci->progeny[pid] != NULL && cj->progeny[pjd] != NULL)
+ cell_activate_subcell_rt_tasks(ci->progeny[pid], cj->progeny[pjd], s);
+ }
+ }
+
+ /* Otherwise, activate the RT tasks. */
+ else if (cell_is_active_hydro(ci, e) || cell_is_active_hydro(cj, e)) {
+
+ /* Activate the drifts if the cells are local. */
+ for (struct link *l = ci->hydro.rt_inject; l != NULL; l = l->next) {
+ struct task *t = l->t;
+ const int ci_active = cell_is_active_hydro(ci, e);
+ const int cj_active = (cj != NULL) ? cell_is_active_hydro(cj, e) : 0;
+#ifdef WITH_MPI
+ const int ci_nodeID = ci->nodeID;
+ const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
+#else
+ const int ci_nodeID = e->nodeID;
+ const int cj_nodeID = e->nodeID;
+#endif
+
+ /* Only activate tasks that involve a local active cell. */
+ if ((ci_active && ci_nodeID == e->nodeID) ||
+ (cj_active && cj_nodeID == e->nodeID)) {
+ scheduler_activate(s, t);
+ }
+ }
+ }
+ }
+}
+
+/**
+ * @brief Un-skips all the hydro tasks associated with a given cell and checks
+ * if the space needs to be rebuilt.
+ *
+ * @param c the #cell.
+ * @param s the #scheduler.
+ *
+ * @return 1 If the space needs rebuilding. 0 otherwise.
+ */
+int cell_unskip_hydro_tasks(struct cell *c, struct scheduler *s) {
+ struct engine *e = s->space->e;
+ const int nodeID = e->nodeID;
+ const int with_feedback = e->policy & engine_policy_feedback;
+ const int with_timestep_limiter =
+ (e->policy & engine_policy_timestep_limiter);
+
+#ifdef WITH_MPI
+ const int with_star_formation = e->policy & engine_policy_star_formation;
+#endif
+ int rebuild = 0;
+
+ /* Un-skip the density tasks involved with this cell. */
+ for (struct link *l = c->hydro.density; l != NULL; l = l->next) {
+ struct task *t = l->t;
+ struct cell *ci = t->ci;
+ struct cell *cj = t->cj;
+ const int ci_active = cell_is_active_hydro(ci, e);
+ const int cj_active = (cj != NULL) ? cell_is_active_hydro(cj, e) : 0;
+#ifdef WITH_MPI
+ const int ci_nodeID = ci->nodeID;
+ const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
+#else
+ const int ci_nodeID = nodeID;
+ const int cj_nodeID = nodeID;
+#endif
+
+ /* Only activate tasks that involve a local active cell. */
+ if ((ci_active && ci_nodeID == nodeID) ||
+ (cj_active && cj_nodeID == nodeID)) {
+ scheduler_activate(s, t);
+
+ /* Activate hydro drift */
+ if (t->type == task_type_self) {
+ if (ci_nodeID == nodeID) cell_activate_drift_part(ci, s);
+ if (ci_nodeID == nodeID && with_timestep_limiter)
+ cell_activate_limiter(ci, s);
+ }
+
+ /* Set the correct sorting flags and activate hydro drifts */
+ else if (t->type == task_type_pair) {
+ /* Store some values. */
+ atomic_or(&ci->hydro.requires_sorts, 1 << t->flags);
+ atomic_or(&cj->hydro.requires_sorts, 1 << t->flags);
+ ci->hydro.dx_max_sort_old = ci->hydro.dx_max_sort;
+ cj->hydro.dx_max_sort_old = cj->hydro.dx_max_sort;
+
+ /* Activate the drift tasks. */
+ if (ci_nodeID == nodeID) cell_activate_drift_part(ci, s);
+ if (cj_nodeID == nodeID) cell_activate_drift_part(cj, s);
+
+ /* Activate the limiter tasks. */
+ if (ci_nodeID == nodeID && with_timestep_limiter)
+ cell_activate_limiter(ci, s);
+ if (cj_nodeID == nodeID && with_timestep_limiter)
+ cell_activate_limiter(cj, s);
+
+ /* Check the sorts and activate them if needed. */
+ cell_activate_hydro_sorts(ci, t->flags, s);
+ cell_activate_hydro_sorts(cj, t->flags, s);
+ }
+
+ /* Store current values of dx_max and h_max. */
+ else if (t->type == task_type_sub_self) {
+ cell_activate_subcell_hydro_tasks(ci, NULL, s, with_timestep_limiter);
+ }
+
+ /* Store current values of dx_max and h_max. */
+ else if (t->type == task_type_sub_pair) {
+ cell_activate_subcell_hydro_tasks(ci, cj, s, with_timestep_limiter);
+ }
+ }
+
+ /* Only interested in pair interactions as of here. */
+ if (t->type == task_type_pair || t->type == task_type_sub_pair) {
+ /* Check whether there was too much particle motion, i.e. the
+ cell neighbour conditions were violated. */
+ if (cell_need_rebuild_for_hydro_pair(ci, cj)) rebuild = 1;
+
+#ifdef WITH_MPI
+ /* Activate the send/recv tasks. */
+ if (ci_nodeID != nodeID) {
+ /* If the local cell is active, receive data from the foreign cell. */
+ if (cj_active) {
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_xv);
+ if (ci_active) {
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_rho);
+
+#ifdef EXTRA_HYDRO_LOOP
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_gradient);
+#endif
+ }
+ }
+
+ /* If the foreign cell is active, we want its particles for the limiter
+ */
+ if (ci_active && with_timestep_limiter)
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_limiter);
+
+ /* If the foreign cell is active, we want its ti_end values. */
+ if (ci_active)
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_tend_part);
+
+ /* Is the foreign cell active and will need stuff from us? */
+ if (ci_active) {
+
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_xv, ci_nodeID);
+
+ /* Drift the cell which will be sent; note that not all sent
+ particles will be drifted, only those that are needed. */
+ cell_activate_drift_part(cj, s);
+ if (with_timestep_limiter) cell_activate_limiter(cj, s);
+
+ /* If the local cell is also active, more stuff will be needed. */
+ if (cj_active) {
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_rho,
+ ci_nodeID);
+
+#ifdef EXTRA_HYDRO_LOOP
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_gradient,
+ ci_nodeID);
+#endif
+ }
+ }
+
+ /* If the local cell is active, send its particles for the limiting. */
+ if (cj_active && with_timestep_limiter)
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_limiter,
+ ci_nodeID);
+
+ /* If the local cell is active, send its ti_end values. */
+ if (cj_active)
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_tend_part,
+ ci_nodeID);
+
+ /* Propagating new star counts? */
+ if (with_star_formation && with_feedback) {
+ if (ci_active && ci->hydro.count > 0) {
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_sf_counts);
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_tend_spart);
+ }
+ if (cj_active && cj->hydro.count > 0) {
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_sf_counts,
+ ci_nodeID);
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_tend_spart,
+ ci_nodeID);
+ }
+ }
+
+ } else if (cj_nodeID != nodeID) {
+ /* If the local cell is active, receive data from the foreign cell. */
+ if (ci_active) {
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_xv);
+ if (cj_active) {
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_rho);
+
+#ifdef EXTRA_HYDRO_LOOP
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_gradient);
+#endif
+ }
+ }
+
+ /* If the foreign cell is active, we want its particles for the limiter
+ */
+ if (cj_active && with_timestep_limiter)
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_limiter);
+
+ /* If the foreign cell is active, we want its ti_end values. */
+ if (cj_active)
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_tend_part);
+
+ /* Is the foreign cell active and will need stuff from us? */
+ if (cj_active) {
+
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_xv, cj_nodeID);
+
+ /* Drift the cell which will be sent; note that not all sent
+ particles will be drifted, only those that are needed. */
+ cell_activate_drift_part(ci, s);
+ if (with_timestep_limiter) cell_activate_limiter(ci, s);
+
+ /* If the local cell is also active, more stuff will be needed. */
+ if (ci_active) {
+
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_rho,
+ cj_nodeID);
+
+#ifdef EXTRA_HYDRO_LOOP
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_gradient,
+ cj_nodeID);
+#endif
+ }
+ }
+
+ /* If the local cell is active, send its particles for the limiting. */
+ if (ci_active && with_timestep_limiter)
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_limiter,
+ cj_nodeID);
+
+ /* If the local cell is active, send its ti_end values. */
+ if (ci_active)
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_tend_part,
+ cj_nodeID);
+
+ /* Propagating new star counts? */
+ if (with_star_formation && with_feedback) {
+ if (cj_active && cj->hydro.count > 0) {
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_sf_counts);
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_tend_spart);
+ }
+ if (ci_active && ci->hydro.count > 0) {
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_sf_counts,
+ cj_nodeID);
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_tend_spart,
+ cj_nodeID);
+ }
+ }
+ }
+#endif
+ }
+ }
+
+ /* Unskip all the other task types. */
+ if (c->nodeID == nodeID && cell_is_active_hydro(c, e)) {
+ for (struct link *l = c->hydro.gradient; l != NULL; l = l->next)
+ scheduler_activate(s, l->t);
+ for (struct link *l = c->hydro.force; l != NULL; l = l->next)
+ scheduler_activate(s, l->t);
+ for (struct link *l = c->hydro.limiter; l != NULL; l = l->next)
+ scheduler_activate(s, l->t);
+
+ if (c->hydro.extra_ghost != NULL)
+ scheduler_activate(s, c->hydro.extra_ghost);
+ if (c->hydro.ghost_in != NULL) cell_activate_hydro_ghosts(c, s, e);
+ if (c->kick1 != NULL) scheduler_activate(s, c->kick1);
+ if (c->kick2 != NULL) scheduler_activate(s, c->kick2);
+ if (c->timestep != NULL) scheduler_activate(s, c->timestep);
+ if (c->hydro.end_force != NULL) scheduler_activate(s, c->hydro.end_force);
+ if (c->hydro.cooling_in != NULL) cell_activate_cooling(c, s, e);
+#ifdef WITH_LOGGER
+ if (c->logger != NULL) scheduler_activate(s, c->logger);
+#endif
+
+ if (c->top->hydro.star_formation != NULL) {
+ cell_activate_star_formation_tasks(c->top, s, with_feedback);
+ }
+ if (c->top->hydro.sink_formation != NULL) {
+ cell_activate_sink_formation_tasks(c->top, s);
+ }
+ }
+
+ return rebuild;
+}
+
+/**
+ * @brief Un-skips all the gravity tasks associated with a given cell and checks
+ * if the space needs to be rebuilt.
+ *
+ * @param c the #cell.
+ * @param s the #scheduler.
+ *
+ * @return 1 If the space needs rebuilding. 0 otherwise.
+ */
+int cell_unskip_gravity_tasks(struct cell *c, struct scheduler *s) {
+ struct engine *e = s->space->e;
+ const int nodeID = e->nodeID;
+ int rebuild = 0;
+
+ /* Un-skip the gravity tasks involved with this cell. */
+ for (struct link *l = c->grav.grav; l != NULL; l = l->next) {
+ struct task *t = l->t;
+ struct cell *ci = t->ci;
+ struct cell *cj = t->cj;
+ const int ci_active = cell_is_active_gravity(ci, e);
+ const int cj_active = (cj != NULL) ? cell_is_active_gravity(cj, e) : 0;
+#ifdef WITH_MPI
+ const int ci_nodeID = ci->nodeID;
+ const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
+#else
+ const int ci_nodeID = nodeID;
+ const int cj_nodeID = nodeID;
+#endif
+
+ /* Only activate tasks that involve a local active cell. */
+ if ((ci_active && ci_nodeID == nodeID) ||
+ (cj_active && cj_nodeID == nodeID)) {
+ scheduler_activate(s, t);
+
+ /* Set the drifting flags */
+ if (t->type == task_type_self &&
+ t->subtype == task_subtype_external_grav) {
+ cell_activate_subcell_external_grav_tasks(ci, s);
+ } else if (t->type == task_type_self && t->subtype == task_subtype_grav) {
+ cell_activate_subcell_grav_tasks(ci, NULL, s);
+ } else if (t->type == task_type_pair) {
+ cell_activate_subcell_grav_tasks(ci, cj, s);
+ } else if (t->type == task_type_grav_mm) {
+#ifdef SWIFT_DEBUG_CHECKS
+ error("Incorrectly linked M-M task!");
+#endif
+ }
+ }
+
+ if (t->type == task_type_pair) {
+#ifdef WITH_MPI
+ /* Activate the send/recv tasks. */
+ if (ci_nodeID != nodeID) {
+ /* If the local cell is active, receive data from the foreign cell. */
+ if (cj_active)
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_gpart);
+
+ /* If the foreign cell is active, we want its ti_end values. */
+ if (ci_active)
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_tend_gpart);
+
+ /* Is the foreign cell active and will need stuff from us? */
+ if (ci_active) {
+
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_gpart,
+ ci_nodeID);
+
+ /* Drift the cell which will be sent at the level at which it is
+ sent, i.e. drift the cell specified in the send task (l->t)
+ itself. */
+ cell_activate_drift_gpart(cj, s);
+ }
+
+ /* If the local cell is active, send its ti_end values. */
+ if (cj_active)
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_tend_gpart,
+ ci_nodeID);
+
+ } else if (cj_nodeID != nodeID) {
+ /* If the local cell is active, receive data from the foreign cell. */
+ if (ci_active)
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_gpart);
+
+ /* If the foreign cell is active, we want its ti_end values. */
+ if (cj_active)
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_tend_gpart);
+
+ /* Is the foreign cell active and will need stuff from us? */
+ if (cj_active) {
+
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_gpart,
+ cj_nodeID);
+
+ /* Drift the cell which will be sent at the level at which it is
+ sent, i.e. drift the cell specified in the send task (l->t)
+ itself. */
+ cell_activate_drift_gpart(ci, s);
+ }
+
+ /* If the local cell is active, send its ti_end values. */
+ if (ci_active)
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_tend_gpart,
+ cj_nodeID);
+ }
+#endif
+ }
+ }
+
+ for (struct link *l = c->grav.mm; l != NULL; l = l->next) {
+ struct task *t = l->t;
+ struct cell *ci = t->ci;
+ struct cell *cj = t->cj;
+ const int ci_active = cell_is_active_gravity_mm(ci, e);
+ const int cj_active = cell_is_active_gravity_mm(cj, e);
+#ifdef WITH_MPI
+ const int ci_nodeID = ci->nodeID;
+ const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
+#else
+ const int ci_nodeID = nodeID;
+ const int cj_nodeID = nodeID;
+#endif
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (t->type != task_type_grav_mm) error("Incorrectly linked gravity task!");
+#endif
+
+ /* Only activate tasks that involve a local active cell. */
+ if ((ci_active && ci_nodeID == nodeID) ||
+ (cj_active && cj_nodeID == nodeID)) {
+ scheduler_activate(s, t);
+ }
+ }
+
+ /* Unskip all the other task types. */
+ if (c->nodeID == nodeID && cell_is_active_gravity(c, e)) {
+ if (c->grav.init != NULL) scheduler_activate(s, c->grav.init);
+ if (c->grav.init_out != NULL) scheduler_activate(s, c->grav.init_out);
+ if (c->kick1 != NULL) scheduler_activate(s, c->kick1);
+ if (c->kick2 != NULL) scheduler_activate(s, c->kick2);
+ if (c->timestep != NULL) scheduler_activate(s, c->timestep);
+ if (c->grav.down != NULL) scheduler_activate(s, c->grav.down);
+ if (c->grav.down_in != NULL) scheduler_activate(s, c->grav.down_in);
+ if (c->grav.long_range != NULL) scheduler_activate(s, c->grav.long_range);
+ if (c->grav.end_force != NULL) scheduler_activate(s, c->grav.end_force);
+#ifdef WITH_LOGGER
+ if (c->logger != NULL) scheduler_activate(s, c->logger);
+#endif
+ }
+
+ return rebuild;
+}
+
+/**
+ * @brief Un-skips all the stars tasks associated with a given cell and checks
+ * if the space needs to be rebuilt.
+ *
+ * @param c the #cell.
+ * @param s the #scheduler.
+ * @param with_star_formation Are we running with star formation switched on?
+ *
+ * @return 1 If the space needs rebuilding. 0 otherwise.
+ */
+int cell_unskip_stars_tasks(struct cell *c, struct scheduler *s,
+ const int with_star_formation) {
+
+ struct engine *e = s->space->e;
+ const int with_timestep_sync = (e->policy & engine_policy_timestep_sync);
+ const int nodeID = e->nodeID;
+ int rebuild = 0;
+
+ if (c->stars.drift != NULL) {
+ if (cell_is_active_stars(c, e) ||
+ (with_star_formation && cell_is_active_hydro(c, e))) {
+
+ cell_activate_drift_spart(c, s);
+ }
+ }
+
+ /* Un-skip the density tasks involved with this cell. */
+ for (struct link *l = c->stars.density; l != NULL; l = l->next) {
+ struct task *t = l->t;
+ struct cell *ci = t->ci;
+ struct cell *cj = t->cj;
+#ifdef WITH_MPI
+ const int ci_nodeID = ci->nodeID;
+ const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
+#else
+ const int ci_nodeID = nodeID;
+ const int cj_nodeID = nodeID;
+#endif
+
+ const int ci_active = cell_is_active_stars(ci, e) ||
+ (with_star_formation && cell_is_active_hydro(ci, e));
+
+ const int cj_active =
+ (cj != NULL) && (cell_is_active_stars(cj, e) ||
+ (with_star_formation && cell_is_active_hydro(cj, e)));
+
+ /* Activate the drifts */
+ if (t->type == task_type_self && ci_active) {
+ cell_activate_drift_spart(ci, s);
+ cell_activate_drift_part(ci, s);
+ if (with_timestep_sync) cell_activate_sync_part(ci, s);
+ }
+
+ /* Only activate tasks that involve a local active cell. */
+ if ((ci_active || cj_active) &&
+ (ci_nodeID == nodeID || cj_nodeID == nodeID)) {
+ scheduler_activate(s, t);
+
+ if (t->type == task_type_pair) {
+ /* Do ci */
+ if (ci_active) {
+ /* stars for ci */
+ atomic_or(&ci->stars.requires_sorts, 1 << t->flags);
+ ci->stars.dx_max_sort_old = ci->stars.dx_max_sort;
+
+ /* hydro for cj */
+ atomic_or(&cj->hydro.requires_sorts, 1 << t->flags);
+ cj->hydro.dx_max_sort_old = cj->hydro.dx_max_sort;
+
+ /* Activate the drift tasks. */
+ if (ci_nodeID == nodeID) cell_activate_drift_spart(ci, s);
+ if (cj_nodeID == nodeID) cell_activate_drift_part(cj, s);
+ if (cj_nodeID == nodeID && with_timestep_sync)
+ cell_activate_sync_part(cj, s);
+
+ /* Check the sorts and activate them if needed. */
+ cell_activate_stars_sorts(ci, t->flags, s);
+ cell_activate_hydro_sorts(cj, t->flags, s);
+ }
+
+ /* Do cj */
+ if (cj_active) {
+ /* hydro for ci */
+ atomic_or(&ci->hydro.requires_sorts, 1 << t->flags);
+ ci->hydro.dx_max_sort_old = ci->hydro.dx_max_sort;
+
+ /* stars for cj */
+ atomic_or(&cj->stars.requires_sorts, 1 << t->flags);
+ cj->stars.dx_max_sort_old = cj->stars.dx_max_sort;
+
+ /* Activate the drift tasks. */
+ if (cj_nodeID == nodeID) cell_activate_drift_spart(cj, s);
+ if (ci_nodeID == nodeID) cell_activate_drift_part(ci, s);
+ if (ci_nodeID == nodeID && with_timestep_sync)
+ cell_activate_sync_part(ci, s);
+
+ /* Check the sorts and activate them if needed. */
+ cell_activate_hydro_sorts(ci, t->flags, s);
+ cell_activate_stars_sorts(cj, t->flags, s);
+ }
+ }
+
+ else if (t->type == task_type_sub_self) {
+ cell_activate_subcell_stars_tasks(ci, NULL, s, with_star_formation,
+ with_timestep_sync);
+ }
+
+ else if (t->type == task_type_sub_pair) {
+ cell_activate_subcell_stars_tasks(ci, cj, s, with_star_formation,
+ with_timestep_sync);
+ }
+ }
+
+ /* Only interested in pair interactions as of here. */
+ if (t->type == task_type_pair || t->type == task_type_sub_pair) {
+ /* Check whether there was too much particle motion, i.e. the
+ cell neighbour conditions were violated. */
+ if (cell_need_rebuild_for_stars_pair(ci, cj)) rebuild = 1;
+ if (cell_need_rebuild_for_stars_pair(cj, ci)) rebuild = 1;
+
+#ifdef WITH_MPI
+ /* Activate the send/recv tasks. */
+ if (ci_nodeID != nodeID) {
+ if (cj_active) {
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_xv);
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_rho);
+
+ /* If the local cell is active, more stuff will be needed. */
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_spart,
+ ci_nodeID);
+ cell_activate_drift_spart(cj, s);
+
+ /* If the local cell is active, send its ti_end values. */
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_tend_spart,
+ ci_nodeID);
+ }
+
+ if (ci_active) {
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_spart);
+
+ /* If the foreign cell is active, we want its ti_end values. */
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_tend_spart);
+
+ /* Is the foreign cell active and will need stuff from us? */
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_xv, ci_nodeID);
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_rho, ci_nodeID);
+
+ /* Drift the cell which will be sent; note that not all sent
+ particles will be drifted, only those that are needed. */
+ cell_activate_drift_part(cj, s);
+ }
+
+ } else if (cj_nodeID != nodeID) {
+ /* If the local cell is active, receive data from the foreign cell. */
+ if (ci_active) {
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_xv);
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_rho);
+
+ /* If the local cell is active, more stuff will be needed. */
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_spart,
+ cj_nodeID);
+ cell_activate_drift_spart(ci, s);
+
+ /* If the local cell is active, send its ti_end values. */
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_tend_spart,
+ cj_nodeID);
+ }
+
+ if (cj_active) {
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_spart);
+
+ /* If the foreign cell is active, we want its ti_end values. */
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_tend_spart);
+
+ /* Is the foreign cell active and will need stuff from us? */
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_xv, cj_nodeID);
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_rho, cj_nodeID);
+
+ /* Drift the cell which will be sent; note that not all sent
+ particles will be drifted, only those that are needed. */
+ cell_activate_drift_part(ci, s);
+ }
+ }
+#endif
+ }
+ }
+
+ /* Un-skip the feedback tasks involved with this cell. */
+ for (struct link *l = c->stars.feedback; l != NULL; l = l->next) {
+ struct task *t = l->t;
+ struct cell *ci = t->ci;
+ struct cell *cj = t->cj;
+#ifdef WITH_MPI
+ const int ci_nodeID = ci->nodeID;
+ const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
+#else
+ const int ci_nodeID = nodeID;
+ const int cj_nodeID = nodeID;
+#endif
+
+ const int ci_active = cell_is_active_stars(ci, e) ||
+ (with_star_formation && cell_is_active_hydro(ci, e));
+
+ const int cj_active =
+ (cj != NULL) && (cell_is_active_stars(cj, e) ||
+ (with_star_formation && cell_is_active_hydro(cj, e)));
+
+ if (t->type == task_type_self && ci_active) {
+ scheduler_activate(s, t);
+ }
+
+ else if (t->type == task_type_sub_self && ci_active) {
+ scheduler_activate(s, t);
+ }
+
+ else if (t->type == task_type_pair || t->type == task_type_sub_pair) {
+ /* We only want to activate the task if the cell is active and is
+ going to update some gas on the *local* node */
+ if ((ci_nodeID == nodeID && cj_nodeID == nodeID) &&
+ (ci_active || cj_active)) {
+ scheduler_activate(s, t);
+
+ } else if ((ci_nodeID == nodeID && cj_nodeID != nodeID) && (cj_active)) {
+ scheduler_activate(s, t);
+
+ } else if ((ci_nodeID != nodeID && cj_nodeID == nodeID) && (ci_active)) {
+ scheduler_activate(s, t);
+ }
+ }
+
+ /* Nothing more to do here, all drifts and sorts activated above */
+ }
+
+ /* Unskip all the other task types. */
+ if (c->nodeID == nodeID) {
+ if (cell_is_active_stars(c, e) ||
+ (with_star_formation && cell_is_active_hydro(c, e))) {
+
+ if (c->stars.ghost != NULL) scheduler_activate(s, c->stars.ghost);
+ if (c->stars.stars_in != NULL) scheduler_activate(s, c->stars.stars_in);
+ if (c->stars.stars_out != NULL) scheduler_activate(s, c->stars.stars_out);
+ if (c->kick1 != NULL) scheduler_activate(s, c->kick1);
+ if (c->kick2 != NULL) scheduler_activate(s, c->kick2);
+ if (c->timestep != NULL) scheduler_activate(s, c->timestep);
+#ifdef WITH_LOGGER
+ if (c->logger != NULL) scheduler_activate(s, c->logger);
+#endif
+ }
+ }
+
+ return rebuild;
+}
+
+/**
+ * @brief Un-skips all the black hole tasks associated with a given cell and
+ * checks if the space needs to be rebuilt.
+ *
+ * @param c the #cell.
+ * @param s the #scheduler.
+ *
+ * @return 1 If the space needs rebuilding. 0 otherwise.
+ */
+int cell_unskip_black_holes_tasks(struct cell *c, struct scheduler *s) {
+
+ struct engine *e = s->space->e;
+ const int with_timestep_sync = (e->policy & engine_policy_timestep_sync);
+ const int nodeID = e->nodeID;
+ int rebuild = 0;
+
+ if (c->black_holes.drift != NULL && cell_is_active_black_holes(c, e)) {
+ cell_activate_drift_bpart(c, s);
+ }
+
+ /* Un-skip the density tasks involved with this cell. */
+ for (struct link *l = c->black_holes.density; l != NULL; l = l->next) {
+ struct task *t = l->t;
+ struct cell *ci = t->ci;
+ struct cell *cj = t->cj;
+ const int ci_active = cell_is_active_black_holes(ci, e);
+ const int cj_active = (cj != NULL) ? cell_is_active_black_holes(cj, e) : 0;
+#ifdef WITH_MPI
+ const int ci_nodeID = ci->nodeID;
+ const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
+#else
+ const int ci_nodeID = nodeID;
+ const int cj_nodeID = nodeID;
+#endif
+
+ /* Only activate tasks that involve a local active cell. */
+ if ((ci_active || cj_active) &&
+ (ci_nodeID == nodeID || cj_nodeID == nodeID)) {
+
+ scheduler_activate(s, t);
+
+ /* Activate the drifts */
+ if (t->type == task_type_self) {
+ cell_activate_drift_part(ci, s);
+ cell_activate_drift_bpart(ci, s);
+ }
+
+ /* Activate the drifts */
+ else if (t->type == task_type_pair) {
+
+ /* Activate the drift tasks. */
+ if (ci_nodeID == nodeID) cell_activate_drift_bpart(ci, s);
+ if (ci_nodeID == nodeID) cell_activate_drift_part(ci, s);
+
+ if (cj_nodeID == nodeID) cell_activate_drift_part(cj, s);
+ if (cj_nodeID == nodeID) cell_activate_drift_bpart(cj, s);
+ }
+
+ /* Store current values of dx_max and h_max. */
+ else if (t->type == task_type_sub_self) {
+ cell_activate_subcell_black_holes_tasks(ci, NULL, s,
+ with_timestep_sync);
+ }
+
+ /* Store current values of dx_max and h_max. */
+ else if (t->type == task_type_sub_pair) {
+ cell_activate_subcell_black_holes_tasks(ci, cj, s, with_timestep_sync);
+ }
+ }
+
+ /* Only interested in pair interactions as of here. */
+ if (t->type == task_type_pair || t->type == task_type_sub_pair) {
+
+ /* Check whether there was too much particle motion, i.e. the
+ cell neighbour conditions were violated. */
+ if (cell_need_rebuild_for_black_holes_pair(ci, cj)) rebuild = 1;
+ if (cell_need_rebuild_for_black_holes_pair(cj, ci)) rebuild = 1;
+
+ scheduler_activate(s, ci->hydro.super->black_holes.swallow_ghost[0]);
+ scheduler_activate(s, cj->hydro.super->black_holes.swallow_ghost[0]);
+
+#ifdef WITH_MPI
+ /* Activate the send/recv tasks. */
+ if (ci_nodeID != nodeID) {
+
+ /* Receive the foreign parts to compute BH accretion rates and do the
+ * swallowing */
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_rho);
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_part_swallow);
+
+ /* Send the local BHs to tag the particles to swallow and to do feedback
+ */
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_bpart_rho,
+ ci_nodeID);
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_bpart_feedback,
+ ci_nodeID);
+
+ /* Drift before you send */
+ cell_activate_drift_bpart(cj, s);
+
+ /* Send the new BH time-steps */
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_tend_bpart,
+ ci_nodeID);
+
+ /* Receive the foreign BHs to tag particles to swallow and for feedback
+ */
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_bpart_rho);
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_bpart_feedback);
+
+ /* Receive the foreign BH time-steps */
+ scheduler_activate_recv(s, ci->mpi.recv, task_subtype_tend_bpart);
+
+ /* Send the local part information */
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_rho, ci_nodeID);
+ scheduler_activate_send(s, cj->mpi.send, task_subtype_part_swallow,
+ ci_nodeID);
+
+ /* Drift the cell which will be sent; note that not all sent
+ particles will be drifted, only those that are needed. */
+ cell_activate_drift_part(cj, s);
+
+ } else if (cj_nodeID != nodeID) {
+
+ /* Receive the foreign parts to compute BH accretion rates and do the
+ * swallowing */
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_rho);
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_part_swallow);
+
+ /* Send the local BHs to tag the particles to swallow and to do feedback
+ */
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_bpart_rho,
+ cj_nodeID);
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_bpart_feedback,
+ cj_nodeID);
+
+ /* Drift before you send */
+ cell_activate_drift_bpart(ci, s);
+
+ /* Send the new BH time-steps */
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_tend_bpart,
+ cj_nodeID);
+
+ /* Receive the foreign BHs to tag particles to swallow and for feedback
+ */
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_bpart_rho);
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_bpart_feedback);
+
+ /* Receive the foreign BH time-steps */
+ scheduler_activate_recv(s, cj->mpi.recv, task_subtype_tend_bpart);
+
+ /* Send the local part information */
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_rho, cj_nodeID);
+ scheduler_activate_send(s, ci->mpi.send, task_subtype_part_swallow,
+ cj_nodeID);
+
+ /* Drift the cell which will be sent; note that not all sent
+ particles will be drifted, only those that are needed. */
+ cell_activate_drift_part(ci, s);
+ }
+#endif
+ }
+ }
+
+ /* Un-skip the swallow tasks involved with this cell. */
+ for (struct link *l = c->black_holes.swallow; l != NULL; l = l->next) {
+ struct task *t = l->t;
+ struct cell *ci = t->ci;
+ struct cell *cj = t->cj;
+ const int ci_active = cell_is_active_black_holes(ci, e);
+ const int cj_active = (cj != NULL) ? cell_is_active_black_holes(cj, e) : 0;
+#ifdef WITH_MPI
+ const int ci_nodeID = ci->nodeID;
+ const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
+#else
+ const int ci_nodeID = nodeID;
+ const int cj_nodeID = nodeID;
+#endif
+
+ /* Only activate tasks that involve a local active cell. */
+ if ((ci_active || cj_active) &&
+ (ci_nodeID == nodeID || cj_nodeID == nodeID)) {
+
+ scheduler_activate(s, t);
+ }
+ }
+
+ /* Un-skip the swallow tasks involved with this cell. */
+ for (struct link *l = c->black_holes.do_gas_swallow; l != NULL; l = l->next) {
+ struct task *t = l->t;
+ struct cell *ci = t->ci;
+ struct cell *cj = t->cj;
+ const int ci_active = cell_is_active_black_holes(ci, e);
+ const int cj_active = (cj != NULL) ? cell_is_active_black_holes(cj, e) : 0;
+#ifdef WITH_MPI
+ const int ci_nodeID = ci->nodeID;
+ const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
+#else
+ const int ci_nodeID = nodeID;
+ const int cj_nodeID = nodeID;
+#endif
+
+ /* Only activate tasks that involve a local active cell. */
+ if ((ci_active || cj_active) &&
+ (ci_nodeID == nodeID || cj_nodeID == nodeID)) {
+
+ scheduler_activate(s, t);
+ }
+ }
+
+ /* Un-skip the swallow tasks involved with this cell. */
+ for (struct link *l = c->black_holes.do_bh_swallow; l != NULL; l = l->next) {
+ struct task *t = l->t;
+ struct cell *ci = t->ci;
+ struct cell *cj = t->cj;
+ const int ci_active = cell_is_active_black_holes(ci, e);
+ const int cj_active = (cj != NULL) ? cell_is_active_black_holes(cj, e) : 0;
+#ifdef WITH_MPI
+ const int ci_nodeID = ci->nodeID;
+ const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
+#else
+ const int ci_nodeID = nodeID;
+ const int cj_nodeID = nodeID;
+#endif
+
+ /* Only activate tasks that involve a local active cell. */
+ if ((ci_active || cj_active) &&
+ (ci_nodeID == nodeID || cj_nodeID == nodeID)) {
+
+ scheduler_activate(s, t);
+ }
+ }
+
+ /* Un-skip the feedback tasks involved with this cell. */
+ for (struct link *l = c->black_holes.feedback; l != NULL; l = l->next) {
+ struct task *t = l->t;
+ struct cell *ci = t->ci;
+ struct cell *cj = t->cj;
+ const int ci_active = cell_is_active_black_holes(ci, e);
+ const int cj_active = (cj != NULL) ? cell_is_active_black_holes(cj, e) : 0;
+#ifdef WITH_MPI
+ const int ci_nodeID = ci->nodeID;
+ const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
+#else
+ const int ci_nodeID = nodeID;
+ const int cj_nodeID = nodeID;
+#endif
+
+ /* Only activate tasks that involve a local active cell. */
+ if ((ci_active || cj_active) &&
+ (ci_nodeID == nodeID || cj_nodeID == nodeID)) {
+
+ scheduler_activate(s, t);
+ }
+ }
+
+ /* Unskip all the other task types. */
+ if (c->nodeID == nodeID && cell_is_active_black_holes(c, e)) {
+
+ if (c->black_holes.density_ghost != NULL)
+ scheduler_activate(s, c->black_holes.density_ghost);
+ if (c->black_holes.swallow_ghost[0] != NULL)
+ scheduler_activate(s, c->black_holes.swallow_ghost[0]);
+ if (c->black_holes.swallow_ghost[1] != NULL)
+ scheduler_activate(s, c->black_holes.swallow_ghost[1]);
+ if (c->black_holes.swallow_ghost[2] != NULL)
+ scheduler_activate(s, c->black_holes.swallow_ghost[2]);
+ if (c->black_holes.black_holes_in != NULL)
+ scheduler_activate(s, c->black_holes.black_holes_in);
+ if (c->black_holes.black_holes_out != NULL)
+ scheduler_activate(s, c->black_holes.black_holes_out);
+ if (c->kick1 != NULL) scheduler_activate(s, c->kick1);
+ if (c->kick2 != NULL) scheduler_activate(s, c->kick2);
+ if (c->timestep != NULL) scheduler_activate(s, c->timestep);
+#ifdef WITH_LOGGER
+ if (c->logger != NULL) scheduler_activate(s, c->logger);
+#endif
+ }
+
+ return rebuild;
+}
+
+/**
+ * @brief Un-skips all the sinks tasks associated with a given cell and
+ * checks if the space needs to be rebuilt.
+ *
+ * @param c the #cell.
+ * @param s the #scheduler.
+ *
+ * @return 1 If the space needs rebuilding. 0 otherwise.
+ */
+int cell_unskip_sinks_tasks(struct cell *c, struct scheduler *s) {
+
+ struct engine *e = s->space->e;
+ const int nodeID = e->nodeID;
+ int rebuild = 0;
+
+ if (c->sinks.drift != NULL)
+ if (cell_is_active_sinks(c, e) || cell_is_active_hydro(c, e)) {
+ cell_activate_drift_sink(c, s);
+ }
+
+ /* Unskip all the other task types. */
+ if (c->nodeID == nodeID)
+ if (cell_is_active_sinks(c, e) || cell_is_active_hydro(c, e)) {
+ if (c->sinks.sink_in != NULL) scheduler_activate(s, c->sinks.sink_in);
+ if (c->sinks.sink_out != NULL) scheduler_activate(s, c->sinks.sink_out);
+ if (c->kick1 != NULL) scheduler_activate(s, c->kick1);
+ if (c->kick2 != NULL) scheduler_activate(s, c->kick2);
+ if (c->timestep != NULL) scheduler_activate(s, c->timestep);
+#ifdef WITH_LOGGER
+ if (c->logger != NULL) scheduler_activate(s, c->logger);
+#endif
+ }
+
+ return rebuild;
+}
+
+/**
+ * @brief Un-skips all the RT tasks associated with a given cell and checks
+ * if the space needs to be rebuilt.
+ *
+ * @param c the #cell.
+ * @param s the #scheduler.
+ *
+ * @return 1 If the space needs rebuilding. 0 otherwise.
+ */
+int cell_unskip_rt_tasks(struct cell *c, struct scheduler *s) {
+ struct engine *e = s->space->e;
+ const int nodeID = e->nodeID;
+
+ /* TODO: implement rebuild conditions */
+ int rebuild = 0;
+
+ for (struct link *l = c->hydro.rt_inject; l != NULL; l = l->next) {
+ struct task *t = l->t;
+ struct cell *ci = t->ci;
+ struct cell *cj = t->cj;
+ const int ci_active = cell_is_active_hydro(ci, e);
+ const int cj_active = (cj != NULL) ? cell_is_active_hydro(cj, e) : 0;
+#ifdef WITH_MPI
+ const int ci_nodeID = ci->nodeID;
+ const int cj_nodeID = (cj != NULL) ? cj->nodeID : -1;
+#else
+ const int ci_nodeID = nodeID;
+ const int cj_nodeID = nodeID;
+#endif
+
+ /* Only activate tasks that involve a local active cell. */
+ if ((ci_active && ci_nodeID == nodeID) ||
+ (cj_active && cj_nodeID == nodeID)) {
+
+ scheduler_activate(s, t);
+
+ if (t->type == task_type_sub_self) {
+ cell_activate_subcell_rt_tasks(ci, NULL, s);
+ }
+
+ else if (t->type == task_type_sub_pair) {
+ cell_activate_subcell_rt_tasks(ci, cj, s);
+ }
+ }
+ }
+
+ /* Unskip all the other task types */
+ if (c->nodeID == nodeID) {
+ if (cell_is_active_hydro(c, e)) {
+ if (c->hydro.rt_in != NULL) scheduler_activate(s, c->hydro.rt_in);
+ if (c->hydro.rt_ghost1 != NULL) scheduler_activate(s, c->hydro.rt_ghost1);
+ if (c->hydro.rt_out != NULL) scheduler_activate(s, c->hydro.rt_out);
+ }
+ }
+
+ return rebuild;
+}
diff --git a/src/common_io.c b/src/common_io.c
index ec0d085d914dffa8093b4a67b0b08647097159c5..081d6edac332f934ef0d914c0796783661cd2b36 100644
--- a/src/common_io.c
+++ b/src/common_io.c
@@ -24,35 +24,24 @@
/* This object's header. */
#include "common_io.h"
-/* Pre-inclusion as needed in other headers */
-#include "engine.h"
-
/* Local includes. */
-#include "black_holes_io.h"
-#include "chemistry_io.h"
-#include "const.h"
-#include "cooling_io.h"
+#include "engine.h"
#include "error.h"
-#include "feedback.h"
-#include "fof_io.h"
-#include "gravity_io.h"
-#include "hydro.h"
-#include "hydro_io.h"
-#include "io_properties.h"
#include "kernel_hydro.h"
#include "part.h"
#include "part_type.h"
-#include "rt_io.h"
-#include "sink_io.h"
-#include "star_formation_io.h"
-#include "stars_io.h"
#include "threadpool.h"
#include "tools.h"
-#include "tracers_io.h"
-#include "units.h"
-#include "velociraptor_io.h"
#include "version.h"
+/* I/O functions of each sub-module */
+#include "chemistry_io.h"
+#include "cooling_io.h"
+#include "feedback.h"
+#include "hydro_io.h"
+#include "stars_io.h"
+#include "tracers_io.h"
+
/* Some standard headers. */
#include <math.h>
#include <stddef.h>
@@ -763,457 +752,6 @@ void io_write_engine_policy(hid_t h_file, const struct engine* e) {
H5Gclose(h_grp);
}
-static long long cell_count_non_inhibited_gas(const struct cell* c) {
- const int total_count = c->hydro.count;
- struct part* parts = c->hydro.parts;
- long long count = 0;
- for (int i = 0; i < total_count; ++i) {
- if ((parts[i].time_bin != time_bin_inhibited) &&
- (parts[i].time_bin != time_bin_not_created)) {
- ++count;
- }
- }
- return count;
-}
-
-static long long cell_count_non_inhibited_dark_matter(const struct cell* c) {
- const int total_count = c->grav.count;
- struct gpart* gparts = c->grav.parts;
- long long count = 0;
- for (int i = 0; i < total_count; ++i) {
- if ((gparts[i].time_bin != time_bin_inhibited) &&
- (gparts[i].time_bin != time_bin_not_created) &&
- (gparts[i].type == swift_type_dark_matter)) {
- ++count;
- }
- }
- return count;
-}
-
-static long long cell_count_non_inhibited_background_dark_matter(
- const struct cell* c) {
- const int total_count = c->grav.count;
- struct gpart* gparts = c->grav.parts;
- long long count = 0;
- for (int i = 0; i < total_count; ++i) {
- if ((gparts[i].time_bin != time_bin_inhibited) &&
- (gparts[i].time_bin != time_bin_not_created) &&
- (gparts[i].type == swift_type_dark_matter_background)) {
- ++count;
- }
- }
- return count;
-}
-
-static long long cell_count_non_inhibited_stars(const struct cell* c) {
- const int total_count = c->stars.count;
- struct spart* sparts = c->stars.parts;
- long long count = 0;
- for (int i = 0; i < total_count; ++i) {
- if ((sparts[i].time_bin != time_bin_inhibited) &&
- (sparts[i].time_bin != time_bin_not_created)) {
- ++count;
- }
- }
- return count;
-}
-
-static long long cell_count_non_inhibited_black_holes(const struct cell* c) {
- const int total_count = c->black_holes.count;
- struct bpart* bparts = c->black_holes.parts;
- long long count = 0;
- for (int i = 0; i < total_count; ++i) {
- if ((bparts[i].time_bin != time_bin_inhibited) &&
- (bparts[i].time_bin != time_bin_not_created)) {
- ++count;
- }
- }
- return count;
-}
-
-static long long cell_count_non_inhibited_sinks(const struct cell* c) {
- const int total_count = c->sinks.count;
- struct sink* sinks = c->sinks.parts;
- long long count = 0;
- for (int i = 0; i < total_count; ++i) {
- if ((sinks[i].time_bin != time_bin_inhibited) &&
- (sinks[i].time_bin != time_bin_not_created)) {
- ++count;
- }
- }
- return count;
-}
-
-/**
- * @brief Write a single 1D array to a hdf5 group.
- *
- * This creates a simple Nx1 array with a chunk size of 1024x1.
- * The Fletcher-32 filter is applied to the array.
- *
- * @param h_grp The open hdf5 group.
- * @param n The number of elements in the array.
- * @param array The data to write.
- * @param type The type of the data to write.
- * @param name The name of the array.
- * @param array_content The name of the parent group (only used for error
- * messages).
- */
-void io_write_array(hid_t h_grp, const int n, const void* array,
- const enum IO_DATA_TYPE type, const char* name,
- const char* array_content) {
-
- /* Create memory space */
- const hsize_t shape[2] = {(hsize_t)n, 1};
- hid_t h_space = H5Screate(H5S_SIMPLE);
- if (h_space < 0)
- error("Error while creating data space for %s %s", name, array_content);
- hid_t h_err = H5Sset_extent_simple(h_space, 1, shape, shape);
- if (h_err < 0)
- error("Error while changing shape of %s %s data space.", name,
- array_content);
-
- /* Dataset type */
- hid_t h_type = H5Tcopy(io_hdf5_type(type));
-
- const hsize_t chunk[2] = {(1024 > n ? n : 1024), 1};
- hid_t h_prop = H5Pcreate(H5P_DATASET_CREATE);
- h_err = H5Pset_chunk(h_prop, 1, chunk);
- if (h_err < 0)
- error("Error while setting chunk shapes of %s %s data space.", name,
- array_content);
-
- /* Impose check-sum to verify data corruption */
- h_err = H5Pset_fletcher32(h_prop);
- if (h_err < 0)
- error("Error while setting check-sum filter on %s %s data space.", name,
- array_content);
-
- /* Write */
- hid_t h_data =
- H5Dcreate(h_grp, name, h_type, h_space, H5P_DEFAULT, h_prop, H5P_DEFAULT);
- if (h_data < 0)
- error("Error while creating dataspace for %s %s.", name, array_content);
- h_err = H5Dwrite(h_data, io_hdf5_type(type), h_space, H5S_ALL, H5P_DEFAULT,
- array);
- if (h_err < 0) error("Error while writing %s %s.", name, array_content);
- H5Tclose(h_type);
- H5Dclose(h_data);
- H5Pclose(h_prop);
- H5Sclose(h_space);
-}
-
-void io_write_cell_offsets(hid_t h_grp, const int cdim[3], const double dim[3],
- const struct cell* cells_top, const int nr_cells,
- const double width[3], const int nodeID,
- const int distributed,
- const long long global_counts[swift_type_count],
- const long long global_offsets[swift_type_count],
- const int num_fields[swift_type_count],
- const struct unit_system* internal_units,
- const struct unit_system* snapshot_units) {
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (distributed) {
- if (global_offsets[0] != 0 || global_offsets[1] != 0 ||
- global_offsets[2] != 0 || global_offsets[3] != 0 ||
- global_offsets[4] != 0 || global_offsets[5] != 0)
- error("Global offset non-zero in the distributed io case!");
- }
-#endif
-
- /* Abort if we don't have any cells yet (i.e. haven't constructed the space)
- */
- if (nr_cells == 0) return;
-
- double cell_width[3] = {width[0], width[1], width[2]};
-
- /* Temporary memory for the cell-by-cell information */
- double* centres = NULL;
- centres = (double*)malloc(3 * nr_cells * sizeof(double));
-
- /* Temporary memory for the cell files ID */
- int* files = NULL;
- files = (int*)malloc(nr_cells * sizeof(int));
-
- /* Count of particles in each cell */
- long long *count_part = NULL, *count_gpart = NULL,
- *count_background_gpart = NULL, *count_spart = NULL,
- *count_bpart = NULL, *count_sink = NULL;
- count_part = (long long*)malloc(nr_cells * sizeof(long long));
- count_gpart = (long long*)malloc(nr_cells * sizeof(long long));
- count_background_gpart = (long long*)malloc(nr_cells * sizeof(long long));
- count_spart = (long long*)malloc(nr_cells * sizeof(long long));
- count_bpart = (long long*)malloc(nr_cells * sizeof(long long));
- count_sink = (long long*)malloc(nr_cells * sizeof(long long));
-
- /* Global offsets of particles in each cell */
- long long *offset_part = NULL, *offset_gpart = NULL,
- *offset_background_gpart = NULL, *offset_spart = NULL,
- *offset_bpart = NULL, *offset_sink = NULL;
- offset_part = (long long*)malloc(nr_cells * sizeof(long long));
- offset_gpart = (long long*)malloc(nr_cells * sizeof(long long));
- offset_background_gpart = (long long*)malloc(nr_cells * sizeof(long long));
- offset_spart = (long long*)malloc(nr_cells * sizeof(long long));
- offset_bpart = (long long*)malloc(nr_cells * sizeof(long long));
- offset_sink = (long long*)malloc(nr_cells * sizeof(long long));
-
- /* Offsets of the 0^th element */
- offset_part[0] = 0;
- offset_gpart[0] = 0;
- offset_background_gpart[0] = 0;
- offset_spart[0] = 0;
- offset_bpart[0] = 0;
- offset_sink[0] = 0;
-
- /* Collect the cell information of *local* cells */
- long long local_offset_part = 0;
- long long local_offset_gpart = 0;
- long long local_offset_background_gpart = 0;
- long long local_offset_spart = 0;
- long long local_offset_bpart = 0;
- long long local_offset_sink = 0;
- for (int i = 0; i < nr_cells; ++i) {
-
- /* Store in which file this cell will be found */
- if (distributed) {
- files[i] = cells_top[i].nodeID;
- } else {
- files[i] = 0;
- }
-
- /* Is the cell on this node (i.e. we have full information */
- if (cells_top[i].nodeID == nodeID) {
-
- /* Centre of each cell */
- centres[i * 3 + 0] = cells_top[i].loc[0] + cell_width[0] * 0.5;
- centres[i * 3 + 1] = cells_top[i].loc[1] + cell_width[1] * 0.5;
- centres[i * 3 + 2] = cells_top[i].loc[2] + cell_width[2] * 0.5;
-
- /* Finish by box wrapping to match what is done to the particles */
- centres[i * 3 + 0] = box_wrap(centres[i * 3 + 0], 0.0, dim[0]);
- centres[i * 3 + 1] = box_wrap(centres[i * 3 + 1], 0.0, dim[1]);
- centres[i * 3 + 2] = box_wrap(centres[i * 3 + 2], 0.0, dim[2]);
-
- /* Count real particles that will be written */
- count_part[i] = cell_count_non_inhibited_gas(&cells_top[i]);
- count_gpart[i] = cell_count_non_inhibited_dark_matter(&cells_top[i]);
- count_background_gpart[i] =
- cell_count_non_inhibited_background_dark_matter(&cells_top[i]);
- count_spart[i] = cell_count_non_inhibited_stars(&cells_top[i]);
- count_bpart[i] = cell_count_non_inhibited_black_holes(&cells_top[i]);
- count_sink[i] = cell_count_non_inhibited_sinks(&cells_top[i]);
-
- /* Offsets including the global offset of all particles on this MPI rank
- * Note that in the distributed case, the global offsets are 0 such that
- * we actually compute the offset in the file written by this rank. */
- offset_part[i] = local_offset_part + global_offsets[swift_type_gas];
- offset_gpart[i] =
- local_offset_gpart + global_offsets[swift_type_dark_matter];
- offset_background_gpart[i] =
- local_offset_background_gpart +
- global_offsets[swift_type_dark_matter_background];
- offset_spart[i] = local_offset_spart + global_offsets[swift_type_stars];
- offset_bpart[i] =
- local_offset_bpart + global_offsets[swift_type_black_hole];
- offset_sink[i] = local_offset_sink + global_offsets[swift_type_sink];
-
- local_offset_part += count_part[i];
- local_offset_gpart += count_gpart[i];
- local_offset_background_gpart += count_background_gpart[i];
- local_offset_spart += count_spart[i];
- local_offset_bpart += count_bpart[i];
- local_offset_sink += count_sink[i];
-
- } else {
-
- /* Just zero everything for the foregin cells */
-
- centres[i * 3 + 0] = 0.;
- centres[i * 3 + 1] = 0.;
- centres[i * 3 + 2] = 0.;
-
- count_part[i] = 0;
- count_gpart[i] = 0;
- count_background_gpart[i] = 0;
- count_spart[i] = 0;
- count_bpart[i] = 0;
- count_sink[i] = 0;
-
- offset_part[i] = 0;
- offset_gpart[i] = 0;
- offset_background_gpart[i] = 0;
- offset_spart[i] = 0;
- offset_bpart[i] = 0;
- offset_sink[i] = 0;
- }
- }
-
-#ifdef WITH_MPI
- /* Now, reduce all the arrays. Note that we use a bit-wise OR here. This
- is safe as we made sure only local cells have non-zero values. */
- MPI_Allreduce(MPI_IN_PLACE, count_part, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
- MPI_COMM_WORLD);
- MPI_Allreduce(MPI_IN_PLACE, count_gpart, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
- MPI_COMM_WORLD);
- MPI_Allreduce(MPI_IN_PLACE, count_background_gpart, nr_cells,
- MPI_LONG_LONG_INT, MPI_BOR, MPI_COMM_WORLD);
- MPI_Allreduce(MPI_IN_PLACE, count_sink, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
- MPI_COMM_WORLD);
- MPI_Allreduce(MPI_IN_PLACE, count_spart, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
- MPI_COMM_WORLD);
- MPI_Allreduce(MPI_IN_PLACE, count_bpart, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
- MPI_COMM_WORLD);
-
- MPI_Allreduce(MPI_IN_PLACE, offset_part, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
- MPI_COMM_WORLD);
- MPI_Allreduce(MPI_IN_PLACE, offset_gpart, nr_cells, MPI_LONG_LONG_INT,
- MPI_BOR, MPI_COMM_WORLD);
- MPI_Allreduce(MPI_IN_PLACE, offset_background_gpart, nr_cells,
- MPI_LONG_LONG_INT, MPI_BOR, MPI_COMM_WORLD);
- MPI_Allreduce(MPI_IN_PLACE, offset_sink, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
- MPI_COMM_WORLD);
- MPI_Allreduce(MPI_IN_PLACE, offset_spart, nr_cells, MPI_LONG_LONG_INT,
- MPI_BOR, MPI_COMM_WORLD);
- MPI_Allreduce(MPI_IN_PLACE, offset_bpart, nr_cells, MPI_LONG_LONG_INT,
- MPI_BOR, MPI_COMM_WORLD);
-
- /* For the centres we use a sum as MPI does not like bit-wise operations
- on floating point numbers */
- MPI_Allreduce(MPI_IN_PLACE, centres, 3 * nr_cells, MPI_DOUBLE, MPI_SUM,
- MPI_COMM_WORLD);
-#endif
-
- /* When writing a single file, only rank 0 writes the meta-data */
- if ((distributed) || (!distributed && nodeID == 0)) {
-
- /* Unit conversion if necessary */
- const double factor = units_conversion_factor(
- internal_units, snapshot_units, UNIT_CONV_LENGTH);
- if (factor != 1.) {
-
- /* Convert the cell centres */
- for (int i = 0; i < nr_cells; ++i) {
- centres[i * 3 + 0] *= factor;
- centres[i * 3 + 1] *= factor;
- centres[i * 3 + 2] *= factor;
- }
-
- /* Convert the cell widths */
- cell_width[0] *= factor;
- cell_width[1] *= factor;
- cell_width[2] *= factor;
- }
-
- /* Write some meta-information first */
- hid_t h_subgrp =
- H5Gcreate(h_grp, "Meta-data", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
- if (h_subgrp < 0) error("Error while creating meta-data sub-group");
- io_write_attribute(h_subgrp, "nr_cells", INT, &nr_cells, 1);
- io_write_attribute(h_subgrp, "size", DOUBLE, cell_width, 3);
- io_write_attribute(h_subgrp, "dimension", INT, cdim, 3);
- H5Gclose(h_subgrp);
-
- /* Write the centres to the group */
- hsize_t shape[2] = {(hsize_t)nr_cells, 3};
- hid_t h_space = H5Screate(H5S_SIMPLE);
- if (h_space < 0) error("Error while creating data space for cell centres");
- hid_t h_err = H5Sset_extent_simple(h_space, 2, shape, shape);
- if (h_err < 0)
- error("Error while changing shape of gas offsets data space.");
- hid_t h_data = H5Dcreate(h_grp, "Centres", io_hdf5_type(DOUBLE), h_space,
- H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
- if (h_data < 0) error("Error while creating dataspace for gas offsets.");
- h_err = H5Dwrite(h_data, io_hdf5_type(DOUBLE), h_space, H5S_ALL,
- H5P_DEFAULT, centres);
- if (h_err < 0) error("Error while writing centres.");
- H5Dclose(h_data);
- H5Sclose(h_space);
-
- /* Group containing the offsets and counts for each particle type */
- hid_t h_grp_offsets = H5Gcreate(h_grp, "OffsetsInFile", H5P_DEFAULT,
- H5P_DEFAULT, H5P_DEFAULT);
- if (h_grp_offsets < 0) error("Error while creating offsets sub-group");
- hid_t h_grp_files =
- H5Gcreate(h_grp, "Files", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
- if (h_grp_files < 0) error("Error while creating filess sub-group");
- hid_t h_grp_counts =
- H5Gcreate(h_grp, "Counts", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
- if (h_grp_counts < 0) error("Error while creating counts sub-group");
-
- if (global_counts[swift_type_gas] > 0 && num_fields[swift_type_gas] > 0) {
- io_write_array(h_grp_files, nr_cells, files, INT, "PartType0", "files");
- io_write_array(h_grp_offsets, nr_cells, offset_part, LONGLONG,
- "PartType0", "offsets");
- io_write_array(h_grp_counts, nr_cells, count_part, LONGLONG, "PartType0",
- "counts");
- }
-
- if (global_counts[swift_type_dark_matter] > 0 &&
- num_fields[swift_type_dark_matter] > 0) {
- io_write_array(h_grp_files, nr_cells, files, INT, "PartType1", "files");
- io_write_array(h_grp_offsets, nr_cells, offset_gpart, LONGLONG,
- "PartType1", "offsets");
- io_write_array(h_grp_counts, nr_cells, count_gpart, LONGLONG, "PartType1",
- "counts");
- }
-
- if (global_counts[swift_type_dark_matter_background] > 0 &&
- num_fields[swift_type_dark_matter_background] > 0) {
- io_write_array(h_grp_files, nr_cells, files, INT, "PartType2", "files");
- io_write_array(h_grp_offsets, nr_cells, offset_background_gpart, LONGLONG,
- "PartType2", "offsets");
- io_write_array(h_grp_counts, nr_cells, count_background_gpart, LONGLONG,
- "PartType2", "counts");
- }
-
- if (global_counts[swift_type_sink] > 0 && num_fields[swift_type_sink] > 0) {
- io_write_array(h_grp_files, nr_cells, files, INT, "PartType3", "files");
- io_write_array(h_grp_offsets, nr_cells, offset_sink, LONGLONG,
- "PartType3", "offsets");
- io_write_array(h_grp_counts, nr_cells, count_sink, LONGLONG, "PartType3",
- "counts");
- }
-
- if (global_counts[swift_type_stars] > 0 &&
- num_fields[swift_type_stars] > 0) {
- io_write_array(h_grp_files, nr_cells, files, INT, "PartType4", "files");
- io_write_array(h_grp_offsets, nr_cells, offset_spart, LONGLONG,
- "PartType4", "offsets");
- io_write_array(h_grp_counts, nr_cells, count_spart, LONGLONG, "PartType4",
- "counts");
- }
-
- if (global_counts[swift_type_black_hole] > 0 &&
- num_fields[swift_type_black_hole] > 0) {
- io_write_array(h_grp_files, nr_cells, files, INT, "PartType5", "files");
- io_write_array(h_grp_offsets, nr_cells, offset_bpart, LONGLONG,
- "PartType5", "offsets");
- io_write_array(h_grp_counts, nr_cells, count_bpart, LONGLONG, "PartType5",
- "counts");
- }
-
- H5Gclose(h_grp_offsets);
- H5Gclose(h_grp_files);
- H5Gclose(h_grp_counts);
- }
-
- /* Free everything we allocated */
- free(centres);
- free(files);
- free(count_part);
- free(count_gpart);
- free(count_background_gpart);
- free(count_spart);
- free(count_bpart);
- free(count_sink);
- free(offset_part);
- free(offset_gpart);
- free(offset_background_gpart);
- free(offset_spart);
- free(offset_bpart);
- free(offset_sink);
-}
-
#endif /* HAVE_HDF5 */
/**
@@ -1250,730 +788,6 @@ size_t io_sizeof_type(enum IO_DATA_TYPE type) {
}
}
-/**
- * @brief Mapper function to copy #part or #gpart fields into a buffer.
- */
-void io_copy_mapper(void* restrict temp, int N, void* restrict extra_data) {
-
- const struct io_props props = *((const struct io_props*)extra_data);
- const size_t typeSize = io_sizeof_type(props.type);
- const size_t copySize = typeSize * props.dimension;
-
- /* How far are we with this chunk? */
- char* restrict temp_c = (char*)temp;
- const ptrdiff_t delta = (temp_c - props.start_temp_c) / copySize;
-
- for (int k = 0; k < N; k++) {
- memcpy(&temp_c[k * copySize], props.field + (delta + k) * props.partSize,
- copySize);
- }
-}
-
-/**
- * @brief Mapper function to copy #part into a buffer of floats using a
- * conversion function.
- */
-void io_convert_part_f_mapper(void* restrict temp, int N,
- void* restrict extra_data) {
-
- const struct io_props props = *((const struct io_props*)extra_data);
- const struct part* restrict parts = props.parts;
- const struct xpart* restrict xparts = props.xparts;
- const struct engine* e = props.e;
- const size_t dim = props.dimension;
-
- /* How far are we with this chunk? */
- float* restrict temp_f = (float*)temp;
- const ptrdiff_t delta = (temp_f - props.start_temp_f) / dim;
-
- for (int i = 0; i < N; i++)
- props.convert_part_f(e, parts + delta + i, xparts + delta + i,
- &temp_f[i * dim]);
-}
-
-/**
- * @brief Mapper function to copy #part into a buffer of ints using a
- * conversion function.
- */
-void io_convert_part_i_mapper(void* restrict temp, int N,
- void* restrict extra_data) {
-
- const struct io_props props = *((const struct io_props*)extra_data);
- const struct part* restrict parts = props.parts;
- const struct xpart* restrict xparts = props.xparts;
- const struct engine* e = props.e;
- const size_t dim = props.dimension;
-
- /* How far are we with this chunk? */
- int* restrict temp_i = (int*)temp;
- const ptrdiff_t delta = (temp_i - props.start_temp_i) / dim;
-
- for (int i = 0; i < N; i++)
- props.convert_part_i(e, parts + delta + i, xparts + delta + i,
- &temp_i[i * dim]);
-}
-
-/**
- * @brief Mapper function to copy #part into a buffer of doubles using a
- * conversion function.
- */
-void io_convert_part_d_mapper(void* restrict temp, int N,
- void* restrict extra_data) {
-
- const struct io_props props = *((const struct io_props*)extra_data);
- const struct part* restrict parts = props.parts;
- const struct xpart* restrict xparts = props.xparts;
- const struct engine* e = props.e;
- const size_t dim = props.dimension;
-
- /* How far are we with this chunk? */
- double* restrict temp_d = (double*)temp;
- const ptrdiff_t delta = (temp_d - props.start_temp_d) / dim;
-
- for (int i = 0; i < N; i++)
- props.convert_part_d(e, parts + delta + i, xparts + delta + i,
- &temp_d[i * dim]);
-}
-
-/**
- * @brief Mapper function to copy #part into a buffer of doubles using a
- * conversion function.
- */
-void io_convert_part_l_mapper(void* restrict temp, int N,
- void* restrict extra_data) {
-
- const struct io_props props = *((const struct io_props*)extra_data);
- const struct part* restrict parts = props.parts;
- const struct xpart* restrict xparts = props.xparts;
- const struct engine* e = props.e;
- const size_t dim = props.dimension;
-
- /* How far are we with this chunk? */
- long long* restrict temp_l = (long long*)temp;
- const ptrdiff_t delta = (temp_l - props.start_temp_l) / dim;
-
- for (int i = 0; i < N; i++)
- props.convert_part_l(e, parts + delta + i, xparts + delta + i,
- &temp_l[i * dim]);
-}
-
-/**
- * @brief Mapper function to copy #gpart into a buffer of floats using a
- * conversion function.
- */
-void io_convert_gpart_f_mapper(void* restrict temp, int N,
- void* restrict extra_data) {
-
- const struct io_props props = *((const struct io_props*)extra_data);
- const struct gpart* restrict gparts = props.gparts;
- const struct engine* e = props.e;
- const size_t dim = props.dimension;
-
- /* How far are we with this chunk? */
- float* restrict temp_f = (float*)temp;
- const ptrdiff_t delta = (temp_f - props.start_temp_f) / dim;
-
- for (int i = 0; i < N; i++)
- props.convert_gpart_f(e, gparts + delta + i, &temp_f[i * dim]);
-}
-
-/**
- * @brief Mapper function to copy #gpart into a buffer of ints using a
- * conversion function.
- */
-void io_convert_gpart_i_mapper(void* restrict temp, int N,
- void* restrict extra_data) {
-
- const struct io_props props = *((const struct io_props*)extra_data);
- const struct gpart* restrict gparts = props.gparts;
- const struct engine* e = props.e;
- const size_t dim = props.dimension;
-
- /* How far are we with this chunk? */
- int* restrict temp_i = (int*)temp;
- const ptrdiff_t delta = (temp_i - props.start_temp_i) / dim;
-
- for (int i = 0; i < N; i++)
- props.convert_gpart_i(e, gparts + delta + i, &temp_i[i * dim]);
-}
-
-/**
- * @brief Mapper function to copy #gpart into a buffer of doubles using a
- * conversion function.
- */
-void io_convert_gpart_d_mapper(void* restrict temp, int N,
- void* restrict extra_data) {
-
- const struct io_props props = *((const struct io_props*)extra_data);
- const struct gpart* restrict gparts = props.gparts;
- const struct engine* e = props.e;
- const size_t dim = props.dimension;
-
- /* How far are we with this chunk? */
- double* restrict temp_d = (double*)temp;
- const ptrdiff_t delta = (temp_d - props.start_temp_d) / dim;
-
- for (int i = 0; i < N; i++)
- props.convert_gpart_d(e, gparts + delta + i, &temp_d[i * dim]);
-}
-
-/**
- * @brief Mapper function to copy #gpart into a buffer of doubles using a
- * conversion function.
- */
-void io_convert_gpart_l_mapper(void* restrict temp, int N,
- void* restrict extra_data) {
-
- const struct io_props props = *((const struct io_props*)extra_data);
- const struct gpart* restrict gparts = props.gparts;
- const struct engine* e = props.e;
- const size_t dim = props.dimension;
-
- /* How far are we with this chunk? */
- long long* restrict temp_l = (long long*)temp;
- const ptrdiff_t delta = (temp_l - props.start_temp_l) / dim;
-
- for (int i = 0; i < N; i++)
- props.convert_gpart_l(e, gparts + delta + i, &temp_l[i * dim]);
-}
-
-/**
- * @brief Mapper function to copy #spart into a buffer of floats using a
- * conversion function.
- */
-void io_convert_spart_f_mapper(void* restrict temp, int N,
- void* restrict extra_data) {
-
- const struct io_props props = *((const struct io_props*)extra_data);
- const struct spart* restrict sparts = props.sparts;
- const struct engine* e = props.e;
- const size_t dim = props.dimension;
-
- /* How far are we with this chunk? */
- float* restrict temp_f = (float*)temp;
- const ptrdiff_t delta = (temp_f - props.start_temp_f) / dim;
-
- for (int i = 0; i < N; i++)
- props.convert_spart_f(e, sparts + delta + i, &temp_f[i * dim]);
-}
-
-/**
- * @brief Mapper function to copy #spart into a buffer of ints using a
- * conversion function.
- */
-void io_convert_spart_i_mapper(void* restrict temp, int N,
- void* restrict extra_data) {
-
- const struct io_props props = *((const struct io_props*)extra_data);
- const struct spart* restrict sparts = props.sparts;
- const struct engine* e = props.e;
- const size_t dim = props.dimension;
-
- /* How far are we with this chunk? */
- int* restrict temp_i = (int*)temp;
- const ptrdiff_t delta = (temp_i - props.start_temp_i) / dim;
-
- for (int i = 0; i < N; i++)
- props.convert_spart_i(e, sparts + delta + i, &temp_i[i * dim]);
-}
-
-/**
- * @brief Mapper function to copy #spart into a buffer of doubles using a
- * conversion function.
- */
-void io_convert_spart_d_mapper(void* restrict temp, int N,
- void* restrict extra_data) {
-
- const struct io_props props = *((const struct io_props*)extra_data);
- const struct spart* restrict sparts = props.sparts;
- const struct engine* e = props.e;
- const size_t dim = props.dimension;
-
- /* How far are we with this chunk? */
- double* restrict temp_d = (double*)temp;
- const ptrdiff_t delta = (temp_d - props.start_temp_d) / dim;
-
- for (int i = 0; i < N; i++)
- props.convert_spart_d(e, sparts + delta + i, &temp_d[i * dim]);
-}
-
-/**
- * @brief Mapper function to copy #spart into a buffer of doubles using a
- * conversion function.
- */
-void io_convert_spart_l_mapper(void* restrict temp, int N,
- void* restrict extra_data) {
-
- const struct io_props props = *((const struct io_props*)extra_data);
- const struct spart* restrict sparts = props.sparts;
- const struct engine* e = props.e;
- const size_t dim = props.dimension;
-
- /* How far are we with this chunk? */
- long long* restrict temp_l = (long long*)temp;
- const ptrdiff_t delta = (temp_l - props.start_temp_l) / dim;
-
- for (int i = 0; i < N; i++)
- props.convert_spart_l(e, sparts + delta + i, &temp_l[i * dim]);
-}
-
-/**
- * @brief Mapper function to copy #bpart into a buffer of floats using a
- * conversion function.
- */
-void io_convert_bpart_f_mapper(void* restrict temp, int N,
- void* restrict extra_data) {
-
- const struct io_props props = *((const struct io_props*)extra_data);
- const struct bpart* restrict bparts = props.bparts;
- const struct engine* e = props.e;
- const size_t dim = props.dimension;
-
- /* How far are we with this chunk? */
- float* restrict temp_f = (float*)temp;
- const ptrdiff_t delta = (temp_f - props.start_temp_f) / dim;
-
- for (int i = 0; i < N; i++)
- props.convert_bpart_f(e, bparts + delta + i, &temp_f[i * dim]);
-}
-
-/**
- * @brief Mapper function to copy #bpart into a buffer of ints using a
- * conversion function.
- */
-void io_convert_bpart_i_mapper(void* restrict temp, int N,
- void* restrict extra_data) {
-
- const struct io_props props = *((const struct io_props*)extra_data);
- const struct bpart* restrict bparts = props.bparts;
- const struct engine* e = props.e;
- const size_t dim = props.dimension;
-
- /* How far are we with this chunk? */
- int* restrict temp_i = (int*)temp;
- const ptrdiff_t delta = (temp_i - props.start_temp_i) / dim;
-
- for (int i = 0; i < N; i++)
- props.convert_bpart_i(e, bparts + delta + i, &temp_i[i * dim]);
-}
-
-/**
- * @brief Mapper function to copy #bpart into a buffer of doubles using a
- * conversion function.
- */
-void io_convert_bpart_d_mapper(void* restrict temp, int N,
- void* restrict extra_data) {
-
- const struct io_props props = *((const struct io_props*)extra_data);
- const struct bpart* restrict bparts = props.bparts;
- const struct engine* e = props.e;
- const size_t dim = props.dimension;
-
- /* How far are we with this chunk? */
- double* restrict temp_d = (double*)temp;
- const ptrdiff_t delta = (temp_d - props.start_temp_d) / dim;
-
- for (int i = 0; i < N; i++)
- props.convert_bpart_d(e, bparts + delta + i, &temp_d[i * dim]);
-}
-
-/**
- * @brief Mapper function to copy #bpart into a buffer of doubles using a
- * conversion function.
- */
-void io_convert_bpart_l_mapper(void* restrict temp, int N,
- void* restrict extra_data) {
-
- const struct io_props props = *((const struct io_props*)extra_data);
- const struct bpart* restrict bparts = props.bparts;
- const struct engine* e = props.e;
- const size_t dim = props.dimension;
-
- /* How far are we with this chunk? */
- long long* restrict temp_l = (long long*)temp;
- const ptrdiff_t delta = (temp_l - props.start_temp_l) / dim;
-
- for (int i = 0; i < N; i++)
- props.convert_bpart_l(e, bparts + delta + i, &temp_l[i * dim]);
-}
-
-/**
- * @brief Mapper function to copy #sink into a buffer of floats using a
- * conversion function.
- */
-void io_convert_sink_f_mapper(void* restrict temp, int N,
- void* restrict extra_data) {
-
- const struct io_props props = *((const struct io_props*)extra_data);
- const struct sink* restrict sinks = props.sinks;
- const struct engine* e = props.e;
- const size_t dim = props.dimension;
-
- /* How far are we with this chunk? */
- float* restrict temp_f = (float*)temp;
- const ptrdiff_t delta = (temp_f - props.start_temp_f) / dim;
-
- for (int i = 0; i < N; i++)
- props.convert_sink_f(e, sinks + delta + i, &temp_f[i * dim]);
-}
-
-/**
- * @brief Mapper function to copy #sink into a buffer of ints using a
- * conversion function.
- */
-void io_convert_sink_i_mapper(void* restrict temp, int N,
- void* restrict extra_data) {
-
- const struct io_props props = *((const struct io_props*)extra_data);
- const struct sink* restrict sinks = props.sinks;
- const struct engine* e = props.e;
- const size_t dim = props.dimension;
-
- /* How far are we with this chunk? */
- int* restrict temp_i = (int*)temp;
- const ptrdiff_t delta = (temp_i - props.start_temp_i) / dim;
-
- for (int i = 0; i < N; i++)
- props.convert_sink_i(e, sinks + delta + i, &temp_i[i * dim]);
-}
-
-/**
- * @brief Mapper function to copy #sink into a buffer of doubles using a
- * conversion function.
- */
-void io_convert_sink_d_mapper(void* restrict temp, int N,
- void* restrict extra_data) {
-
- const struct io_props props = *((const struct io_props*)extra_data);
- const struct sink* restrict sinks = props.sinks;
- const struct engine* e = props.e;
- const size_t dim = props.dimension;
-
- /* How far are we with this chunk? */
- double* restrict temp_d = (double*)temp;
- const ptrdiff_t delta = (temp_d - props.start_temp_d) / dim;
-
- for (int i = 0; i < N; i++)
- props.convert_sink_d(e, sinks + delta + i, &temp_d[i * dim]);
-}
-
-/**
- * @brief Mapper function to copy #sink into a buffer of doubles using a
- * conversion function.
- */
-void io_convert_sink_l_mapper(void* restrict temp, int N,
- void* restrict extra_data) {
-
- const struct io_props props = *((const struct io_props*)extra_data);
- const struct sink* restrict sinks = props.sinks;
- const struct engine* e = props.e;
- const size_t dim = props.dimension;
-
- /* How far are we with this chunk? */
- long long* restrict temp_l = (long long*)temp;
- const ptrdiff_t delta = (temp_l - props.start_temp_l) / dim;
-
- for (int i = 0; i < N; i++)
- props.convert_sink_l(e, sinks + delta + i, &temp_l[i * dim]);
-}
-
-/**
- * @brief Copy the particle data into a temporary buffer ready for i/o.
- *
- * @param temp The buffer to be filled. Must be allocated and aligned properly.
- * @param e The #engine.
- * @param props The #io_props corresponding to the particle field we are
- * copying.
- * @param N The number of particles to copy
- * @param internal_units The system of units used internally.
- * @param snapshot_units The system of units used for the snapshots.
- */
-void io_copy_temp_buffer(void* temp, const struct engine* e,
- struct io_props props, size_t N,
- const struct unit_system* internal_units,
- const struct unit_system* snapshot_units) {
-
- const size_t typeSize = io_sizeof_type(props.type);
- const size_t copySize = typeSize * props.dimension;
- const size_t num_elements = N * props.dimension;
-
- /* Copy particle data to temporary buffer */
- if (props.conversion == 0) { /* No conversion */
-
- /* Prepare some parameters */
- char* temp_c = (char*)temp;
- props.start_temp_c = temp_c;
-
- /* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*)&e->threadpool, io_copy_mapper, temp_c,
- N, copySize, threadpool_auto_chunk_size, (void*)&props);
-
- } else { /* Converting particle to data */
-
- if (props.convert_part_f != NULL) {
-
- /* Prepare some parameters */
- float* temp_f = (float*)temp;
- props.start_temp_f = (float*)temp;
- props.e = e;
-
- /* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*)&e->threadpool,
- io_convert_part_f_mapper, temp_f, N, copySize,
- threadpool_auto_chunk_size, (void*)&props);
-
- } else if (props.convert_part_i != NULL) {
-
- /* Prepare some parameters */
- int* temp_i = (int*)temp;
- props.start_temp_i = (int*)temp;
- props.e = e;
-
- /* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*)&e->threadpool,
- io_convert_part_i_mapper, temp_i, N, copySize,
- threadpool_auto_chunk_size, (void*)&props);
-
- } else if (props.convert_part_d != NULL) {
-
- /* Prepare some parameters */
- double* temp_d = (double*)temp;
- props.start_temp_d = (double*)temp;
- props.e = e;
-
- /* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*)&e->threadpool,
- io_convert_part_d_mapper, temp_d, N, copySize,
- threadpool_auto_chunk_size, (void*)&props);
-
- } else if (props.convert_part_l != NULL) {
-
- /* Prepare some parameters */
- long long* temp_l = (long long*)temp;
- props.start_temp_l = (long long*)temp;
- props.e = e;
-
- /* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*)&e->threadpool,
- io_convert_part_l_mapper, temp_l, N, copySize,
- threadpool_auto_chunk_size, (void*)&props);
-
- } else if (props.convert_gpart_f != NULL) {
-
- /* Prepare some parameters */
- float* temp_f = (float*)temp;
- props.start_temp_f = (float*)temp;
- props.e = e;
-
- /* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*)&e->threadpool,
- io_convert_gpart_f_mapper, temp_f, N, copySize,
- threadpool_auto_chunk_size, (void*)&props);
-
- } else if (props.convert_gpart_i != NULL) {
-
- /* Prepare some parameters */
- int* temp_i = (int*)temp;
- props.start_temp_i = (int*)temp;
- props.e = e;
-
- /* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*)&e->threadpool,
- io_convert_gpart_i_mapper, temp_i, N, copySize,
- threadpool_auto_chunk_size, (void*)&props);
-
- } else if (props.convert_gpart_d != NULL) {
-
- /* Prepare some parameters */
- double* temp_d = (double*)temp;
- props.start_temp_d = (double*)temp;
- props.e = e;
-
- /* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*)&e->threadpool,
- io_convert_gpart_d_mapper, temp_d, N, copySize,
- threadpool_auto_chunk_size, (void*)&props);
-
- } else if (props.convert_gpart_l != NULL) {
-
- /* Prepare some parameters */
- long long* temp_l = (long long*)temp;
- props.start_temp_l = (long long*)temp;
- props.e = e;
-
- /* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*)&e->threadpool,
- io_convert_gpart_l_mapper, temp_l, N, copySize,
- threadpool_auto_chunk_size, (void*)&props);
-
- } else if (props.convert_spart_f != NULL) {
-
- /* Prepare some parameters */
- float* temp_f = (float*)temp;
- props.start_temp_f = (float*)temp;
- props.e = e;
-
- /* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*)&e->threadpool,
- io_convert_spart_f_mapper, temp_f, N, copySize,
- threadpool_auto_chunk_size, (void*)&props);
-
- } else if (props.convert_spart_i != NULL) {
-
- /* Prepare some parameters */
- int* temp_i = (int*)temp;
- props.start_temp_i = (int*)temp;
- props.e = e;
-
- /* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*)&e->threadpool,
- io_convert_spart_i_mapper, temp_i, N, copySize,
- threadpool_auto_chunk_size, (void*)&props);
-
- } else if (props.convert_spart_d != NULL) {
-
- /* Prepare some parameters */
- double* temp_d = (double*)temp;
- props.start_temp_d = (double*)temp;
- props.e = e;
-
- /* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*)&e->threadpool,
- io_convert_spart_d_mapper, temp_d, N, copySize,
- threadpool_auto_chunk_size, (void*)&props);
-
- } else if (props.convert_spart_l != NULL) {
-
- /* Prepare some parameters */
- long long* temp_l = (long long*)temp;
- props.start_temp_l = (long long*)temp;
- props.e = e;
-
- /* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*)&e->threadpool,
- io_convert_spart_l_mapper, temp_l, N, copySize,
- threadpool_auto_chunk_size, (void*)&props);
-
- } else if (props.convert_sink_f != NULL) {
-
- /* Prepare some parameters */
- float* temp_f = (float*)temp;
- props.start_temp_f = (float*)temp;
- props.e = e;
-
- /* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*)&e->threadpool,
- io_convert_sink_f_mapper, temp_f, N, copySize,
- threadpool_auto_chunk_size, (void*)&props);
-
- } else if (props.convert_sink_i != NULL) {
-
- /* Prepare some parameters */
- int* temp_i = (int*)temp;
- props.start_temp_i = (int*)temp;
- props.e = e;
-
- /* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*)&e->threadpool,
- io_convert_sink_i_mapper, temp_i, N, copySize,
- threadpool_auto_chunk_size, (void*)&props);
-
- } else if (props.convert_sink_d != NULL) {
-
- /* Prepare some parameters */
- double* temp_d = (double*)temp;
- props.start_temp_d = (double*)temp;
- props.e = e;
-
- /* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*)&e->threadpool,
- io_convert_sink_d_mapper, temp_d, N, copySize,
- threadpool_auto_chunk_size, (void*)&props);
-
- } else if (props.convert_sink_l != NULL) {
-
- /* Prepare some parameters */
- long long* temp_l = (long long*)temp;
- props.start_temp_l = (long long*)temp;
- props.e = e;
-
- /* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*)&e->threadpool,
- io_convert_sink_l_mapper, temp_l, N, copySize,
- threadpool_auto_chunk_size, (void*)&props);
-
- } else if (props.convert_bpart_f != NULL) {
-
- /* Prepare some parameters */
- float* temp_f = (float*)temp;
- props.start_temp_f = (float*)temp;
- props.e = e;
-
- /* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*)&e->threadpool,
- io_convert_bpart_f_mapper, temp_f, N, copySize,
- threadpool_auto_chunk_size, (void*)&props);
-
- } else if (props.convert_bpart_i != NULL) {
-
- /* Prepare some parameters */
- int* temp_i = (int*)temp;
- props.start_temp_i = (int*)temp;
- props.e = e;
-
- /* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*)&e->threadpool,
- io_convert_bpart_i_mapper, temp_i, N, copySize,
- threadpool_auto_chunk_size, (void*)&props);
-
- } else if (props.convert_bpart_d != NULL) {
-
- /* Prepare some parameters */
- double* temp_d = (double*)temp;
- props.start_temp_d = (double*)temp;
- props.e = e;
-
- /* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*)&e->threadpool,
- io_convert_bpart_d_mapper, temp_d, N, copySize,
- threadpool_auto_chunk_size, (void*)&props);
-
- } else if (props.convert_bpart_l != NULL) {
-
- /* Prepare some parameters */
- long long* temp_l = (long long*)temp;
- props.start_temp_l = (long long*)temp;
- props.e = e;
-
- /* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*)&e->threadpool,
- io_convert_bpart_l_mapper, temp_l, N, copySize,
- threadpool_auto_chunk_size, (void*)&props);
-
- } else {
- error("Missing conversion function");
- }
- }
-
- /* Unit conversion if necessary */
- const double factor =
- units_conversion_factor(internal_units, snapshot_units, props.units);
- if (factor != 1.) {
-
- /* message("Converting ! factor=%e", factor); */
-
- if (io_is_double_precision(props.type)) {
- swift_declare_aligned_ptr(double, temp_d, (double*)temp,
- IO_BUFFER_ALIGNMENT);
- for (size_t i = 0; i < num_elements; ++i) temp_d[i] *= factor;
- } else {
- swift_declare_aligned_ptr(float, temp_f, (float*)temp,
- IO_BUFFER_ALIGNMENT);
- for (size_t i = 0; i < num_elements; ++i) temp_f[i] *= factor;
- }
- }
-}
-
void io_prepare_dm_gparts_mapper(void* restrict data, int Ndm, void* dummy) {
struct gpart* restrict gparts = (struct gpart*)data;
@@ -2536,243 +1350,6 @@ void io_collect_gparts_background_to_write(
count, Ngparts_written);
}
-/**
- * @brief Prepare the output option fields according to the user's choices and
- * verify that they are valid.
- *
- * @param output_options The #output_options for this run
- * @param with_cosmology Ran with cosmology?
- * @param with_fof Are we running with on-the-fly Fof?
- * @param with_stf Are we running with on-the-fly structure finder?
- */
-void io_prepare_output_fields(struct output_options* output_options,
- const int with_cosmology, const int with_fof,
- const int with_stf) {
-
- const int MAX_NUM_PTYPE_FIELDS = 100;
-
- /* Parameter struct for the output options */
- struct swift_params* params = output_options->select_output;
-
- /* Get all possible outputs per particle type */
- int ptype_num_fields_total[swift_type_count] = {0};
- struct io_props field_list[swift_type_count][MAX_NUM_PTYPE_FIELDS];
-
- for (int ptype = 0; ptype < swift_type_count; ptype++)
- ptype_num_fields_total[ptype] = get_ptype_fields(
- ptype, field_list[ptype], with_cosmology, with_fof, with_stf);
-
- /* Check for whether we have a `Default` section */
- int have_default = 0;
-
- /* Loop over each section, i.e. different class of output */
- for (int section_id = 0; section_id < params->sectionCount; section_id++) {
-
- /* Get the name of current (selection) section, without a trailing colon */
- char section_name[FIELD_BUFFER_SIZE];
- strcpy(section_name, params->section[section_id].name);
- section_name[strlen(section_name) - 1] = 0;
-
- /* Is this the `Default` section? */
- if (strcmp(section_name, select_output_header_default_name) == 0)
- have_default = 1;
-
- /* How many fields should each ptype write by default? */
- int ptype_num_fields_to_write[swift_type_count];
-
- /* What is the default writing status for each ptype (on/off)? */
- int ptype_default_write_status[swift_type_count];
-
- /* Initialise section-specific writing counters for each particle type.
- * If default is 'write', then we start from the total to deduct any fields
- * that are switched off. If the default is 'off', we have to start from
- * zero and then count upwards for each field that is switched back on. */
- for (int ptype = 0; ptype < swift_type_count; ptype++) {
-
- /* Internally also verifies that the default level is allowed */
- const enum lossy_compression_schemes compression_level_current_default =
- output_options_get_ptype_default_compression(params, section_name,
- (enum part_type)ptype);
-
- if (compression_level_current_default == compression_do_not_write) {
- ptype_default_write_status[ptype] = 0;
- ptype_num_fields_to_write[ptype] = 0;
- } else {
- ptype_default_write_status[ptype] = 1;
- ptype_num_fields_to_write[ptype] = ptype_num_fields_total[ptype];
- }
-
- } /* ends loop over particle types */
-
- /* Loop over each parameter */
- for (int param_id = 0; param_id < params->paramCount; param_id++) {
-
- /* Full name of the parameter to check */
- const char* param_name = params->data[param_id].name;
-
- /* Check whether the file still contains the old, now inappropriate
- * 'SelectOutput' section */
- if (strstr(param_name, "SelectOutput:") != NULL) {
- error(
- "Output selection files no longer require the use of top level "
- "SelectOutput; see the documentation for changes.");
- }
-
- /* Skip if the parameter belongs to another output class or is a
- * 'Standard' parameter */
- if (strstr(param_name, section_name) == NULL) continue;
- if (strstr(param_name, ":Standard_") != NULL) continue;
-
- /* Get the particle type for current parameter
- * (raises an error if it could not determine it) */
- const int param_ptype = get_param_ptype(param_name);
-
- /* Issue a warning if this parameter does not pertain to any of the
- * known fields from this ptype. */
- int field_id = 0;
- char field_name[PARSER_MAX_LINE_SIZE];
- for (field_id = 0; field_id < ptype_num_fields_total[param_ptype];
- field_id++) {
-
- sprintf(field_name, "%s:%.*s_%s", section_name, FIELD_BUFFER_SIZE,
- field_list[param_ptype][field_id].name,
- part_type_names[param_ptype]);
-
- if (strcmp(param_name, field_name) == 0) break;
- }
-
- int param_is_known = 0; /* Update below if it is a known one */
- if (field_id < ptype_num_fields_total[param_ptype])
- param_is_known = 1;
- else
- message(
- "WARNING: Trying to change behaviour of field '%s' (read from "
- "'%s') that does not exist. This may be because you are not "
- "running with all of the physics that you compiled the code with.",
- param_name, params->fileName);
-
- /* Perform a correctness check on the _value_ of the parameter */
- char param_value[FIELD_BUFFER_SIZE];
- parser_get_param_string(params, param_name, param_value);
-
- int value_id = 0;
- for (value_id = 0; value_id < compression_level_count; value_id++)
- if (strcmp(param_value, lossy_compression_schemes_names[value_id]) == 0)
- break;
-
- if (value_id == compression_level_count)
- error("Choice of output selection parameter %s ('%s') is invalid.",
- param_name, param_value);
-
- /* Adjust number of fields to be written for param_ptype, if this field's
- * status is different from default and it is a known one. */
- if (param_is_known) {
- const int is_on =
- strcmp(param_value,
- lossy_compression_schemes_names[compression_do_not_write]) !=
- 0;
-
- if (is_on && !ptype_default_write_status[param_ptype]) {
- /* Particle should be written even though default is off:
- * increase field count */
- ptype_num_fields_to_write[param_ptype] += 1;
- }
- if (!is_on && ptype_default_write_status[param_ptype]) {
- /* Particle should not be written, even though default is on:
- * decrease field count */
- ptype_num_fields_to_write[param_ptype] -= 1;
- }
- }
- } /* ends loop over parameters */
-
- /* Second loop over ptypes, to write out total number of fields to write */
- for (int ptype = 0; ptype < swift_type_count; ptype++) {
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Sanity check: is the number of fields to write non-negative? */
- if (ptype_num_fields_to_write[ptype] < 0)
- error(
- "We seem to have subtracted too many fields for particle "
- "type %d in output class %s (total to write is %d)",
- ptype, section_name, ptype_num_fields_to_write[ptype]);
-#endif
- output_options->num_fields_to_write[section_id][ptype] =
- ptype_num_fields_to_write[ptype];
- }
- } /* Ends loop over sections, for different output classes */
-
- /* Add field numbers for (possible) implicit `Default` output class */
- if (!have_default) {
- const int default_id = output_options->select_output->sectionCount;
- for (int ptype = 0; ptype < swift_type_count; ptype++)
- output_options->num_fields_to_write[default_id][ptype] =
- ptype_num_fields_total[ptype];
- }
-}
-
-/**
- * @brief Write the output field parameters file
- *
- * @param filename The file to write.
- * @param with_cosmology Use cosmological name variant?
- */
-void io_write_output_field_parameter(const char* filename, int with_cosmology) {
-
- FILE* file = fopen(filename, "w");
- if (file == NULL) error("Error opening file '%s'", filename);
-
- /* Create a fake unit system for the snapshots */
- struct unit_system snapshot_units;
- units_init_cgs(&snapshot_units);
-
- /* Loop over all particle types */
- fprintf(file, "Default:\n");
- for (int ptype = 0; ptype < swift_type_count; ptype++) {
-
- struct io_props list[100];
- int num_fields = get_ptype_fields(ptype, list, with_cosmology,
- /*with_fof=*/1, /*with_stf=*/1);
-
- if (num_fields == 0) continue;
-
- /* Output a header for that particle type */
- fprintf(file, " # Particle Type %s\n", part_type_names[ptype]);
-
- /* Write all the fields of this particle type */
- for (int i = 0; i < num_fields; ++i) {
-
- char unit_buffer[FIELD_BUFFER_SIZE] = {0};
- units_cgs_conversion_string(unit_buffer, &snapshot_units, list[i].units,
- list[i].scale_factor_exponent);
-
- /* Need to buffer with a maximal size - otherwise we can't read in again
- * because comments are too long */
- char comment_write_buffer[PARSER_MAX_LINE_SIZE / 2];
-
- sprintf(comment_write_buffer, "%.*s", PARSER_MAX_LINE_SIZE / 2 - 1,
- list[i].description);
-
- /* If our string is too long, replace the last few characters (before
- * \0) with ... for 'fancy printing' */
- if (strlen(comment_write_buffer) > PARSER_MAX_LINE_SIZE / 2 - 3) {
- strcpy(&comment_write_buffer[PARSER_MAX_LINE_SIZE / 2 - 4], "...");
- }
-
- fprintf(file, " %s_%s: %s # %s : %s\n", list[i].name,
- part_type_names[ptype], "on", comment_write_buffer, unit_buffer);
- }
-
- fprintf(file, "\n");
- }
-
- fclose(file);
-
- printf(
- "List of valid ouput fields for the particle in snapshots dumped in "
- "'%s'.\n",
- filename);
-}
-
/**
* @brief Create the subdirectory for snapshots if the user demanded one.
*
@@ -2831,114 +1408,6 @@ void io_get_snapshot_filename(char filename[1024], char xmf_filename[1024],
sprintf(xmf_filename, "%s.xmf", basename);
}
}
-
-/**
- * @brief Return the number and names of all output fields of a given ptype.
- *
- * @param ptype The index of the particle type under consideration.
- * @param list An io_props list that will hold the individual fields.
- * @param with_cosmology Use cosmological name variant?
- * @param with_fof Include FoF related fields?
- * @param with_stf Include STF related fields?
- *
- * @return The total number of fields that can be written for the ptype.
- */
-int get_ptype_fields(const int ptype, struct io_props* list,
- const int with_cosmology, const int with_fof,
- const int with_stf) {
-
- int num_fields = 0;
-
- switch (ptype) {
-
- case swift_type_gas:
- hydro_write_particles(NULL, NULL, list, &num_fields);
- num_fields += chemistry_write_particles(NULL, NULL, list + num_fields,
- with_cosmology);
- num_fields +=
- cooling_write_particles(NULL, NULL, list + num_fields, NULL);
- num_fields += tracers_write_particles(NULL, NULL, list + num_fields,
- with_cosmology);
- num_fields +=
- star_formation_write_particles(NULL, NULL, list + num_fields);
- if (with_fof)
- num_fields += fof_write_parts(NULL, NULL, list + num_fields);
- if (with_stf)
- num_fields += velociraptor_write_parts(NULL, NULL, list + num_fields);
- num_fields += rt_write_particles(NULL, list + num_fields);
- break;
-
- case swift_type_dark_matter:
- darkmatter_write_particles(NULL, list, &num_fields);
- if (with_fof) num_fields += fof_write_gparts(NULL, list + num_fields);
- if (with_stf)
- num_fields += velociraptor_write_gparts(NULL, list + num_fields);
- break;
-
- case swift_type_dark_matter_background:
- darkmatter_write_particles(NULL, list, &num_fields);
- if (with_fof) num_fields += fof_write_gparts(NULL, list + num_fields);
- if (with_stf)
- num_fields += velociraptor_write_gparts(NULL, list + num_fields);
- break;
-
- case swift_type_stars:
- stars_write_particles(NULL, list, &num_fields, with_cosmology);
- num_fields += chemistry_write_sparticles(NULL, list + num_fields);
- num_fields +=
- tracers_write_sparticles(NULL, list + num_fields, with_cosmology);
- num_fields += star_formation_write_sparticles(NULL, list + num_fields);
- if (with_fof) num_fields += fof_write_sparts(NULL, list + num_fields);
- if (with_stf)
- num_fields += velociraptor_write_sparts(NULL, list + num_fields);
- num_fields += rt_write_stars(NULL, list + num_fields);
- break;
-
- case swift_type_sink:
- sink_write_particles(NULL, list, &num_fields, with_cosmology);
- break;
-
- case swift_type_black_hole:
- black_holes_write_particles(NULL, list, &num_fields, with_cosmology);
- num_fields += chemistry_write_bparticles(NULL, list + num_fields);
- if (with_fof) num_fields += fof_write_bparts(NULL, list + num_fields);
- if (with_stf)
- num_fields += velociraptor_write_bparts(NULL, list + num_fields);
- break;
-
- default:
- error("Particle Type %d not yet supported. Aborting", ptype);
- }
-
- return num_fields;
-}
-
-/**
- * @brief Return the particle type code of a select_output parameter
- *
- * @param name The name of the parameter under consideration.
- *
- * @return The (integer) particle type of the parameter.
- */
-int get_param_ptype(const char* name) {
-
- const int name_len = strlen(name);
-
- for (int ptype = 0; ptype < swift_type_count; ptype++) {
- const int ptype_name_len = strlen(part_type_names[ptype]);
- if (name_len >= ptype_name_len &&
- strcmp(&name[name_len - ptype_name_len], part_type_names[ptype]) == 0)
- return ptype;
- }
-
- /* If we get here, we could not match the name, so something's gone wrong. */
- error("Could not determine the particle type for parameter '%s'.", name);
-
- /* We can never get here, but the compiler may complain if we don't return
- * an int after promising to do so... */
- return -1;
-}
-
/**
* @brief Set all ParticleIDs for each gpart to 1.
*
@@ -2950,6 +1419,6 @@ int get_param_ptype(const char* name) {
* @param gparts The array of loaded gparts.
* @param Ngparts Number of loaded gparts.
*/
-void set_ids_to_one(struct gpart* gparts, const size_t Ngparts) {
+void io_set_ids_to_one(struct gpart* gparts, const size_t Ngparts) {
for (size_t i = 0; i < Ngparts; i++) gparts[i].id_or_neg_offset = 1;
}
diff --git a/src/common_io.h b/src/common_io.h
index e8fc1fc4f68fbdcae027974593b6d4f124cd1fc7..95f00cf6c3426cb99b6c1d045dd265302cbad468 100644
--- a/src/common_io.h
+++ b/src/common_io.h
@@ -193,10 +193,6 @@ void io_get_snapshot_filename(char filename[1024], char xmf_filename[1024],
const int stf_count, const int snap_count,
const char* subdir, const char* basename);
-int get_ptype_fields(const int ptype, struct io_props* list,
- const int with_cosmology, const int with_fof,
- const int with_stf);
-int get_param_ptype(const char* name);
-void set_ids_to_one(struct gpart* gparts, const size_t Ngparts);
+void io_set_ids_to_one(struct gpart* gparts, const size_t Ngparts);
#endif /* SWIFT_COMMON_IO_H */
diff --git a/src/common_io_cells.c b/src/common_io_cells.c
new file mode 100644
index 0000000000000000000000000000000000000000..31bc6521e80cf5ae4379e3df7f96e474d0e172b0
--- /dev/null
+++ b/src/common_io_cells.c
@@ -0,0 +1,491 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2020 Matthieu Schaller (schaller@strw.leidenuniv.nl)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* This object's header. */
+#include "common_io.h"
+
+/* Local includes. */
+#include "cell.h"
+#include "timeline.h"
+#include "units.h"
+
+#if defined(HAVE_HDF5)
+
+#include <hdf5.h>
+
+/* MPI headers. */
+#ifdef WITH_MPI
+#include <mpi.h>
+#endif
+
+static long long cell_count_non_inhibited_gas(const struct cell* c) {
+ const int total_count = c->hydro.count;
+ struct part* parts = c->hydro.parts;
+ long long count = 0;
+ for (int i = 0; i < total_count; ++i) {
+ if ((parts[i].time_bin != time_bin_inhibited) &&
+ (parts[i].time_bin != time_bin_not_created)) {
+ ++count;
+ }
+ }
+ return count;
+}
+
+static long long cell_count_non_inhibited_dark_matter(const struct cell* c) {
+ const int total_count = c->grav.count;
+ struct gpart* gparts = c->grav.parts;
+ long long count = 0;
+ for (int i = 0; i < total_count; ++i) {
+ if ((gparts[i].time_bin != time_bin_inhibited) &&
+ (gparts[i].time_bin != time_bin_not_created) &&
+ (gparts[i].type == swift_type_dark_matter)) {
+ ++count;
+ }
+ }
+ return count;
+}
+
+static long long cell_count_non_inhibited_background_dark_matter(
+ const struct cell* c) {
+ const int total_count = c->grav.count;
+ struct gpart* gparts = c->grav.parts;
+ long long count = 0;
+ for (int i = 0; i < total_count; ++i) {
+ if ((gparts[i].time_bin != time_bin_inhibited) &&
+ (gparts[i].time_bin != time_bin_not_created) &&
+ (gparts[i].type == swift_type_dark_matter_background)) {
+ ++count;
+ }
+ }
+ return count;
+}
+
+static long long cell_count_non_inhibited_stars(const struct cell* c) {
+ const int total_count = c->stars.count;
+ struct spart* sparts = c->stars.parts;
+ long long count = 0;
+ for (int i = 0; i < total_count; ++i) {
+ if ((sparts[i].time_bin != time_bin_inhibited) &&
+ (sparts[i].time_bin != time_bin_not_created)) {
+ ++count;
+ }
+ }
+ return count;
+}
+
+static long long cell_count_non_inhibited_black_holes(const struct cell* c) {
+ const int total_count = c->black_holes.count;
+ struct bpart* bparts = c->black_holes.parts;
+ long long count = 0;
+ for (int i = 0; i < total_count; ++i) {
+ if ((bparts[i].time_bin != time_bin_inhibited) &&
+ (bparts[i].time_bin != time_bin_not_created)) {
+ ++count;
+ }
+ }
+ return count;
+}
+
+static long long cell_count_non_inhibited_sinks(const struct cell* c) {
+ const int total_count = c->sinks.count;
+ struct sink* sinks = c->sinks.parts;
+ long long count = 0;
+ for (int i = 0; i < total_count; ++i) {
+ if ((sinks[i].time_bin != time_bin_inhibited) &&
+ (sinks[i].time_bin != time_bin_not_created)) {
+ ++count;
+ }
+ }
+ return count;
+}
+
+/**
+ * @brief Write a single 1D array to a hdf5 group.
+ *
+ * This creates a simple Nx1 array with a chunk size of 1024x1.
+ * The Fletcher-32 filter is applied to the array.
+ *
+ * @param h_grp The open hdf5 group.
+ * @param n The number of elements in the array.
+ * @param array The data to write.
+ * @param type The type of the data to write.
+ * @param name The name of the array.
+ * @param array_content The name of the parent group (only used for error
+ * messages).
+ */
+void io_write_array(hid_t h_grp, const int n, const void* array,
+ const enum IO_DATA_TYPE type, const char* name,
+ const char* array_content) {
+
+ /* Create memory space */
+ const hsize_t shape[2] = {(hsize_t)n, 1};
+ hid_t h_space = H5Screate(H5S_SIMPLE);
+ if (h_space < 0)
+ error("Error while creating data space for %s %s", name, array_content);
+ hid_t h_err = H5Sset_extent_simple(h_space, 1, shape, shape);
+ if (h_err < 0)
+ error("Error while changing shape of %s %s data space.", name,
+ array_content);
+
+ /* Dataset type */
+ hid_t h_type = H5Tcopy(io_hdf5_type(type));
+
+ const hsize_t chunk[2] = {(1024 > n ? n : 1024), 1};
+ hid_t h_prop = H5Pcreate(H5P_DATASET_CREATE);
+ h_err = H5Pset_chunk(h_prop, 1, chunk);
+ if (h_err < 0)
+ error("Error while setting chunk shapes of %s %s data space.", name,
+ array_content);
+
+ /* Impose check-sum to verify data corruption */
+ h_err = H5Pset_fletcher32(h_prop);
+ if (h_err < 0)
+ error("Error while setting check-sum filter on %s %s data space.", name,
+ array_content);
+
+ /* Write */
+ hid_t h_data =
+ H5Dcreate(h_grp, name, h_type, h_space, H5P_DEFAULT, h_prop, H5P_DEFAULT);
+ if (h_data < 0)
+ error("Error while creating dataspace for %s %s.", name, array_content);
+ h_err = H5Dwrite(h_data, io_hdf5_type(type), h_space, H5S_ALL, H5P_DEFAULT,
+ array);
+ if (h_err < 0) error("Error while writing %s %s.", name, array_content);
+ H5Tclose(h_type);
+ H5Dclose(h_data);
+ H5Pclose(h_prop);
+ H5Sclose(h_space);
+}
+
+void io_write_cell_offsets(hid_t h_grp, const int cdim[3], const double dim[3],
+ const struct cell* cells_top, const int nr_cells,
+ const double width[3], const int nodeID,
+ const int distributed,
+ const long long global_counts[swift_type_count],
+ const long long global_offsets[swift_type_count],
+ const int num_fields[swift_type_count],
+ const struct unit_system* internal_units,
+ const struct unit_system* snapshot_units) {
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (distributed) {
+ if (global_offsets[0] != 0 || global_offsets[1] != 0 ||
+ global_offsets[2] != 0 || global_offsets[3] != 0 ||
+ global_offsets[4] != 0 || global_offsets[5] != 0)
+ error("Global offset non-zero in the distributed io case!");
+ }
+#endif
+
+ /* Abort if we don't have any cells yet (i.e. haven't constructed the space)
+ */
+ if (nr_cells == 0) return;
+
+ double cell_width[3] = {width[0], width[1], width[2]};
+
+ /* Temporary memory for the cell-by-cell information */
+ double* centres = NULL;
+ centres = (double*)malloc(3 * nr_cells * sizeof(double));
+
+ /* Temporary memory for the cell files ID */
+ int* files = NULL;
+ files = (int*)malloc(nr_cells * sizeof(int));
+
+ /* Count of particles in each cell */
+ long long *count_part = NULL, *count_gpart = NULL,
+ *count_background_gpart = NULL, *count_spart = NULL,
+ *count_bpart = NULL, *count_sink = NULL;
+ count_part = (long long*)malloc(nr_cells * sizeof(long long));
+ count_gpart = (long long*)malloc(nr_cells * sizeof(long long));
+ count_background_gpart = (long long*)malloc(nr_cells * sizeof(long long));
+ count_spart = (long long*)malloc(nr_cells * sizeof(long long));
+ count_bpart = (long long*)malloc(nr_cells * sizeof(long long));
+ count_sink = (long long*)malloc(nr_cells * sizeof(long long));
+
+ /* Global offsets of particles in each cell */
+ long long *offset_part = NULL, *offset_gpart = NULL,
+ *offset_background_gpart = NULL, *offset_spart = NULL,
+ *offset_bpart = NULL, *offset_sink = NULL;
+ offset_part = (long long*)malloc(nr_cells * sizeof(long long));
+ offset_gpart = (long long*)malloc(nr_cells * sizeof(long long));
+ offset_background_gpart = (long long*)malloc(nr_cells * sizeof(long long));
+ offset_spart = (long long*)malloc(nr_cells * sizeof(long long));
+ offset_bpart = (long long*)malloc(nr_cells * sizeof(long long));
+ offset_sink = (long long*)malloc(nr_cells * sizeof(long long));
+
+ /* Offsets of the 0^th element */
+ offset_part[0] = 0;
+ offset_gpart[0] = 0;
+ offset_background_gpart[0] = 0;
+ offset_spart[0] = 0;
+ offset_bpart[0] = 0;
+ offset_sink[0] = 0;
+
+ /* Collect the cell information of *local* cells */
+ long long local_offset_part = 0;
+ long long local_offset_gpart = 0;
+ long long local_offset_background_gpart = 0;
+ long long local_offset_spart = 0;
+ long long local_offset_bpart = 0;
+ long long local_offset_sink = 0;
+ for (int i = 0; i < nr_cells; ++i) {
+
+ /* Store in which file this cell will be found */
+ if (distributed) {
+ files[i] = cells_top[i].nodeID;
+ } else {
+ files[i] = 0;
+ }
+
+ /* Is the cell on this node (i.e. we have full information */
+ if (cells_top[i].nodeID == nodeID) {
+
+ /* Centre of each cell */
+ centres[i * 3 + 0] = cells_top[i].loc[0] + cell_width[0] * 0.5;
+ centres[i * 3 + 1] = cells_top[i].loc[1] + cell_width[1] * 0.5;
+ centres[i * 3 + 2] = cells_top[i].loc[2] + cell_width[2] * 0.5;
+
+ /* Finish by box wrapping to match what is done to the particles */
+ centres[i * 3 + 0] = box_wrap(centres[i * 3 + 0], 0.0, dim[0]);
+ centres[i * 3 + 1] = box_wrap(centres[i * 3 + 1], 0.0, dim[1]);
+ centres[i * 3 + 2] = box_wrap(centres[i * 3 + 2], 0.0, dim[2]);
+
+ /* Count real particles that will be written */
+ count_part[i] = cell_count_non_inhibited_gas(&cells_top[i]);
+ count_gpart[i] = cell_count_non_inhibited_dark_matter(&cells_top[i]);
+ count_background_gpart[i] =
+ cell_count_non_inhibited_background_dark_matter(&cells_top[i]);
+ count_spart[i] = cell_count_non_inhibited_stars(&cells_top[i]);
+ count_bpart[i] = cell_count_non_inhibited_black_holes(&cells_top[i]);
+ count_sink[i] = cell_count_non_inhibited_sinks(&cells_top[i]);
+
+ /* Offsets including the global offset of all particles on this MPI rank
+ * Note that in the distributed case, the global offsets are 0 such that
+ * we actually compute the offset in the file written by this rank. */
+ offset_part[i] = local_offset_part + global_offsets[swift_type_gas];
+ offset_gpart[i] =
+ local_offset_gpart + global_offsets[swift_type_dark_matter];
+ offset_background_gpart[i] =
+ local_offset_background_gpart +
+ global_offsets[swift_type_dark_matter_background];
+ offset_spart[i] = local_offset_spart + global_offsets[swift_type_stars];
+ offset_bpart[i] =
+ local_offset_bpart + global_offsets[swift_type_black_hole];
+ offset_sink[i] = local_offset_sink + global_offsets[swift_type_sink];
+
+ local_offset_part += count_part[i];
+ local_offset_gpart += count_gpart[i];
+ local_offset_background_gpart += count_background_gpart[i];
+ local_offset_spart += count_spart[i];
+ local_offset_bpart += count_bpart[i];
+ local_offset_sink += count_sink[i];
+
+ } else {
+
+ /* Just zero everything for the foregin cells */
+
+ centres[i * 3 + 0] = 0.;
+ centres[i * 3 + 1] = 0.;
+ centres[i * 3 + 2] = 0.;
+
+ count_part[i] = 0;
+ count_gpart[i] = 0;
+ count_background_gpart[i] = 0;
+ count_spart[i] = 0;
+ count_bpart[i] = 0;
+ count_sink[i] = 0;
+
+ offset_part[i] = 0;
+ offset_gpart[i] = 0;
+ offset_background_gpart[i] = 0;
+ offset_spart[i] = 0;
+ offset_bpart[i] = 0;
+ offset_sink[i] = 0;
+ }
+ }
+
+#ifdef WITH_MPI
+ /* Now, reduce all the arrays. Note that we use a bit-wise OR here. This
+ is safe as we made sure only local cells have non-zero values. */
+ MPI_Allreduce(MPI_IN_PLACE, count_part, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
+ MPI_COMM_WORLD);
+ MPI_Allreduce(MPI_IN_PLACE, count_gpart, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
+ MPI_COMM_WORLD);
+ MPI_Allreduce(MPI_IN_PLACE, count_background_gpart, nr_cells,
+ MPI_LONG_LONG_INT, MPI_BOR, MPI_COMM_WORLD);
+ MPI_Allreduce(MPI_IN_PLACE, count_sink, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
+ MPI_COMM_WORLD);
+ MPI_Allreduce(MPI_IN_PLACE, count_spart, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
+ MPI_COMM_WORLD);
+ MPI_Allreduce(MPI_IN_PLACE, count_bpart, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
+ MPI_COMM_WORLD);
+
+ MPI_Allreduce(MPI_IN_PLACE, offset_part, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
+ MPI_COMM_WORLD);
+ MPI_Allreduce(MPI_IN_PLACE, offset_gpart, nr_cells, MPI_LONG_LONG_INT,
+ MPI_BOR, MPI_COMM_WORLD);
+ MPI_Allreduce(MPI_IN_PLACE, offset_background_gpart, nr_cells,
+ MPI_LONG_LONG_INT, MPI_BOR, MPI_COMM_WORLD);
+ MPI_Allreduce(MPI_IN_PLACE, offset_sink, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
+ MPI_COMM_WORLD);
+ MPI_Allreduce(MPI_IN_PLACE, offset_spart, nr_cells, MPI_LONG_LONG_INT,
+ MPI_BOR, MPI_COMM_WORLD);
+ MPI_Allreduce(MPI_IN_PLACE, offset_bpart, nr_cells, MPI_LONG_LONG_INT,
+ MPI_BOR, MPI_COMM_WORLD);
+
+ /* For the centres we use a sum as MPI does not like bit-wise operations
+ on floating point numbers */
+ MPI_Allreduce(MPI_IN_PLACE, centres, 3 * nr_cells, MPI_DOUBLE, MPI_SUM,
+ MPI_COMM_WORLD);
+#endif
+
+ /* When writing a single file, only rank 0 writes the meta-data */
+ if ((distributed) || (!distributed && nodeID == 0)) {
+
+ /* Unit conversion if necessary */
+ const double factor = units_conversion_factor(
+ internal_units, snapshot_units, UNIT_CONV_LENGTH);
+ if (factor != 1.) {
+
+ /* Convert the cell centres */
+ for (int i = 0; i < nr_cells; ++i) {
+ centres[i * 3 + 0] *= factor;
+ centres[i * 3 + 1] *= factor;
+ centres[i * 3 + 2] *= factor;
+ }
+
+ /* Convert the cell widths */
+ cell_width[0] *= factor;
+ cell_width[1] *= factor;
+ cell_width[2] *= factor;
+ }
+
+ /* Write some meta-information first */
+ hid_t h_subgrp =
+ H5Gcreate(h_grp, "Meta-data", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ if (h_subgrp < 0) error("Error while creating meta-data sub-group");
+ io_write_attribute(h_subgrp, "nr_cells", INT, &nr_cells, 1);
+ io_write_attribute(h_subgrp, "size", DOUBLE, cell_width, 3);
+ io_write_attribute(h_subgrp, "dimension", INT, cdim, 3);
+ H5Gclose(h_subgrp);
+
+ /* Write the centres to the group */
+ hsize_t shape[2] = {(hsize_t)nr_cells, 3};
+ hid_t h_space = H5Screate(H5S_SIMPLE);
+ if (h_space < 0) error("Error while creating data space for cell centres");
+ hid_t h_err = H5Sset_extent_simple(h_space, 2, shape, shape);
+ if (h_err < 0)
+ error("Error while changing shape of gas offsets data space.");
+ hid_t h_data = H5Dcreate(h_grp, "Centres", io_hdf5_type(DOUBLE), h_space,
+ H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ if (h_data < 0) error("Error while creating dataspace for gas offsets.");
+ h_err = H5Dwrite(h_data, io_hdf5_type(DOUBLE), h_space, H5S_ALL,
+ H5P_DEFAULT, centres);
+ if (h_err < 0) error("Error while writing centres.");
+ H5Dclose(h_data);
+ H5Sclose(h_space);
+
+ /* Group containing the offsets and counts for each particle type */
+ hid_t h_grp_offsets = H5Gcreate(h_grp, "OffsetsInFile", H5P_DEFAULT,
+ H5P_DEFAULT, H5P_DEFAULT);
+ if (h_grp_offsets < 0) error("Error while creating offsets sub-group");
+ hid_t h_grp_files =
+ H5Gcreate(h_grp, "Files", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ if (h_grp_files < 0) error("Error while creating filess sub-group");
+ hid_t h_grp_counts =
+ H5Gcreate(h_grp, "Counts", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ if (h_grp_counts < 0) error("Error while creating counts sub-group");
+
+ if (global_counts[swift_type_gas] > 0 && num_fields[swift_type_gas] > 0) {
+ io_write_array(h_grp_files, nr_cells, files, INT, "PartType0", "files");
+ io_write_array(h_grp_offsets, nr_cells, offset_part, LONGLONG,
+ "PartType0", "offsets");
+ io_write_array(h_grp_counts, nr_cells, count_part, LONGLONG, "PartType0",
+ "counts");
+ }
+
+ if (global_counts[swift_type_dark_matter] > 0 &&
+ num_fields[swift_type_dark_matter] > 0) {
+ io_write_array(h_grp_files, nr_cells, files, INT, "PartType1", "files");
+ io_write_array(h_grp_offsets, nr_cells, offset_gpart, LONGLONG,
+ "PartType1", "offsets");
+ io_write_array(h_grp_counts, nr_cells, count_gpart, LONGLONG, "PartType1",
+ "counts");
+ }
+
+ if (global_counts[swift_type_dark_matter_background] > 0 &&
+ num_fields[swift_type_dark_matter_background] > 0) {
+ io_write_array(h_grp_files, nr_cells, files, INT, "PartType2", "files");
+ io_write_array(h_grp_offsets, nr_cells, offset_background_gpart, LONGLONG,
+ "PartType2", "offsets");
+ io_write_array(h_grp_counts, nr_cells, count_background_gpart, LONGLONG,
+ "PartType2", "counts");
+ }
+
+ if (global_counts[swift_type_sink] > 0 && num_fields[swift_type_sink] > 0) {
+ io_write_array(h_grp_files, nr_cells, files, INT, "PartType3", "files");
+ io_write_array(h_grp_offsets, nr_cells, offset_sink, LONGLONG,
+ "PartType3", "offsets");
+ io_write_array(h_grp_counts, nr_cells, count_sink, LONGLONG, "PartType3",
+ "counts");
+ }
+
+ if (global_counts[swift_type_stars] > 0 &&
+ num_fields[swift_type_stars] > 0) {
+ io_write_array(h_grp_files, nr_cells, files, INT, "PartType4", "files");
+ io_write_array(h_grp_offsets, nr_cells, offset_spart, LONGLONG,
+ "PartType4", "offsets");
+ io_write_array(h_grp_counts, nr_cells, count_spart, LONGLONG, "PartType4",
+ "counts");
+ }
+
+ if (global_counts[swift_type_black_hole] > 0 &&
+ num_fields[swift_type_black_hole] > 0) {
+ io_write_array(h_grp_files, nr_cells, files, INT, "PartType5", "files");
+ io_write_array(h_grp_offsets, nr_cells, offset_bpart, LONGLONG,
+ "PartType5", "offsets");
+ io_write_array(h_grp_counts, nr_cells, count_bpart, LONGLONG, "PartType5",
+ "counts");
+ }
+
+ H5Gclose(h_grp_offsets);
+ H5Gclose(h_grp_files);
+ H5Gclose(h_grp_counts);
+ }
+
+ /* Free everything we allocated */
+ free(centres);
+ free(files);
+ free(count_part);
+ free(count_gpart);
+ free(count_background_gpart);
+ free(count_spart);
+ free(count_bpart);
+ free(count_sink);
+ free(offset_part);
+ free(offset_gpart);
+ free(offset_background_gpart);
+ free(offset_spart);
+ free(offset_bpart);
+ free(offset_sink);
+}
+
+#endif /* HAVE_HDF5 */
diff --git a/src/common_io_copy.c b/src/common_io_copy.c
new file mode 100644
index 0000000000000000000000000000000000000000..e7e9f157726591a8e19c15c51df947e5617132ca
--- /dev/null
+++ b/src/common_io_copy.c
@@ -0,0 +1,753 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2017 Matthieu Schaller (matthieu.schaller@durham.ac.uk).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* This object's header. */
+#include "common_io.h"
+
+/* Local includes. */
+#include "engine.h"
+#include "io_properties.h"
+#include "threadpool.h"
+
+/**
+ * @brief Mapper function to copy #part or #gpart fields into a buffer.
+ */
+void io_copy_mapper(void* restrict temp, int N, void* restrict extra_data) {
+
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const size_t typeSize = io_sizeof_type(props.type);
+ const size_t copySize = typeSize * props.dimension;
+
+ /* How far are we with this chunk? */
+ char* restrict temp_c = (char*)temp;
+ const ptrdiff_t delta = (temp_c - props.start_temp_c) / copySize;
+
+ for (int k = 0; k < N; k++) {
+ memcpy(&temp_c[k * copySize], props.field + (delta + k) * props.partSize,
+ copySize);
+ }
+}
+
+/**
+ * @brief Mapper function to copy #part into a buffer of floats using a
+ * conversion function.
+ */
+void io_convert_part_f_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
+
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct part* restrict parts = props.parts;
+ const struct xpart* restrict xparts = props.xparts;
+ const struct engine* e = props.e;
+ const size_t dim = props.dimension;
+
+ /* How far are we with this chunk? */
+ float* restrict temp_f = (float*)temp;
+ const ptrdiff_t delta = (temp_f - props.start_temp_f) / dim;
+
+ for (int i = 0; i < N; i++)
+ props.convert_part_f(e, parts + delta + i, xparts + delta + i,
+ &temp_f[i * dim]);
+}
+
+/**
+ * @brief Mapper function to copy #part into a buffer of ints using a
+ * conversion function.
+ */
+void io_convert_part_i_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
+
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct part* restrict parts = props.parts;
+ const struct xpart* restrict xparts = props.xparts;
+ const struct engine* e = props.e;
+ const size_t dim = props.dimension;
+
+ /* How far are we with this chunk? */
+ int* restrict temp_i = (int*)temp;
+ const ptrdiff_t delta = (temp_i - props.start_temp_i) / dim;
+
+ for (int i = 0; i < N; i++)
+ props.convert_part_i(e, parts + delta + i, xparts + delta + i,
+ &temp_i[i * dim]);
+}
+
+/**
+ * @brief Mapper function to copy #part into a buffer of doubles using a
+ * conversion function.
+ */
+void io_convert_part_d_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
+
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct part* restrict parts = props.parts;
+ const struct xpart* restrict xparts = props.xparts;
+ const struct engine* e = props.e;
+ const size_t dim = props.dimension;
+
+ /* How far are we with this chunk? */
+ double* restrict temp_d = (double*)temp;
+ const ptrdiff_t delta = (temp_d - props.start_temp_d) / dim;
+
+ for (int i = 0; i < N; i++)
+ props.convert_part_d(e, parts + delta + i, xparts + delta + i,
+ &temp_d[i * dim]);
+}
+
+/**
+ * @brief Mapper function to copy #part into a buffer of doubles using a
+ * conversion function.
+ */
+void io_convert_part_l_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
+
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct part* restrict parts = props.parts;
+ const struct xpart* restrict xparts = props.xparts;
+ const struct engine* e = props.e;
+ const size_t dim = props.dimension;
+
+ /* How far are we with this chunk? */
+ long long* restrict temp_l = (long long*)temp;
+ const ptrdiff_t delta = (temp_l - props.start_temp_l) / dim;
+
+ for (int i = 0; i < N; i++)
+ props.convert_part_l(e, parts + delta + i, xparts + delta + i,
+ &temp_l[i * dim]);
+}
+
+/**
+ * @brief Mapper function to copy #gpart into a buffer of floats using a
+ * conversion function.
+ */
+void io_convert_gpart_f_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
+
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct gpart* restrict gparts = props.gparts;
+ const struct engine* e = props.e;
+ const size_t dim = props.dimension;
+
+ /* How far are we with this chunk? */
+ float* restrict temp_f = (float*)temp;
+ const ptrdiff_t delta = (temp_f - props.start_temp_f) / dim;
+
+ for (int i = 0; i < N; i++)
+ props.convert_gpart_f(e, gparts + delta + i, &temp_f[i * dim]);
+}
+
+/**
+ * @brief Mapper function to copy #gpart into a buffer of ints using a
+ * conversion function.
+ */
+void io_convert_gpart_i_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
+
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct gpart* restrict gparts = props.gparts;
+ const struct engine* e = props.e;
+ const size_t dim = props.dimension;
+
+ /* How far are we with this chunk? */
+ int* restrict temp_i = (int*)temp;
+ const ptrdiff_t delta = (temp_i - props.start_temp_i) / dim;
+
+ for (int i = 0; i < N; i++)
+ props.convert_gpart_i(e, gparts + delta + i, &temp_i[i * dim]);
+}
+
+/**
+ * @brief Mapper function to copy #gpart into a buffer of doubles using a
+ * conversion function.
+ */
+void io_convert_gpart_d_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
+
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct gpart* restrict gparts = props.gparts;
+ const struct engine* e = props.e;
+ const size_t dim = props.dimension;
+
+ /* How far are we with this chunk? */
+ double* restrict temp_d = (double*)temp;
+ const ptrdiff_t delta = (temp_d - props.start_temp_d) / dim;
+
+ for (int i = 0; i < N; i++)
+ props.convert_gpart_d(e, gparts + delta + i, &temp_d[i * dim]);
+}
+
+/**
+ * @brief Mapper function to copy #gpart into a buffer of doubles using a
+ * conversion function.
+ */
+void io_convert_gpart_l_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
+
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct gpart* restrict gparts = props.gparts;
+ const struct engine* e = props.e;
+ const size_t dim = props.dimension;
+
+ /* How far are we with this chunk? */
+ long long* restrict temp_l = (long long*)temp;
+ const ptrdiff_t delta = (temp_l - props.start_temp_l) / dim;
+
+ for (int i = 0; i < N; i++)
+ props.convert_gpart_l(e, gparts + delta + i, &temp_l[i * dim]);
+}
+
+/**
+ * @brief Mapper function to copy #spart into a buffer of floats using a
+ * conversion function.
+ */
+void io_convert_spart_f_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
+
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct spart* restrict sparts = props.sparts;
+ const struct engine* e = props.e;
+ const size_t dim = props.dimension;
+
+ /* How far are we with this chunk? */
+ float* restrict temp_f = (float*)temp;
+ const ptrdiff_t delta = (temp_f - props.start_temp_f) / dim;
+
+ for (int i = 0; i < N; i++)
+ props.convert_spart_f(e, sparts + delta + i, &temp_f[i * dim]);
+}
+
+/**
+ * @brief Mapper function to copy #spart into a buffer of ints using a
+ * conversion function.
+ */
+void io_convert_spart_i_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
+
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct spart* restrict sparts = props.sparts;
+ const struct engine* e = props.e;
+ const size_t dim = props.dimension;
+
+ /* How far are we with this chunk? */
+ int* restrict temp_i = (int*)temp;
+ const ptrdiff_t delta = (temp_i - props.start_temp_i) / dim;
+
+ for (int i = 0; i < N; i++)
+ props.convert_spart_i(e, sparts + delta + i, &temp_i[i * dim]);
+}
+
+/**
+ * @brief Mapper function to copy #spart into a buffer of doubles using a
+ * conversion function.
+ */
+void io_convert_spart_d_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
+
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct spart* restrict sparts = props.sparts;
+ const struct engine* e = props.e;
+ const size_t dim = props.dimension;
+
+ /* How far are we with this chunk? */
+ double* restrict temp_d = (double*)temp;
+ const ptrdiff_t delta = (temp_d - props.start_temp_d) / dim;
+
+ for (int i = 0; i < N; i++)
+ props.convert_spart_d(e, sparts + delta + i, &temp_d[i * dim]);
+}
+
+/**
+ * @brief Mapper function to copy #spart into a buffer of doubles using a
+ * conversion function.
+ */
+void io_convert_spart_l_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
+
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct spart* restrict sparts = props.sparts;
+ const struct engine* e = props.e;
+ const size_t dim = props.dimension;
+
+ /* How far are we with this chunk? */
+ long long* restrict temp_l = (long long*)temp;
+ const ptrdiff_t delta = (temp_l - props.start_temp_l) / dim;
+
+ for (int i = 0; i < N; i++)
+ props.convert_spart_l(e, sparts + delta + i, &temp_l[i * dim]);
+}
+
+/**
+ * @brief Mapper function to copy #bpart into a buffer of floats using a
+ * conversion function.
+ */
+void io_convert_bpart_f_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
+
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct bpart* restrict bparts = props.bparts;
+ const struct engine* e = props.e;
+ const size_t dim = props.dimension;
+
+ /* How far are we with this chunk? */
+ float* restrict temp_f = (float*)temp;
+ const ptrdiff_t delta = (temp_f - props.start_temp_f) / dim;
+
+ for (int i = 0; i < N; i++)
+ props.convert_bpart_f(e, bparts + delta + i, &temp_f[i * dim]);
+}
+
+/**
+ * @brief Mapper function to copy #bpart into a buffer of ints using a
+ * conversion function.
+ */
+void io_convert_bpart_i_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
+
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct bpart* restrict bparts = props.bparts;
+ const struct engine* e = props.e;
+ const size_t dim = props.dimension;
+
+ /* How far are we with this chunk? */
+ int* restrict temp_i = (int*)temp;
+ const ptrdiff_t delta = (temp_i - props.start_temp_i) / dim;
+
+ for (int i = 0; i < N; i++)
+ props.convert_bpart_i(e, bparts + delta + i, &temp_i[i * dim]);
+}
+
+/**
+ * @brief Mapper function to copy #bpart into a buffer of doubles using a
+ * conversion function.
+ */
+void io_convert_bpart_d_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
+
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct bpart* restrict bparts = props.bparts;
+ const struct engine* e = props.e;
+ const size_t dim = props.dimension;
+
+ /* How far are we with this chunk? */
+ double* restrict temp_d = (double*)temp;
+ const ptrdiff_t delta = (temp_d - props.start_temp_d) / dim;
+
+ for (int i = 0; i < N; i++)
+ props.convert_bpart_d(e, bparts + delta + i, &temp_d[i * dim]);
+}
+
+/**
+ * @brief Mapper function to copy #bpart into a buffer of doubles using a
+ * conversion function.
+ */
+void io_convert_bpart_l_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
+
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct bpart* restrict bparts = props.bparts;
+ const struct engine* e = props.e;
+ const size_t dim = props.dimension;
+
+ /* How far are we with this chunk? */
+ long long* restrict temp_l = (long long*)temp;
+ const ptrdiff_t delta = (temp_l - props.start_temp_l) / dim;
+
+ for (int i = 0; i < N; i++)
+ props.convert_bpart_l(e, bparts + delta + i, &temp_l[i * dim]);
+}
+
+/**
+ * @brief Mapper function to copy #sink into a buffer of floats using a
+ * conversion function.
+ */
+void io_convert_sink_f_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
+
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct sink* restrict sinks = props.sinks;
+ const struct engine* e = props.e;
+ const size_t dim = props.dimension;
+
+ /* How far are we with this chunk? */
+ float* restrict temp_f = (float*)temp;
+ const ptrdiff_t delta = (temp_f - props.start_temp_f) / dim;
+
+ for (int i = 0; i < N; i++)
+ props.convert_sink_f(e, sinks + delta + i, &temp_f[i * dim]);
+}
+
+/**
+ * @brief Mapper function to copy #sink into a buffer of ints using a
+ * conversion function.
+ */
+void io_convert_sink_i_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
+
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct sink* restrict sinks = props.sinks;
+ const struct engine* e = props.e;
+ const size_t dim = props.dimension;
+
+ /* How far are we with this chunk? */
+ int* restrict temp_i = (int*)temp;
+ const ptrdiff_t delta = (temp_i - props.start_temp_i) / dim;
+
+ for (int i = 0; i < N; i++)
+ props.convert_sink_i(e, sinks + delta + i, &temp_i[i * dim]);
+}
+
+/**
+ * @brief Mapper function to copy #sink into a buffer of doubles using a
+ * conversion function.
+ */
+void io_convert_sink_d_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
+
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct sink* restrict sinks = props.sinks;
+ const struct engine* e = props.e;
+ const size_t dim = props.dimension;
+
+ /* How far are we with this chunk? */
+ double* restrict temp_d = (double*)temp;
+ const ptrdiff_t delta = (temp_d - props.start_temp_d) / dim;
+
+ for (int i = 0; i < N; i++)
+ props.convert_sink_d(e, sinks + delta + i, &temp_d[i * dim]);
+}
+
+/**
+ * @brief Mapper function to copy #sink into a buffer of doubles using a
+ * conversion function.
+ */
+void io_convert_sink_l_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
+
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct sink* restrict sinks = props.sinks;
+ const struct engine* e = props.e;
+ const size_t dim = props.dimension;
+
+ /* How far are we with this chunk? */
+ long long* restrict temp_l = (long long*)temp;
+ const ptrdiff_t delta = (temp_l - props.start_temp_l) / dim;
+
+ for (int i = 0; i < N; i++)
+ props.convert_sink_l(e, sinks + delta + i, &temp_l[i * dim]);
+}
+
+/**
+ * @brief Copy the particle data into a temporary buffer ready for i/o.
+ *
+ * @param temp The buffer to be filled. Must be allocated and aligned properly.
+ * @param e The #engine.
+ * @param props The #io_props corresponding to the particle field we are
+ * copying.
+ * @param N The number of particles to copy
+ * @param internal_units The system of units used internally.
+ * @param snapshot_units The system of units used for the snapshots.
+ */
+void io_copy_temp_buffer(void* temp, const struct engine* e,
+ struct io_props props, size_t N,
+ const struct unit_system* internal_units,
+ const struct unit_system* snapshot_units) {
+
+ const size_t typeSize = io_sizeof_type(props.type);
+ const size_t copySize = typeSize * props.dimension;
+ const size_t num_elements = N * props.dimension;
+
+ /* Copy particle data to temporary buffer */
+ if (props.conversion == 0) { /* No conversion */
+
+ /* Prepare some parameters */
+ char* temp_c = (char*)temp;
+ props.start_temp_c = temp_c;
+
+ /* Copy the whole thing into a buffer */
+ threadpool_map((struct threadpool*)&e->threadpool, io_copy_mapper, temp_c,
+ N, copySize, threadpool_auto_chunk_size, (void*)&props);
+
+ } else { /* Converting particle to data */
+
+ if (props.convert_part_f != NULL) {
+
+ /* Prepare some parameters */
+ float* temp_f = (float*)temp;
+ props.start_temp_f = (float*)temp;
+ props.e = e;
+
+ /* Copy the whole thing into a buffer */
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_part_f_mapper, temp_f, N, copySize,
+ threadpool_auto_chunk_size, (void*)&props);
+
+ } else if (props.convert_part_i != NULL) {
+
+ /* Prepare some parameters */
+ int* temp_i = (int*)temp;
+ props.start_temp_i = (int*)temp;
+ props.e = e;
+
+ /* Copy the whole thing into a buffer */
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_part_i_mapper, temp_i, N, copySize,
+ threadpool_auto_chunk_size, (void*)&props);
+
+ } else if (props.convert_part_d != NULL) {
+
+ /* Prepare some parameters */
+ double* temp_d = (double*)temp;
+ props.start_temp_d = (double*)temp;
+ props.e = e;
+
+ /* Copy the whole thing into a buffer */
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_part_d_mapper, temp_d, N, copySize,
+ threadpool_auto_chunk_size, (void*)&props);
+
+ } else if (props.convert_part_l != NULL) {
+
+ /* Prepare some parameters */
+ long long* temp_l = (long long*)temp;
+ props.start_temp_l = (long long*)temp;
+ props.e = e;
+
+ /* Copy the whole thing into a buffer */
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_part_l_mapper, temp_l, N, copySize,
+ threadpool_auto_chunk_size, (void*)&props);
+
+ } else if (props.convert_gpart_f != NULL) {
+
+ /* Prepare some parameters */
+ float* temp_f = (float*)temp;
+ props.start_temp_f = (float*)temp;
+ props.e = e;
+
+ /* Copy the whole thing into a buffer */
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_gpart_f_mapper, temp_f, N, copySize,
+ threadpool_auto_chunk_size, (void*)&props);
+
+ } else if (props.convert_gpart_i != NULL) {
+
+ /* Prepare some parameters */
+ int* temp_i = (int*)temp;
+ props.start_temp_i = (int*)temp;
+ props.e = e;
+
+ /* Copy the whole thing into a buffer */
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_gpart_i_mapper, temp_i, N, copySize,
+ threadpool_auto_chunk_size, (void*)&props);
+
+ } else if (props.convert_gpart_d != NULL) {
+
+ /* Prepare some parameters */
+ double* temp_d = (double*)temp;
+ props.start_temp_d = (double*)temp;
+ props.e = e;
+
+ /* Copy the whole thing into a buffer */
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_gpart_d_mapper, temp_d, N, copySize,
+ threadpool_auto_chunk_size, (void*)&props);
+
+ } else if (props.convert_gpart_l != NULL) {
+
+ /* Prepare some parameters */
+ long long* temp_l = (long long*)temp;
+ props.start_temp_l = (long long*)temp;
+ props.e = e;
+
+ /* Copy the whole thing into a buffer */
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_gpart_l_mapper, temp_l, N, copySize,
+ threadpool_auto_chunk_size, (void*)&props);
+
+ } else if (props.convert_spart_f != NULL) {
+
+ /* Prepare some parameters */
+ float* temp_f = (float*)temp;
+ props.start_temp_f = (float*)temp;
+ props.e = e;
+
+ /* Copy the whole thing into a buffer */
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_spart_f_mapper, temp_f, N, copySize,
+ threadpool_auto_chunk_size, (void*)&props);
+
+ } else if (props.convert_spart_i != NULL) {
+
+ /* Prepare some parameters */
+ int* temp_i = (int*)temp;
+ props.start_temp_i = (int*)temp;
+ props.e = e;
+
+ /* Copy the whole thing into a buffer */
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_spart_i_mapper, temp_i, N, copySize,
+ threadpool_auto_chunk_size, (void*)&props);
+
+ } else if (props.convert_spart_d != NULL) {
+
+ /* Prepare some parameters */
+ double* temp_d = (double*)temp;
+ props.start_temp_d = (double*)temp;
+ props.e = e;
+
+ /* Copy the whole thing into a buffer */
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_spart_d_mapper, temp_d, N, copySize,
+ threadpool_auto_chunk_size, (void*)&props);
+
+ } else if (props.convert_spart_l != NULL) {
+
+ /* Prepare some parameters */
+ long long* temp_l = (long long*)temp;
+ props.start_temp_l = (long long*)temp;
+ props.e = e;
+
+ /* Copy the whole thing into a buffer */
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_spart_l_mapper, temp_l, N, copySize,
+ threadpool_auto_chunk_size, (void*)&props);
+
+ } else if (props.convert_sink_f != NULL) {
+
+ /* Prepare some parameters */
+ float* temp_f = (float*)temp;
+ props.start_temp_f = (float*)temp;
+ props.e = e;
+
+ /* Copy the whole thing into a buffer */
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_sink_f_mapper, temp_f, N, copySize,
+ threadpool_auto_chunk_size, (void*)&props);
+
+ } else if (props.convert_sink_i != NULL) {
+
+ /* Prepare some parameters */
+ int* temp_i = (int*)temp;
+ props.start_temp_i = (int*)temp;
+ props.e = e;
+
+ /* Copy the whole thing into a buffer */
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_sink_i_mapper, temp_i, N, copySize,
+ threadpool_auto_chunk_size, (void*)&props);
+
+ } else if (props.convert_sink_d != NULL) {
+
+ /* Prepare some parameters */
+ double* temp_d = (double*)temp;
+ props.start_temp_d = (double*)temp;
+ props.e = e;
+
+ /* Copy the whole thing into a buffer */
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_sink_d_mapper, temp_d, N, copySize,
+ threadpool_auto_chunk_size, (void*)&props);
+
+ } else if (props.convert_sink_l != NULL) {
+
+ /* Prepare some parameters */
+ long long* temp_l = (long long*)temp;
+ props.start_temp_l = (long long*)temp;
+ props.e = e;
+
+ /* Copy the whole thing into a buffer */
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_sink_l_mapper, temp_l, N, copySize,
+ threadpool_auto_chunk_size, (void*)&props);
+
+ } else if (props.convert_bpart_f != NULL) {
+
+ /* Prepare some parameters */
+ float* temp_f = (float*)temp;
+ props.start_temp_f = (float*)temp;
+ props.e = e;
+
+ /* Copy the whole thing into a buffer */
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_bpart_f_mapper, temp_f, N, copySize,
+ threadpool_auto_chunk_size, (void*)&props);
+
+ } else if (props.convert_bpart_i != NULL) {
+
+ /* Prepare some parameters */
+ int* temp_i = (int*)temp;
+ props.start_temp_i = (int*)temp;
+ props.e = e;
+
+ /* Copy the whole thing into a buffer */
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_bpart_i_mapper, temp_i, N, copySize,
+ threadpool_auto_chunk_size, (void*)&props);
+
+ } else if (props.convert_bpart_d != NULL) {
+
+ /* Prepare some parameters */
+ double* temp_d = (double*)temp;
+ props.start_temp_d = (double*)temp;
+ props.e = e;
+
+ /* Copy the whole thing into a buffer */
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_bpart_d_mapper, temp_d, N, copySize,
+ threadpool_auto_chunk_size, (void*)&props);
+
+ } else if (props.convert_bpart_l != NULL) {
+
+ /* Prepare some parameters */
+ long long* temp_l = (long long*)temp;
+ props.start_temp_l = (long long*)temp;
+ props.e = e;
+
+ /* Copy the whole thing into a buffer */
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_bpart_l_mapper, temp_l, N, copySize,
+ threadpool_auto_chunk_size, (void*)&props);
+
+ } else {
+ error("Missing conversion function");
+ }
+ }
+
+ /* Unit conversion if necessary */
+ const double factor =
+ units_conversion_factor(internal_units, snapshot_units, props.units);
+ if (factor != 1.) {
+
+ /* message("Converting ! factor=%e", factor); */
+
+ if (io_is_double_precision(props.type)) {
+ swift_declare_aligned_ptr(double, temp_d, (double*)temp,
+ IO_BUFFER_ALIGNMENT);
+ for (size_t i = 0; i < num_elements; ++i) temp_d[i] *= factor;
+ } else {
+ swift_declare_aligned_ptr(float, temp_f, (float*)temp,
+ IO_BUFFER_ALIGNMENT);
+ for (size_t i = 0; i < num_elements; ++i) temp_f[i] *= factor;
+ }
+ }
+}
diff --git a/src/common_io_fields.c b/src/common_io_fields.c
new file mode 100644
index 0000000000000000000000000000000000000000..1eb9acfd75268ee83a7d865bd7c55821abc7f2bf
--- /dev/null
+++ b/src/common_io_fields.c
@@ -0,0 +1,389 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2017 Matthieu Schaller (matthieu.schaller@durham.ac.uk).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* This object's header. */
+#include "common_io.h"
+
+/* Local includes. */
+#include "error.h"
+#include "units.h"
+
+/* I/O functions of each sub-module */
+#include "black_holes_io.h"
+#include "chemistry_io.h"
+#include "cooling_io.h"
+#include "fof_io.h"
+#include "gravity_io.h"
+#include "hydro_io.h"
+#include "rt_io.h"
+#include "sink_io.h"
+#include "star_formation_io.h"
+#include "stars_io.h"
+#include "tracers_io.h"
+#include "velociraptor_io.h"
+
+/* Some standard headers. */
+#include <string.h>
+
+/**
+ * @brief Return the particle type code of a select_output parameter
+ *
+ * @param name The name of the parameter under consideration.
+ *
+ * @return The (integer) particle type of the parameter.
+ */
+int io_get_param_ptype(const char* name) {
+
+ const int name_len = strlen(name);
+
+ for (int ptype = 0; ptype < swift_type_count; ptype++) {
+ const int ptype_name_len = strlen(part_type_names[ptype]);
+ if (name_len >= ptype_name_len &&
+ strcmp(&name[name_len - ptype_name_len], part_type_names[ptype]) == 0)
+ return ptype;
+ }
+
+ /* If we get here, we could not match the name, so something's gone wrong. */
+ error("Could not determine the particle type for parameter '%s'.", name);
+
+ /* We can never get here, but the compiler may complain if we don't return
+ * an int after promising to do so... */
+ return -1;
+}
+
+/**
+ * @brief Return the number and names of all output fields of a given ptype.
+ *
+ * @param ptype The index of the particle type under consideration.
+ * @param list An io_props list that will hold the individual fields.
+ * @param with_cosmology Use cosmological name variant?
+ * @param with_fof Include FoF related fields?
+ * @param with_stf Include STF related fields?
+ *
+ * @return The total number of fields that can be written for the ptype.
+ */
+int io_get_ptype_fields(const int ptype, struct io_props* list,
+ const int with_cosmology, const int with_fof,
+ const int with_stf) {
+
+ int num_fields = 0;
+
+ switch (ptype) {
+
+ case swift_type_gas:
+ hydro_write_particles(NULL, NULL, list, &num_fields);
+ num_fields += chemistry_write_particles(NULL, NULL, list + num_fields,
+ with_cosmology);
+ num_fields +=
+ cooling_write_particles(NULL, NULL, list + num_fields, NULL);
+ num_fields += tracers_write_particles(NULL, NULL, list + num_fields,
+ with_cosmology);
+ num_fields +=
+ star_formation_write_particles(NULL, NULL, list + num_fields);
+ if (with_fof)
+ num_fields += fof_write_parts(NULL, NULL, list + num_fields);
+ if (with_stf)
+ num_fields += velociraptor_write_parts(NULL, NULL, list + num_fields);
+ num_fields += rt_write_particles(NULL, list + num_fields);
+ break;
+
+ case swift_type_dark_matter:
+ darkmatter_write_particles(NULL, list, &num_fields);
+ if (with_fof) num_fields += fof_write_gparts(NULL, list + num_fields);
+ if (with_stf)
+ num_fields += velociraptor_write_gparts(NULL, list + num_fields);
+ break;
+
+ case swift_type_dark_matter_background:
+ darkmatter_write_particles(NULL, list, &num_fields);
+ if (with_fof) num_fields += fof_write_gparts(NULL, list + num_fields);
+ if (with_stf)
+ num_fields += velociraptor_write_gparts(NULL, list + num_fields);
+ break;
+
+ case swift_type_stars:
+ stars_write_particles(NULL, list, &num_fields, with_cosmology);
+ num_fields += chemistry_write_sparticles(NULL, list + num_fields);
+ num_fields +=
+ tracers_write_sparticles(NULL, list + num_fields, with_cosmology);
+ num_fields += star_formation_write_sparticles(NULL, list + num_fields);
+ if (with_fof) num_fields += fof_write_sparts(NULL, list + num_fields);
+ if (with_stf)
+ num_fields += velociraptor_write_sparts(NULL, list + num_fields);
+ num_fields += rt_write_stars(NULL, list + num_fields);
+ break;
+
+ case swift_type_sink:
+ sink_write_particles(NULL, list, &num_fields, with_cosmology);
+ break;
+
+ case swift_type_black_hole:
+ black_holes_write_particles(NULL, list, &num_fields, with_cosmology);
+ num_fields += chemistry_write_bparticles(NULL, list + num_fields);
+ if (with_fof) num_fields += fof_write_bparts(NULL, list + num_fields);
+ if (with_stf)
+ num_fields += velociraptor_write_bparts(NULL, list + num_fields);
+ break;
+
+ default:
+ error("Particle Type %d not yet supported. Aborting", ptype);
+ }
+
+ return num_fields;
+}
+
+/**
+ * @brief Prepare the output option fields according to the user's choices and
+ * verify that they are valid.
+ *
+ * @param output_options The #output_options for this run
+ * @param with_cosmology Ran with cosmology?
+ * @param with_fof Are we running with on-the-fly Fof?
+ * @param with_stf Are we running with on-the-fly structure finder?
+ */
+void io_prepare_output_fields(struct output_options* output_options,
+ const int with_cosmology, const int with_fof,
+ const int with_stf) {
+
+ const int MAX_NUM_PTYPE_FIELDS = 100;
+
+ /* Parameter struct for the output options */
+ struct swift_params* params = output_options->select_output;
+
+ /* Get all possible outputs per particle type */
+ int ptype_num_fields_total[swift_type_count] = {0};
+ struct io_props field_list[swift_type_count][MAX_NUM_PTYPE_FIELDS];
+
+ for (int ptype = 0; ptype < swift_type_count; ptype++)
+ ptype_num_fields_total[ptype] = io_get_ptype_fields(
+ ptype, field_list[ptype], with_cosmology, with_fof, with_stf);
+
+ /* Check for whether we have a `Default` section */
+ int have_default = 0;
+
+ /* Loop over each section, i.e. different class of output */
+ for (int section_id = 0; section_id < params->sectionCount; section_id++) {
+
+ /* Get the name of current (selection) section, without a trailing colon */
+ char section_name[FIELD_BUFFER_SIZE];
+ strcpy(section_name, params->section[section_id].name);
+ section_name[strlen(section_name) - 1] = 0;
+
+ /* Is this the `Default` section? */
+ if (strcmp(section_name, select_output_header_default_name) == 0)
+ have_default = 1;
+
+ /* How many fields should each ptype write by default? */
+ int ptype_num_fields_to_write[swift_type_count];
+
+ /* What is the default writing status for each ptype (on/off)? */
+ int ptype_default_write_status[swift_type_count];
+
+ /* Initialise section-specific writing counters for each particle type.
+ * If default is 'write', then we start from the total to deduct any fields
+ * that are switched off. If the default is 'off', we have to start from
+ * zero and then count upwards for each field that is switched back on. */
+ for (int ptype = 0; ptype < swift_type_count; ptype++) {
+
+ /* Internally also verifies that the default level is allowed */
+ const enum lossy_compression_schemes compression_level_current_default =
+ output_options_get_ptype_default_compression(params, section_name,
+ (enum part_type)ptype);
+
+ if (compression_level_current_default == compression_do_not_write) {
+ ptype_default_write_status[ptype] = 0;
+ ptype_num_fields_to_write[ptype] = 0;
+ } else {
+ ptype_default_write_status[ptype] = 1;
+ ptype_num_fields_to_write[ptype] = ptype_num_fields_total[ptype];
+ }
+
+ } /* ends loop over particle types */
+
+ /* Loop over each parameter */
+ for (int param_id = 0; param_id < params->paramCount; param_id++) {
+
+ /* Full name of the parameter to check */
+ const char* param_name = params->data[param_id].name;
+
+ /* Check whether the file still contains the old, now inappropriate
+ * 'SelectOutput' section */
+ if (strstr(param_name, "SelectOutput:") != NULL) {
+ error(
+ "Output selection files no longer require the use of top level "
+ "SelectOutput; see the documentation for changes.");
+ }
+
+ /* Skip if the parameter belongs to another output class or is a
+ * 'Standard' parameter */
+ if (strstr(param_name, section_name) == NULL) continue;
+ if (strstr(param_name, ":Standard_") != NULL) continue;
+
+ /* Get the particle type for current parameter
+ * (raises an error if it could not determine it) */
+ const int param_ptype = io_get_param_ptype(param_name);
+
+ /* Issue a warning if this parameter does not pertain to any of the
+ * known fields from this ptype. */
+ int field_id = 0;
+ char field_name[PARSER_MAX_LINE_SIZE];
+ for (field_id = 0; field_id < ptype_num_fields_total[param_ptype];
+ field_id++) {
+
+ sprintf(field_name, "%s:%.*s_%s", section_name, FIELD_BUFFER_SIZE,
+ field_list[param_ptype][field_id].name,
+ part_type_names[param_ptype]);
+
+ if (strcmp(param_name, field_name) == 0) break;
+ }
+
+ int param_is_known = 0; /* Update below if it is a known one */
+ if (field_id < ptype_num_fields_total[param_ptype])
+ param_is_known = 1;
+ else
+ message(
+ "WARNING: Trying to change behaviour of field '%s' (read from "
+ "'%s') that does not exist. This may be because you are not "
+ "running with all of the physics that you compiled the code with.",
+ param_name, params->fileName);
+
+ /* Perform a correctness check on the _value_ of the parameter */
+ char param_value[FIELD_BUFFER_SIZE];
+ parser_get_param_string(params, param_name, param_value);
+
+ int value_id = 0;
+ for (value_id = 0; value_id < compression_level_count; value_id++)
+ if (strcmp(param_value, lossy_compression_schemes_names[value_id]) == 0)
+ break;
+
+ if (value_id == compression_level_count)
+ error("Choice of output selection parameter %s ('%s') is invalid.",
+ param_name, param_value);
+
+ /* Adjust number of fields to be written for param_ptype, if this field's
+ * status is different from default and it is a known one. */
+ if (param_is_known) {
+ const int is_on =
+ strcmp(param_value,
+ lossy_compression_schemes_names[compression_do_not_write]) !=
+ 0;
+
+ if (is_on && !ptype_default_write_status[param_ptype]) {
+ /* Particle should be written even though default is off:
+ * increase field count */
+ ptype_num_fields_to_write[param_ptype] += 1;
+ }
+ if (!is_on && ptype_default_write_status[param_ptype]) {
+ /* Particle should not be written, even though default is on:
+ * decrease field count */
+ ptype_num_fields_to_write[param_ptype] -= 1;
+ }
+ }
+ } /* ends loop over parameters */
+
+ /* Second loop over ptypes, to write out total number of fields to write */
+ for (int ptype = 0; ptype < swift_type_count; ptype++) {
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Sanity check: is the number of fields to write non-negative? */
+ if (ptype_num_fields_to_write[ptype] < 0)
+ error(
+ "We seem to have subtracted too many fields for particle "
+ "type %d in output class %s (total to write is %d)",
+ ptype, section_name, ptype_num_fields_to_write[ptype]);
+#endif
+ output_options->num_fields_to_write[section_id][ptype] =
+ ptype_num_fields_to_write[ptype];
+ }
+ } /* Ends loop over sections, for different output classes */
+
+ /* Add field numbers for (possible) implicit `Default` output class */
+ if (!have_default) {
+ const int default_id = output_options->select_output->sectionCount;
+ for (int ptype = 0; ptype < swift_type_count; ptype++)
+ output_options->num_fields_to_write[default_id][ptype] =
+ ptype_num_fields_total[ptype];
+ }
+}
+
+/**
+ * @brief Write the output field parameters file
+ *
+ * @param filename The file to write.
+ * @param with_cosmology Use cosmological name variant?
+ */
+void io_write_output_field_parameter(const char* filename, int with_cosmology) {
+
+ FILE* file = fopen(filename, "w");
+ if (file == NULL) error("Error opening file '%s'", filename);
+
+ /* Create a fake unit system for the snapshots */
+ struct unit_system snapshot_units;
+ units_init_cgs(&snapshot_units);
+
+ /* Loop over all particle types */
+ fprintf(file, "Default:\n");
+ for (int ptype = 0; ptype < swift_type_count; ptype++) {
+
+ struct io_props list[100];
+ int num_fields = io_get_ptype_fields(ptype, list, with_cosmology,
+ /*with_fof=*/1, /*with_stf=*/1);
+
+ if (num_fields == 0) continue;
+
+ /* Output a header for that particle type */
+ fprintf(file, " # Particle Type %s\n", part_type_names[ptype]);
+
+ /* Write all the fields of this particle type */
+ for (int i = 0; i < num_fields; ++i) {
+
+ char unit_buffer[FIELD_BUFFER_SIZE] = {0};
+ units_cgs_conversion_string(unit_buffer, &snapshot_units, list[i].units,
+ list[i].scale_factor_exponent);
+
+ /* Need to buffer with a maximal size - otherwise we can't read in again
+ * because comments are too long */
+ char comment_write_buffer[PARSER_MAX_LINE_SIZE / 2];
+
+ sprintf(comment_write_buffer, "%.*s", PARSER_MAX_LINE_SIZE / 2 - 1,
+ list[i].description);
+
+ /* If our string is too long, replace the last few characters (before
+ * \0) with ... for 'fancy printing' */
+ if (strlen(comment_write_buffer) > PARSER_MAX_LINE_SIZE / 2 - 3) {
+ strcpy(&comment_write_buffer[PARSER_MAX_LINE_SIZE / 2 - 4], "...");
+ }
+
+ fprintf(file, " %s_%s: %s # %s : %s\n", list[i].name,
+ part_type_names[ptype], "on", comment_write_buffer, unit_buffer);
+ }
+
+ fprintf(file, "\n");
+ }
+
+ fclose(file);
+
+ printf(
+ "List of valid ouput fields for the particle in snapshots dumped in "
+ "'%s'.\n",
+ filename);
+}
diff --git a/src/engine.c b/src/engine.c
index 21e2e941dcfe22c41f43688baddc95d5a7f87f8e..71258a75d0da60504dfaf4514ba2d1efc418eca2 100644
--- a/src/engine.c
+++ b/src/engine.c
@@ -62,8 +62,6 @@
#include "cosmology.h"
#include "cycle.h"
#include "debug.h"
-#include "distributed_io.h"
-#include "entropy_floor.h"
#include "equation_of_state.h"
#include "error.h"
#include "feedback.h"
@@ -71,10 +69,8 @@
#include "gravity.h"
#include "gravity_cache.h"
#include "hydro.h"
-#include "kick.h"
#include "line_of_sight.h"
#include "logger.h"
-#include "logger_io.h"
#include "map.h"
#include "memuse.h"
#include "minmax.h"
@@ -82,15 +78,11 @@
#include "multipole_struct.h"
#include "output_list.h"
#include "output_options.h"
-#include "parallel_io.h"
-#include "part.h"
#include "partition.h"
#include "profiler.h"
#include "proxy.h"
#include "restart.h"
#include "runner.h"
-#include "serial_io.h"
-#include "single_io.h"
#include "sink_properties.h"
#include "sort_part.h"
#include "star_formation.h"
@@ -101,10 +93,6 @@
#include "tools.h"
#include "units.h"
#include "velociraptor_interface.h"
-#include "version.h"
-
-/* Particle cache size. */
-#define CACHE_SIZE 512
const char *engine_policy_names[] = {"none",
"rand",
@@ -141,10 +129,6 @@ int engine_rank;
/** The current step of the engine as a global variable (for messages). */
int engine_current_step;
-extern int engine_max_parts_per_ghost;
-extern int engine_max_sparts_per_ghost;
-extern int engine_max_parts_per_cooling;
-
/**
* @brief Link a density/force task to a cell.
*
@@ -481,537 +465,6 @@ void engine_exchange_cells(struct engine *e) {
#endif
}
-/**
- * @brief Exchange straying particles with other nodes.
- *
- * @param e The #engine.
- * @param offset_parts The index in the parts array as of which the foreign
- * parts reside (i.e. the current number of local #part).
- * @param ind_part The foreign #cell ID of each part.
- * @param Npart The number of stray parts, contains the number of parts received
- * on return.
- * @param offset_gparts The index in the gparts array as of which the foreign
- * parts reside (i.e. the current number of local #gpart).
- * @param ind_gpart The foreign #cell ID of each gpart.
- * @param Ngpart The number of stray gparts, contains the number of gparts
- * received on return.
- * @param offset_sparts The index in the sparts array as of which the foreign
- * parts reside (i.e. the current number of local #spart).
- * @param ind_spart The foreign #cell ID of each spart.
- * @param Nspart The number of stray sparts, contains the number of sparts
- * received on return.
- * @param offset_bparts The index in the bparts array as of which the foreign
- * parts reside (i.e. the current number of local #bpart).
- * @param ind_bpart The foreign #cell ID of each bpart.
- * @param Nbpart The number of stray bparts, contains the number of bparts
- * received on return.
- *
- * Note that this function does not mess-up the linkage between parts and
- * gparts, i.e. the received particles have correct linkeage.
- */
-void engine_exchange_strays(struct engine *e, const size_t offset_parts,
- const int *restrict ind_part, size_t *Npart,
- const size_t offset_gparts,
- const int *restrict ind_gpart, size_t *Ngpart,
- const size_t offset_sparts,
- const int *restrict ind_spart, size_t *Nspart,
- const size_t offset_bparts,
- const int *restrict ind_bpart, size_t *Nbpart) {
-
-#ifdef WITH_MPI
- struct space *s = e->s;
- ticks tic = getticks();
-
- /* Re-set the proxies. */
- for (int k = 0; k < e->nr_proxies; k++) {
- e->proxies[k].nr_parts_out = 0;
- e->proxies[k].nr_gparts_out = 0;
- e->proxies[k].nr_sparts_out = 0;
- e->proxies[k].nr_bparts_out = 0;
- }
-
- /* Put the parts into the corresponding proxies. */
- for (size_t k = 0; k < *Npart; k++) {
-
- /* Ignore the particles we want to get rid of (inhibited, ...). */
- if (ind_part[k] == -1) continue;
-
- /* Get the target node and proxy ID. */
- const int node_id = e->s->cells_top[ind_part[k]].nodeID;
- if (node_id < 0 || node_id >= e->nr_nodes)
- error("Bad node ID %i.", node_id);
- const int pid = e->proxy_ind[node_id];
- if (pid < 0) {
- error(
- "Do not have a proxy for the requested nodeID %i for part with "
- "id=%lld, x=[%e,%e,%e].",
- node_id, s->parts[offset_parts + k].id,
- s->parts[offset_parts + k].x[0], s->parts[offset_parts + k].x[1],
- s->parts[offset_parts + k].x[2]);
- }
-
- /* Re-link the associated gpart with the buffer offset of the part. */
- if (s->parts[offset_parts + k].gpart != NULL) {
- s->parts[offset_parts + k].gpart->id_or_neg_offset =
- -e->proxies[pid].nr_parts_out;
- }
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (s->parts[offset_parts + k].time_bin == time_bin_inhibited)
- error("Attempting to exchange an inhibited particle");
-#endif
-
- /* Load the part and xpart into the proxy. */
- proxy_parts_load(&e->proxies[pid], &s->parts[offset_parts + k],
- &s->xparts[offset_parts + k], 1);
-
-#ifdef WITH_LOGGER
- if (e->policy & engine_policy_logger) {
- /* Log the particle when leaving a rank. */
- logger_log_part(
- e->logger, &s->parts[offset_parts + k], &s->xparts[offset_parts + k],
- e, /* log_all_fields */ 1,
- logger_pack_flags_and_data(logger_flag_mpi_exit, node_id));
- }
-#endif
- }
-
- /* Put the sparts into the corresponding proxies. */
- for (size_t k = 0; k < *Nspart; k++) {
-
- /* Ignore the particles we want to get rid of (inhibited, ...). */
- if (ind_spart[k] == -1) continue;
-
- /* Get the target node and proxy ID. */
- const int node_id = e->s->cells_top[ind_spart[k]].nodeID;
- if (node_id < 0 || node_id >= e->nr_nodes)
- error("Bad node ID %i.", node_id);
- const int pid = e->proxy_ind[node_id];
- if (pid < 0) {
- error(
- "Do not have a proxy for the requested nodeID %i for part with "
- "id=%lld, x=[%e,%e,%e].",
- node_id, s->sparts[offset_sparts + k].id,
- s->sparts[offset_sparts + k].x[0], s->sparts[offset_sparts + k].x[1],
- s->sparts[offset_sparts + k].x[2]);
- }
-
- /* Re-link the associated gpart with the buffer offset of the spart. */
- if (s->sparts[offset_sparts + k].gpart != NULL) {
- s->sparts[offset_sparts + k].gpart->id_or_neg_offset =
- -e->proxies[pid].nr_sparts_out;
- }
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (s->sparts[offset_sparts + k].time_bin == time_bin_inhibited)
- error("Attempting to exchange an inhibited particle");
-#endif
-
- /* Load the spart into the proxy */
- proxy_sparts_load(&e->proxies[pid], &s->sparts[offset_sparts + k], 1);
-
-#ifdef WITH_LOGGER
- if (e->policy & engine_policy_logger) {
- /* Log the particle when leaving a rank. */
- logger_log_spart(
- e->logger, &s->sparts[offset_sparts + k], e,
- /* log_all_fields */ 1,
- logger_pack_flags_and_data(logger_flag_mpi_exit, node_id));
- }
-#endif
- }
-
- /* Put the bparts into the corresponding proxies. */
- for (size_t k = 0; k < *Nbpart; k++) {
-
- /* Ignore the particles we want to get rid of (inhibited, ...). */
- if (ind_bpart[k] == -1) continue;
-
- /* Get the target node and proxy ID. */
- const int node_id = e->s->cells_top[ind_bpart[k]].nodeID;
- if (node_id < 0 || node_id >= e->nr_nodes)
- error("Bad node ID %i.", node_id);
- const int pid = e->proxy_ind[node_id];
- if (pid < 0) {
- error(
- "Do not have a proxy for the requested nodeID %i for part with "
- "id=%lld, x=[%e,%e,%e].",
- node_id, s->bparts[offset_bparts + k].id,
- s->bparts[offset_bparts + k].x[0], s->bparts[offset_bparts + k].x[1],
- s->bparts[offset_bparts + k].x[2]);
- }
-
- /* Re-link the associated gpart with the buffer offset of the bpart. */
- if (s->bparts[offset_bparts + k].gpart != NULL) {
- s->bparts[offset_bparts + k].gpart->id_or_neg_offset =
- -e->proxies[pid].nr_bparts_out;
- }
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (s->bparts[offset_bparts + k].time_bin == time_bin_inhibited)
- error("Attempting to exchange an inhibited particle");
-#endif
-
- /* Load the bpart into the proxy */
- proxy_bparts_load(&e->proxies[pid], &s->bparts[offset_bparts + k], 1);
-
-#ifdef WITH_LOGGER
- if (e->policy & engine_policy_logger) {
- error("Not yet implemented.");
- }
-#endif
- }
-
- /* Put the gparts into the corresponding proxies. */
- for (size_t k = 0; k < *Ngpart; k++) {
-
- /* Ignore the particles we want to get rid of (inhibited, ...). */
- if (ind_gpart[k] == -1) continue;
-
- /* Get the target node and proxy ID. */
- const int node_id = e->s->cells_top[ind_gpart[k]].nodeID;
- if (node_id < 0 || node_id >= e->nr_nodes)
- error("Bad node ID %i.", node_id);
- const int pid = e->proxy_ind[node_id];
- if (pid < 0) {
- error(
- "Do not have a proxy for the requested nodeID %i for part with "
- "id=%lli, x=[%e,%e,%e].",
- node_id, s->gparts[offset_gparts + k].id_or_neg_offset,
- s->gparts[offset_gparts + k].x[0], s->gparts[offset_gparts + k].x[1],
- s->gparts[offset_gparts + k].x[2]);
- }
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (s->gparts[offset_gparts + k].time_bin == time_bin_inhibited)
- error("Attempting to exchange an inhibited particle");
-#endif
-
- /* Load the gpart into the proxy */
- proxy_gparts_load(&e->proxies[pid], &s->gparts[offset_gparts + k], 1);
-
-#ifdef WITH_LOGGER
- /* Write only the dark matter particles */
- if ((e->policy & engine_policy_logger) &&
- s->gparts[offset_gparts + k].type == swift_type_dark_matter) {
-
- /* Log the particle when leaving a rank. */
- logger_log_gpart(
- e->logger, &s->gparts[offset_gparts + k], e,
- /* log_all_fields */ 1,
- logger_pack_flags_and_data(logger_flag_mpi_exit, node_id));
- }
-#endif
- }
-
- /* Launch the proxies. */
- MPI_Request reqs_in[5 * engine_maxproxies];
- MPI_Request reqs_out[5 * engine_maxproxies];
- for (int k = 0; k < e->nr_proxies; k++) {
- proxy_parts_exchange_first(&e->proxies[k]);
- reqs_in[k] = e->proxies[k].req_parts_count_in;
- reqs_out[k] = e->proxies[k].req_parts_count_out;
- }
-
- /* Wait for each count to come in and start the recv. */
- for (int k = 0; k < e->nr_proxies; k++) {
- int pid = MPI_UNDEFINED;
- if (MPI_Waitany(e->nr_proxies, reqs_in, &pid, MPI_STATUS_IGNORE) !=
- MPI_SUCCESS ||
- pid == MPI_UNDEFINED)
- error("MPI_Waitany failed.");
- // message( "request from proxy %i has arrived." , pid );
- proxy_parts_exchange_second(&e->proxies[pid]);
- }
-
- /* Wait for all the sends to have finished too. */
- if (MPI_Waitall(e->nr_proxies, reqs_out, MPI_STATUSES_IGNORE) != MPI_SUCCESS)
- error("MPI_Waitall on sends failed.");
-
- /* Count the total number of incoming particles and make sure we have
- enough space to accommodate them. */
- int count_parts_in = 0;
- int count_gparts_in = 0;
- int count_sparts_in = 0;
- int count_bparts_in = 0;
- for (int k = 0; k < e->nr_proxies; k++) {
- count_parts_in += e->proxies[k].nr_parts_in;
- count_gparts_in += e->proxies[k].nr_gparts_in;
- count_sparts_in += e->proxies[k].nr_sparts_in;
- count_bparts_in += e->proxies[k].nr_bparts_in;
- }
- if (e->verbose) {
- message(
- "sent out %zu/%zu/%zu/%zu parts/gparts/sparts/bparts, got %i/%i/%i/%i "
- "back.",
- *Npart, *Ngpart, *Nspart, *Nbpart, count_parts_in, count_gparts_in,
- count_sparts_in, count_bparts_in);
- }
-
- /* Reallocate the particle arrays if necessary */
- if (offset_parts + count_parts_in > s->size_parts) {
- s->size_parts = (offset_parts + count_parts_in) * engine_parts_size_grow;
- struct part *parts_new = NULL;
- struct xpart *xparts_new = NULL;
- if (swift_memalign("parts", (void **)&parts_new, part_align,
- sizeof(struct part) * s->size_parts) != 0 ||
- swift_memalign("xparts", (void **)&xparts_new, xpart_align,
- sizeof(struct xpart) * s->size_parts) != 0)
- error("Failed to allocate new part data.");
- memcpy(parts_new, s->parts, sizeof(struct part) * offset_parts);
- memcpy(xparts_new, s->xparts, sizeof(struct xpart) * offset_parts);
- swift_free("parts", s->parts);
- swift_free("xparts", s->xparts);
- s->parts = parts_new;
- s->xparts = xparts_new;
-
- /* Reset the links */
- for (size_t k = 0; k < offset_parts; k++) {
- if (s->parts[k].gpart != NULL) {
- s->parts[k].gpart->id_or_neg_offset = -k;
- }
- }
- }
-
- if (offset_sparts + count_sparts_in > s->size_sparts) {
- s->size_sparts = (offset_sparts + count_sparts_in) * engine_parts_size_grow;
- struct spart *sparts_new = NULL;
- if (swift_memalign("sparts", (void **)&sparts_new, spart_align,
- sizeof(struct spart) * s->size_sparts) != 0)
- error("Failed to allocate new spart data.");
- memcpy(sparts_new, s->sparts, sizeof(struct spart) * offset_sparts);
- swift_free("sparts", s->sparts);
- s->sparts = sparts_new;
-
- /* Reset the links */
- for (size_t k = 0; k < offset_sparts; k++) {
- if (s->sparts[k].gpart != NULL) {
- s->sparts[k].gpart->id_or_neg_offset = -k;
- }
- }
- }
-
- if (offset_bparts + count_bparts_in > s->size_bparts) {
- s->size_bparts = (offset_bparts + count_bparts_in) * engine_parts_size_grow;
- struct bpart *bparts_new = NULL;
- if (swift_memalign("bparts", (void **)&bparts_new, bpart_align,
- sizeof(struct bpart) * s->size_bparts) != 0)
- error("Failed to allocate new bpart data.");
- memcpy(bparts_new, s->bparts, sizeof(struct bpart) * offset_bparts);
- swift_free("bparts", s->bparts);
- s->bparts = bparts_new;
-
- /* Reset the links */
- for (size_t k = 0; k < offset_bparts; k++) {
- if (s->bparts[k].gpart != NULL) {
- s->bparts[k].gpart->id_or_neg_offset = -k;
- }
- }
- }
-
- if (offset_gparts + count_gparts_in > s->size_gparts) {
- s->size_gparts = (offset_gparts + count_gparts_in) * engine_parts_size_grow;
- struct gpart *gparts_new = NULL;
- if (swift_memalign("gparts", (void **)&gparts_new, gpart_align,
- sizeof(struct gpart) * s->size_gparts) != 0)
- error("Failed to allocate new gpart data.");
- memcpy(gparts_new, s->gparts, sizeof(struct gpart) * offset_gparts);
- swift_free("gparts", s->gparts);
- s->gparts = gparts_new;
-
- /* Reset the links */
- for (size_t k = 0; k < offset_gparts; k++) {
- if (s->gparts[k].type == swift_type_gas) {
- s->parts[-s->gparts[k].id_or_neg_offset].gpart = &s->gparts[k];
- } else if (s->gparts[k].type == swift_type_stars) {
- s->sparts[-s->gparts[k].id_or_neg_offset].gpart = &s->gparts[k];
- } else if (s->gparts[k].type == swift_type_black_hole) {
- s->bparts[-s->gparts[k].id_or_neg_offset].gpart = &s->gparts[k];
- }
- }
- }
-
- /* Collect the requests for the particle data from the proxies. */
- int nr_in = 0, nr_out = 0;
- for (int k = 0; k < e->nr_proxies; k++) {
- if (e->proxies[k].nr_parts_in > 0) {
- reqs_in[5 * k] = e->proxies[k].req_parts_in;
- reqs_in[5 * k + 1] = e->proxies[k].req_xparts_in;
- nr_in += 2;
- } else {
- reqs_in[5 * k] = reqs_in[5 * k + 1] = MPI_REQUEST_NULL;
- }
- if (e->proxies[k].nr_gparts_in > 0) {
- reqs_in[5 * k + 2] = e->proxies[k].req_gparts_in;
- nr_in += 1;
- } else {
- reqs_in[5 * k + 2] = MPI_REQUEST_NULL;
- }
- if (e->proxies[k].nr_sparts_in > 0) {
- reqs_in[5 * k + 3] = e->proxies[k].req_sparts_in;
- nr_in += 1;
- } else {
- reqs_in[5 * k + 3] = MPI_REQUEST_NULL;
- }
- if (e->proxies[k].nr_bparts_in > 0) {
- reqs_in[5 * k + 4] = e->proxies[k].req_bparts_in;
- nr_in += 1;
- } else {
- reqs_in[5 * k + 4] = MPI_REQUEST_NULL;
- }
-
- if (e->proxies[k].nr_parts_out > 0) {
- reqs_out[5 * k] = e->proxies[k].req_parts_out;
- reqs_out[5 * k + 1] = e->proxies[k].req_xparts_out;
- nr_out += 2;
- } else {
- reqs_out[5 * k] = reqs_out[5 * k + 1] = MPI_REQUEST_NULL;
- }
- if (e->proxies[k].nr_gparts_out > 0) {
- reqs_out[5 * k + 2] = e->proxies[k].req_gparts_out;
- nr_out += 1;
- } else {
- reqs_out[5 * k + 2] = MPI_REQUEST_NULL;
- }
- if (e->proxies[k].nr_sparts_out > 0) {
- reqs_out[5 * k + 3] = e->proxies[k].req_sparts_out;
- nr_out += 1;
- } else {
- reqs_out[5 * k + 3] = MPI_REQUEST_NULL;
- }
- if (e->proxies[k].nr_bparts_out > 0) {
- reqs_out[5 * k + 4] = e->proxies[k].req_bparts_out;
- nr_out += 1;
- } else {
- reqs_out[5 * k + 4] = MPI_REQUEST_NULL;
- }
- }
-
- /* Wait for each part array to come in and collect the new
- parts from the proxies. */
- int count_parts = 0, count_gparts = 0, count_sparts = 0, count_bparts = 0;
- for (int k = 0; k < nr_in; k++) {
- int err, pid;
- if ((err = MPI_Waitany(5 * e->nr_proxies, reqs_in, &pid,
- MPI_STATUS_IGNORE)) != MPI_SUCCESS) {
- char buff[MPI_MAX_ERROR_STRING];
- int res;
- MPI_Error_string(err, buff, &res);
- error("MPI_Waitany failed (%s).", buff);
- }
- if (pid == MPI_UNDEFINED) break;
- // message( "request from proxy %i has arrived." , pid / 5 );
- pid = 5 * (pid / 5);
-
- /* If all the requests for a given proxy have arrived... */
- if (reqs_in[pid + 0] == MPI_REQUEST_NULL &&
- reqs_in[pid + 1] == MPI_REQUEST_NULL &&
- reqs_in[pid + 2] == MPI_REQUEST_NULL &&
- reqs_in[pid + 3] == MPI_REQUEST_NULL &&
- reqs_in[pid + 4] == MPI_REQUEST_NULL) {
- /* Copy the particle data to the part/xpart/gpart arrays. */
- struct proxy *prox = &e->proxies[pid / 5];
- memcpy(&s->parts[offset_parts + count_parts], prox->parts_in,
- sizeof(struct part) * prox->nr_parts_in);
- memcpy(&s->xparts[offset_parts + count_parts], prox->xparts_in,
- sizeof(struct xpart) * prox->nr_parts_in);
- memcpy(&s->gparts[offset_gparts + count_gparts], prox->gparts_in,
- sizeof(struct gpart) * prox->nr_gparts_in);
- memcpy(&s->sparts[offset_sparts + count_sparts], prox->sparts_in,
- sizeof(struct spart) * prox->nr_sparts_in);
- memcpy(&s->bparts[offset_bparts + count_bparts], prox->bparts_in,
- sizeof(struct bpart) * prox->nr_bparts_in);
-
-#ifdef WITH_LOGGER
- if (e->policy & engine_policy_logger) {
- const uint32_t flag =
- logger_pack_flags_and_data(logger_flag_mpi_enter, prox->nodeID);
-
- struct part *parts = &s->parts[offset_parts + count_parts];
- struct xpart *xparts = &s->xparts[offset_parts + count_parts];
- struct spart *sparts = &s->sparts[offset_sparts + count_sparts];
- struct gpart *gparts = &s->gparts[offset_gparts + count_gparts];
-
- /* Log the gas particles */
- logger_log_parts(e->logger, parts, xparts, prox->nr_parts_in, e,
- /* log_all_fields */ 1, flag);
-
- /* Log the stellar particles */
- logger_log_sparts(e->logger, sparts, prox->nr_sparts_in, e,
- /* log_all_fields */ 1, flag);
-
- /* Log the gparts */
- logger_log_gparts(e->logger, gparts, prox->nr_gparts_in, e,
- /* log_all_fields */ 1, flag);
-
- /* Log the bparts */
- if (prox->nr_bparts_in > 0) {
- error("TODO");
- }
- }
-#endif
- /* for (int k = offset; k < offset + count; k++)
- message(
- "received particle %lli, x=[%.3e %.3e %.3e], h=%.3e, from node %i.",
- s->parts[k].id, s->parts[k].x[0], s->parts[k].x[1],
- s->parts[k].x[2], s->parts[k].h, p->nodeID); */
-
- /* Re-link the gparts. */
- for (int kk = 0; kk < prox->nr_gparts_in; kk++) {
- struct gpart *gp = &s->gparts[offset_gparts + count_gparts + kk];
-
- if (gp->type == swift_type_gas) {
- struct part *p =
- &s->parts[offset_parts + count_parts - gp->id_or_neg_offset];
- gp->id_or_neg_offset = s->parts - p;
- p->gpart = gp;
- } else if (gp->type == swift_type_stars) {
- struct spart *sp =
- &s->sparts[offset_sparts + count_sparts - gp->id_or_neg_offset];
- gp->id_or_neg_offset = s->sparts - sp;
- sp->gpart = gp;
- } else if (gp->type == swift_type_black_hole) {
- struct bpart *bp =
- &s->bparts[offset_bparts + count_bparts - gp->id_or_neg_offset];
- gp->id_or_neg_offset = s->bparts - bp;
- bp->gpart = gp;
- }
- }
-
- /* Advance the counters. */
- count_parts += prox->nr_parts_in;
- count_gparts += prox->nr_gparts_in;
- count_sparts += prox->nr_sparts_in;
- count_bparts += prox->nr_bparts_in;
- }
- }
-
- /* Wait for all the sends to have finished too. */
- if (nr_out > 0)
- if (MPI_Waitall(5 * e->nr_proxies, reqs_out, MPI_STATUSES_IGNORE) !=
- MPI_SUCCESS)
- error("MPI_Waitall on sends failed.");
-
- /* Free the proxy memory */
- for (int k = 0; k < e->nr_proxies; k++) {
- proxy_free_particle_buffers(&e->proxies[k]);
- }
-
- if (e->verbose)
- message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
- clocks_getunit());
-
- /* Return the number of harvested parts. */
- *Npart = count_parts;
- *Ngpart = count_gparts;
- *Nspart = count_sparts;
- *Nbpart = count_bparts;
-
-#else
- error("SWIFT was not compiled with MPI support.");
-#endif
-}
-
/**
* @brief Exchanges the top-level multipoles between all the nodes
* such that every node has a multipole for each top-level cell.
@@ -2859,681 +2312,101 @@ void engine_step(struct engine *e) {
}
/**
- * @brief Check whether any kind of i/o has to be performed during this
- * step.
- *
- * This includes snapshots, stats and halo finder. We also handle the case
- * of multiple outputs between two steps.
- *
- * @param e The #engine.
+ * @brief Returns 1 if the simulation has reached its end point, 0 otherwise
*/
-void engine_check_for_dumps(struct engine *e) {
- const int with_cosmology = (e->policy & engine_policy_cosmology);
- const int with_stf = (e->policy & engine_policy_structure_finding);
- const int with_los = (e->policy & engine_policy_line_of_sight);
- const int with_fof = (e->policy & engine_policy_fof);
-
- /* What kind of output are we getting? */
- enum output_type {
- output_none,
- output_snapshot,
- output_statistics,
- output_stf,
- output_los,
- };
-
- /* What kind of output do we want? And at which time ?
- * Find the earliest output (amongst all kinds) that takes place
- * before the next time-step */
- enum output_type type = output_none;
- integertime_t ti_output = max_nr_timesteps;
- e->stf_this_timestep = 0;
+int engine_is_done(struct engine *e) {
+ return !(e->ti_current < max_nr_timesteps);
+}
- /* Save some statistics ? */
- if (e->ti_end_min > e->ti_next_stats && e->ti_next_stats > 0) {
- if (e->ti_next_stats < ti_output) {
- ti_output = e->ti_next_stats;
- type = output_statistics;
- }
- }
+void engine_do_reconstruct_multipoles_mapper(void *map_data, int num_elements,
+ void *extra_data) {
- /* Do we want a snapshot? */
- if (e->ti_end_min > e->ti_next_snapshot && e->ti_next_snapshot > 0) {
- if (e->ti_next_snapshot < ti_output) {
- ti_output = e->ti_next_snapshot;
- type = output_snapshot;
- }
- }
+ struct engine *e = (struct engine *)extra_data;
+ struct cell *cells = (struct cell *)map_data;
- /* Do we want to perform structure finding? */
- if (with_stf) {
- if (e->ti_end_min > e->ti_next_stf && e->ti_next_stf > 0) {
- if (e->ti_next_stf < ti_output) {
- ti_output = e->ti_next_stf;
- type = output_stf;
- }
- }
- }
+ for (int ind = 0; ind < num_elements; ind++) {
+ struct cell *c = &cells[ind];
+ if (c != NULL && c->nodeID == e->nodeID) {
- /* Do we want to write a line of sight file? */
- if (with_los) {
- if (e->ti_end_min > e->ti_next_los && e->ti_next_los > 0) {
- if (e->ti_next_los < ti_output) {
- ti_output = e->ti_next_los;
- type = output_los;
- }
+ /* Construct the multipoles in this cell hierarchy */
+ cell_make_multipoles(c, e->ti_current, e->gravity_properties);
}
}
+}
- /* Store information before attempting extra dump-related drifts */
- const integertime_t ti_current = e->ti_current;
- const timebin_t max_active_bin = e->max_active_bin;
- const double time = e->time;
-
- while (type != output_none) {
-
- /* Let's fake that we are at the dump time */
- e->ti_current = ti_output;
- e->max_active_bin = 0;
- if (with_cosmology) {
- cosmology_update(e->cosmology, e->physical_constants, e->ti_current);
- e->time = e->cosmology->time;
- } else {
- e->time = ti_output * e->time_base + e->time_begin;
- }
-
- /* Drift everyone */
- engine_drift_all(e, /*drift_mpole=*/0);
-
- /* Write some form of output */
- switch (type) {
-
- case output_snapshot:
-
-#ifdef SWIFT_GRAVITY_FORCE_CHECKS
- /* Indicate we are allowed to do a brute force calculation now */
- e->force_checks_snapshot_flag = 1;
-#endif
-
- /* Do we want FoF group IDs in the snapshot? */
- if(with_fof && e->snapshot_invoke_fof) {
- engine_fof(e, /*dump_results=*/0, /*seed_black_holes=*/0);
- }
+/**
+ * @brief Reconstruct all the multipoles at all the levels in the tree.
+ *
+ * @param e The #engine.
+ */
+void engine_reconstruct_multipoles(struct engine *e) {
- /* Do we want a corresponding VELOCIraptor output? */
- if (with_stf && e->snapshot_invoke_stf && !e->stf_this_timestep) {
+ const ticks tic = getticks();
-#ifdef HAVE_VELOCIRAPTOR
- velociraptor_invoke(e, /*linked_with_snap=*/1);
- e->step_props |= engine_step_prop_stf;
-#else
- error(
- "Asking for a VELOCIraptor output but SWIFT was compiled without "
- "the interface!");
+#ifdef SWIFT_DEBUG_CHECKS
+ if (e->nodeID == 0) {
+ if (e->policy & engine_policy_cosmology)
+ message("Reconstructing multipoles at a=%e",
+ exp(e->ti_current * e->time_base) * e->cosmology->a_begin);
+ else
+ message("Reconstructing multipoles at t=%e",
+ e->ti_current * e->time_base + e->time_begin);
+ }
#endif
- }
-
- /* Dump... */
- engine_dump_snapshot(e);
- /* Free the memory allocated for VELOCIraptor i/o. */
- if (with_stf && e->snapshot_invoke_stf && e->s->gpart_group_data) {
-#ifdef HAVE_VELOCIRAPTOR
- swift_free("gpart_group_data", e->s->gpart_group_data);
- e->s->gpart_group_data = NULL;
-#endif
- }
+ threadpool_map(&e->threadpool, engine_do_reconstruct_multipoles_mapper,
+ e->s->cells_top, e->s->nr_cells, sizeof(struct cell),
+ threadpool_auto_chunk_size, e);
- /* ... and find the next output time */
- engine_compute_next_snapshot_time(e);
- break;
+ if (e->verbose)
+ message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+ clocks_getunit());
+}
- case output_statistics:
+/**
+ * @brief Split the underlying space into regions and assign to separate nodes.
+ *
+ * @param e The #engine.
+ * @param initial_partition structure defining the cell partition technique
+ */
+void engine_split(struct engine *e, struct partition *initial_partition) {
- /* Dump */
- engine_print_stats(e);
+#ifdef WITH_MPI
+ const ticks tic = getticks();
- /* and move on */
- engine_compute_next_statistics_time(e);
+ struct space *s = e->s;
- break;
+ /* Do the initial partition of the cells. */
+ partition_initial_partition(initial_partition, e->nodeID, e->nr_nodes, s);
- case output_stf:
+ /* Make the proxies. */
+ engine_makeproxies(e);
-#ifdef HAVE_VELOCIRAPTOR
- /* Unleash the raptor! */
- if (!e->stf_this_timestep) {
- velociraptor_invoke(e, /*linked_with_snap=*/0);
- e->step_props |= engine_step_prop_stf;
- }
+ /* Re-allocate the local parts. */
+ if (e->verbose)
+ message("Re-allocating parts array from %zu to %zu.", s->size_parts,
+ (size_t)(s->nr_parts * engine_redistribute_alloc_margin));
+ s->size_parts = s->nr_parts * engine_redistribute_alloc_margin;
+ struct part *parts_new = NULL;
+ struct xpart *xparts_new = NULL;
+ if (swift_memalign("parts", (void **)&parts_new, part_align,
+ sizeof(struct part) * s->size_parts) != 0 ||
+ swift_memalign("xparts", (void **)&xparts_new, xpart_align,
+ sizeof(struct xpart) * s->size_parts) != 0)
+ error("Failed to allocate new part data.");
- /* ... and find the next output time */
- engine_compute_next_stf_time(e);
-#else
- error(
- "Asking for a VELOCIraptor output but SWIFT was compiled without "
- "the interface!");
-#endif
- break;
+ if (s->nr_parts > 0) {
+ memcpy(parts_new, s->parts, sizeof(struct part) * s->nr_parts);
+ memcpy(xparts_new, s->xparts, sizeof(struct xpart) * s->nr_parts);
+ }
+ swift_free("parts", s->parts);
+ swift_free("xparts", s->xparts);
+ s->parts = parts_new;
+ s->xparts = xparts_new;
- case output_los:
-
- /* Compute the LoS */
- do_line_of_sight(e);
-
- /* Move on */
- engine_compute_next_los_time(e);
-
- break;
-
- default:
- error("Invalid dump type");
- }
-
- /* We need to see whether whether we are in the pathological case
- * where there can be another dump before the next step. */
-
- type = output_none;
- ti_output = max_nr_timesteps;
-
- /* Save some statistics ? */
- if (e->ti_end_min > e->ti_next_stats && e->ti_next_stats > 0) {
- if (e->ti_next_stats < ti_output) {
- ti_output = e->ti_next_stats;
- type = output_statistics;
- }
- }
-
- /* Do we want a snapshot? */
- if (e->ti_end_min > e->ti_next_snapshot && e->ti_next_snapshot > 0) {
- if (e->ti_next_snapshot < ti_output) {
- ti_output = e->ti_next_snapshot;
- type = output_snapshot;
- }
- }
-
- /* Do we want to perform structure finding? */
- if (with_stf) {
- if (e->ti_end_min > e->ti_next_stf && e->ti_next_stf > 0) {
- if (e->ti_next_stf < ti_output) {
- ti_output = e->ti_next_stf;
- type = output_stf;
- }
- }
- }
-
- /* Do line of sight ? */
- if (with_los) {
- if (e->ti_end_min > e->ti_next_los && e->ti_next_los > 0) {
- if (e->ti_next_los < ti_output) {
- ti_output = e->ti_next_los;
- type = output_los;
- }
- }
- }
-
- } /* While loop over output types */
-
- /* Restore the information we stored */
- e->ti_current = ti_current;
- if (e->policy & engine_policy_cosmology)
- cosmology_update(e->cosmology, e->physical_constants, e->ti_current);
- e->max_active_bin = max_active_bin;
- e->time = time;
-}
-
-/**
- * @brief Check whether an index file has to be written during this
- * step.
- *
- * @param e The #engine.
- */
-void engine_check_for_index_dump(struct engine *e) {
-#ifdef WITH_LOGGER
- /* Get a few variables */
- struct logger_writer *log = e->logger;
- const size_t dump_size = log->dump.count;
- const size_t old_dump_size = log->index.dump_size_last_output;
- const float mem_frac = log->index.mem_frac;
- const size_t total_nr_parts =
- (e->total_nr_parts + e->total_nr_gparts + e->total_nr_sparts +
- e->total_nr_bparts + e->total_nr_DM_background_gparts);
- const size_t index_file_size =
- total_nr_parts * sizeof(struct logger_part_data);
-
- /* Check if we should write a file */
- if (mem_frac * (dump_size - old_dump_size) > index_file_size) {
- /* Write an index file */
- engine_dump_index(e);
-
- /* Update the dump size for last output */
- log->index.dump_size_last_output = dump_size;
- }
-#else
- error("This function should not be called without the logger.");
-#endif
-}
-
-/**
- * @brief dump restart files if it is time to do so and dumps are enabled.
- *
- * @param e the engine.
- * @param drifted_all true if a drift_all has just been performed.
- * @param force force a dump, if dumping is enabled.
- */
-void engine_dump_restarts(struct engine *e, int drifted_all, int force) {
-
- if (e->restart_dump) {
- ticks tic = getticks();
-
- /* Dump when the time has arrived, or we are told to. */
- int dump = ((tic > e->restart_next) || force);
-
-#ifdef WITH_MPI
- /* Synchronize this action from rank 0 (ticks may differ between
- * machines). */
- MPI_Bcast(&dump, 1, MPI_INT, 0, MPI_COMM_WORLD);
-#endif
- if (dump) {
-
- if (e->nodeID == 0) message("Writing restart files");
-
- /* Clean out the previous saved files, if found. Do this now as we are
- * MPI synchronized. */
- restart_remove_previous(e->restart_file);
-
- /* Drift all particles first (may have just been done). */
- if (!drifted_all) engine_drift_all(e, /*drift_mpole=*/1);
- restart_write(e, e->restart_file);
-
-#ifdef WITH_MPI
- /* Make sure all ranks finished writing to avoid having incomplete
- * sets of restart files should the code crash before all the ranks
- * are done */
- MPI_Barrier(MPI_COMM_WORLD);
-#endif
-
- if (e->verbose)
- message("Dumping restart files took %.3f %s",
- clocks_from_ticks(getticks() - tic), clocks_getunit());
-
- /* Time after which next dump will occur. */
- e->restart_next += e->restart_dt;
-
- /* Flag that we dumped the restarts */
- e->step_props |= engine_step_prop_restarts;
- }
- }
-}
-
-/**
- * @brief Returns 1 if the simulation has reached its end point, 0 otherwise
- */
-int engine_is_done(struct engine *e) {
- return !(e->ti_current < max_nr_timesteps);
-}
-
-void engine_do_reconstruct_multipoles_mapper(void *map_data, int num_elements,
- void *extra_data) {
-
- struct engine *e = (struct engine *)extra_data;
- struct cell *cells = (struct cell *)map_data;
-
- for (int ind = 0; ind < num_elements; ind++) {
- struct cell *c = &cells[ind];
- if (c != NULL && c->nodeID == e->nodeID) {
-
- /* Construct the multipoles in this cell hierarchy */
- cell_make_multipoles(c, e->ti_current, e->gravity_properties);
- }
- }
-}
-
-/**
- * @brief Reconstruct all the multipoles at all the levels in the tree.
- *
- * @param e The #engine.
- */
-void engine_reconstruct_multipoles(struct engine *e) {
-
- const ticks tic = getticks();
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (e->nodeID == 0) {
- if (e->policy & engine_policy_cosmology)
- message("Reconstructing multipoles at a=%e",
- exp(e->ti_current * e->time_base) * e->cosmology->a_begin);
- else
- message("Reconstructing multipoles at t=%e",
- e->ti_current * e->time_base + e->time_begin);
- }
-#endif
-
- threadpool_map(&e->threadpool, engine_do_reconstruct_multipoles_mapper,
- e->s->cells_top, e->s->nr_cells, sizeof(struct cell),
- threadpool_auto_chunk_size, e);
-
- if (e->verbose)
- message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
- clocks_getunit());
-}
-
-/**
- * @brief Create and fill the proxies.
- *
- * @param e The #engine.
- */
-void engine_makeproxies(struct engine *e) {
-
-#ifdef WITH_MPI
- /* Let's time this */
- const ticks tic = getticks();
-
- /* Useful local information */
- const int nodeID = e->nodeID;
- const struct space *s = e->s;
-
- /* Handle on the cells and proxies */
- struct cell *cells = s->cells_top;
- struct proxy *proxies = e->proxies;
-
- /* Some info about the domain */
- const int cdim[3] = {s->cdim[0], s->cdim[1], s->cdim[2]};
- const double dim[3] = {s->dim[0], s->dim[1], s->dim[2]};
- const int periodic = s->periodic;
- const double cell_width[3] = {cells[0].width[0], cells[0].width[1],
- cells[0].width[2]};
-
- /* Get some info about the physics */
- const int with_hydro = (e->policy & engine_policy_hydro);
- const int with_gravity = (e->policy & engine_policy_self_gravity);
- const double theta_crit = e->gravity_properties->theta_crit;
- const double theta_crit_inv = 1. / e->gravity_properties->theta_crit;
- const double max_mesh_dist = e->mesh->r_cut_max;
- const double max_mesh_dist2 = max_mesh_dist * max_mesh_dist;
-
- /* Distance between centre of the cell and corners */
- const double r_diag2 = cell_width[0] * cell_width[0] +
- cell_width[1] * cell_width[1] +
- cell_width[2] * cell_width[2];
- const double r_diag = 0.5 * sqrt(r_diag2);
-
- /* Maximal distance from shifted CoM to any corner */
- const double r_max = 2 * r_diag;
-
- /* Prepare the proxies and the proxy index. */
- if (e->proxy_ind == NULL)
- if ((e->proxy_ind = (int *)malloc(sizeof(int) * e->nr_nodes)) == NULL)
- error("Failed to allocate proxy index.");
- for (int k = 0; k < e->nr_nodes; k++) e->proxy_ind[k] = -1;
- e->nr_proxies = 0;
-
- /* Compute how many cells away we need to walk */
- int delta_cells = 1; /*hydro case */
-
- /* Gravity needs to take the opening angle into account */
- if (with_gravity) {
- const double distance = 2. * r_max * theta_crit_inv;
- delta_cells = (int)(distance / cells[0].dmin) + 1;
- }
-
- /* Turn this into upper and lower bounds for loops */
- int delta_m = delta_cells;
- int delta_p = delta_cells;
-
- /* Special case where every cell is in range of every other one */
- if (delta_cells >= cdim[0] / 2) {
- if (cdim[0] % 2 == 0) {
- delta_m = cdim[0] / 2;
- delta_p = cdim[0] / 2 - 1;
- } else {
- delta_m = cdim[0] / 2;
- delta_p = cdim[0] / 2;
- }
- }
-
- /* Let's be verbose about this choice */
- if (e->verbose)
- message(
- "Looking for proxies up to %d top-level cells away (delta_m=%d "
- "delta_p=%d)",
- delta_cells, delta_m, delta_p);
-
- /* Loop over each cell in the space. */
- for (int i = 0; i < cdim[0]; i++) {
- for (int j = 0; j < cdim[1]; j++) {
- for (int k = 0; k < cdim[2]; k++) {
-
- /* Get the cell ID. */
- const int cid = cell_getid(cdim, i, j, k);
-
- /* Loop over all its neighbours neighbours in range. */
- for (int ii = -delta_m; ii <= delta_p; ii++) {
- int iii = i + ii;
- if (!periodic && (iii < 0 || iii >= cdim[0])) continue;
- iii = (iii + cdim[0]) % cdim[0];
- for (int jj = -delta_m; jj <= delta_p; jj++) {
- int jjj = j + jj;
- if (!periodic && (jjj < 0 || jjj >= cdim[1])) continue;
- jjj = (jjj + cdim[1]) % cdim[1];
- for (int kk = -delta_m; kk <= delta_p; kk++) {
- int kkk = k + kk;
- if (!periodic && (kkk < 0 || kkk >= cdim[2])) continue;
- kkk = (kkk + cdim[2]) % cdim[2];
-
- /* Get the cell ID. */
- const int cjd = cell_getid(cdim, iii, jjj, kkk);
-
- /* Early abort */
- if (cid >= cjd) continue;
-
- /* Early abort (both same node) */
- if (cells[cid].nodeID == nodeID && cells[cjd].nodeID == nodeID)
- continue;
-
- /* Early abort (both foreign node) */
- if (cells[cid].nodeID != nodeID && cells[cjd].nodeID != nodeID)
- continue;
-
- int proxy_type = 0;
-
- /* In the hydro case, only care about direct neighbours */
- if (with_hydro) {
-
- // MATTHIEU: to do: Write a better expression for the
- // non-periodic case.
-
- /* This is super-ugly but checks for direct neighbours */
- /* with periodic BC */
- if (((abs(i - iii) <= 1 || abs(i - iii - cdim[0]) <= 1 ||
- abs(i - iii + cdim[0]) <= 1) &&
- (abs(j - jjj) <= 1 || abs(j - jjj - cdim[1]) <= 1 ||
- abs(j - jjj + cdim[1]) <= 1) &&
- (abs(k - kkk) <= 1 || abs(k - kkk - cdim[2]) <= 1 ||
- abs(k - kkk + cdim[2]) <= 1)))
- proxy_type |= (int)proxy_cell_type_hydro;
- }
-
- /* In the gravity case, check distances using the MAC. */
- if (with_gravity) {
-
- /* First just add the direct neighbours. Then look for
- some further out if the opening angle demands it */
-
- /* This is super-ugly but checks for direct neighbours */
- /* with periodic BC */
- if (((abs(i - iii) <= 1 || abs(i - iii - cdim[0]) <= 1 ||
- abs(i - iii + cdim[0]) <= 1) &&
- (abs(j - jjj) <= 1 || abs(j - jjj - cdim[1]) <= 1 ||
- abs(j - jjj + cdim[1]) <= 1) &&
- (abs(k - kkk) <= 1 || abs(k - kkk - cdim[2]) <= 1 ||
- abs(k - kkk + cdim[2]) <= 1))) {
-
- proxy_type |= (int)proxy_cell_type_gravity;
- } else {
-
- /* We don't have multipoles yet (or their CoMs) so we will
- have to cook up something based on cell locations only. We
- hence need a lower limit on the distance that the CoMs in
- those cells could have and an upper limit on the distance
- of the furthest particle in the multipole from its CoM.
- We then can decide whether we are too close for an M2L
- interaction and hence require a proxy as this pair of cells
- cannot rely on just an M2L calculation. */
-
- /* Minimal distance between any two points in the cells */
- const double min_dist_CoM2 = cell_min_dist2_same_size(
- &cells[cid], &cells[cjd], periodic, dim);
-
- /* Are we beyond the distance where the truncated forces are 0
- * but not too far such that M2L can be used? */
- if (periodic) {
-
- if ((min_dist_CoM2 < max_mesh_dist2) &&
- !(4. * r_max * r_max <
- theta_crit * theta_crit * min_dist_CoM2))
- proxy_type |= (int)proxy_cell_type_gravity;
-
- } else {
-
- if (!(4. * r_max * r_max <
- theta_crit * theta_crit * min_dist_CoM2)) {
- proxy_type |= (int)proxy_cell_type_gravity;
- }
- }
- }
- }
-
- /* Abort if not in range at all */
- if (proxy_type == proxy_cell_type_none) continue;
-
- /* Add to proxies? */
- if (cells[cid].nodeID == nodeID && cells[cjd].nodeID != nodeID) {
-
- /* Do we already have a relationship with this node? */
- int proxy_id = e->proxy_ind[cells[cjd].nodeID];
- if (proxy_id < 0) {
- if (e->nr_proxies == engine_maxproxies)
- error("Maximum number of proxies exceeded.");
-
- /* Ok, start a new proxy for this pair of nodes */
- proxy_init(&proxies[e->nr_proxies], e->nodeID,
- cells[cjd].nodeID);
-
- /* Store the information */
- e->proxy_ind[cells[cjd].nodeID] = e->nr_proxies;
- proxy_id = e->nr_proxies;
- e->nr_proxies += 1;
-
- /* Check the maximal proxy limit */
- if ((size_t)proxy_id > 8 * sizeof(long long))
- error(
- "Created more than %zd proxies. cell.mpi.sendto will "
- "overflow.",
- 8 * sizeof(long long));
- }
-
- /* Add the cell to the proxy */
- proxy_addcell_in(&proxies[proxy_id], &cells[cjd], proxy_type);
- proxy_addcell_out(&proxies[proxy_id], &cells[cid], proxy_type);
-
- /* Store info about where to send the cell */
- cells[cid].mpi.sendto |= (1ULL << proxy_id);
- }
-
- /* Same for the symmetric case? */
- if (cells[cjd].nodeID == nodeID && cells[cid].nodeID != nodeID) {
-
- /* Do we already have a relationship with this node? */
- int proxy_id = e->proxy_ind[cells[cid].nodeID];
- if (proxy_id < 0) {
- if (e->nr_proxies == engine_maxproxies)
- error("Maximum number of proxies exceeded.");
-
- /* Ok, start a new proxy for this pair of nodes */
- proxy_init(&proxies[e->nr_proxies], e->nodeID,
- cells[cid].nodeID);
-
- /* Store the information */
- e->proxy_ind[cells[cid].nodeID] = e->nr_proxies;
- proxy_id = e->nr_proxies;
- e->nr_proxies += 1;
-
- /* Check the maximal proxy limit */
- if ((size_t)proxy_id > 8 * sizeof(long long))
- error(
- "Created more than %zd proxies. cell.mpi.sendto will "
- "overflow.",
- 8 * sizeof(long long));
- }
-
- /* Add the cell to the proxy */
- proxy_addcell_in(&proxies[proxy_id], &cells[cid], proxy_type);
- proxy_addcell_out(&proxies[proxy_id], &cells[cjd], proxy_type);
-
- /* Store info about where to send the cell */
- cells[cjd].mpi.sendto |= (1ULL << proxy_id);
- }
- }
- }
- }
- }
- }
- }
-
- /* Be clear about the time */
- if (e->verbose)
- message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
- clocks_getunit());
-#else
- error("SWIFT was not compiled with MPI support.");
-#endif
-}
-
-/**
- * @brief Split the underlying space into regions and assign to separate nodes.
- *
- * @param e The #engine.
- * @param initial_partition structure defining the cell partition technique
- */
-void engine_split(struct engine *e, struct partition *initial_partition) {
-
-#ifdef WITH_MPI
- const ticks tic = getticks();
-
- struct space *s = e->s;
-
- /* Do the initial partition of the cells. */
- partition_initial_partition(initial_partition, e->nodeID, e->nr_nodes, s);
-
- /* Make the proxies. */
- engine_makeproxies(e);
-
- /* Re-allocate the local parts. */
- if (e->verbose)
- message("Re-allocating parts array from %zu to %zu.", s->size_parts,
- (size_t)(s->nr_parts * engine_redistribute_alloc_margin));
- s->size_parts = s->nr_parts * engine_redistribute_alloc_margin;
- struct part *parts_new = NULL;
- struct xpart *xparts_new = NULL;
- if (swift_memalign("parts", (void **)&parts_new, part_align,
- sizeof(struct part) * s->size_parts) != 0 ||
- swift_memalign("xparts", (void **)&xparts_new, xpart_align,
- sizeof(struct xpart) * s->size_parts) != 0)
- error("Failed to allocate new part data.");
-
- if (s->nr_parts > 0) {
- memcpy(parts_new, s->parts, sizeof(struct part) * s->nr_parts);
- memcpy(xparts_new, s->xparts, sizeof(struct xpart) * s->nr_parts);
- }
- swift_free("parts", s->parts);
- swift_free("xparts", s->xparts);
- s->parts = parts_new;
- s->xparts = xparts_new;
-
- /* Re-link the gparts to their parts. */
- if (s->nr_parts > 0 && s->nr_gparts > 0)
- part_relink_gparts_to_parts(s->parts, s->nr_parts, 0);
+ /* Re-link the gparts to their parts. */
+ if (s->nr_parts > 0 && s->nr_gparts > 0)
+ part_relink_gparts_to_parts(s->parts, s->nr_parts, 0);
/* Re-allocate the local sparts. */
if (e->verbose)
@@ -3683,170 +2556,50 @@ void engine_collect_stars_counter(struct engine *e) {
#endif
+#ifdef HAVE_SETAFFINITY
/**
- * @brief Writes a snapshot with the current state of the engine
- *
- * @param e The #engine.
+ * @brief Returns the initial affinity the main thread is using.
*/
-void engine_dump_snapshot(struct engine *e) {
-
- struct clocks_time time1, time2;
- clocks_gettime(&time1);
+cpu_set_t *engine_entry_affinity(void) {
-#ifdef SWIFT_DEBUG_CHECKS
- /* Check that all cells have been drifted to the current time.
- * That can include cells that have not
- * previously been active on this rank. */
- space_check_drift_point(e->s, e->ti_current, /* check_mpole=*/0);
+ static int use_entry_affinity = 0;
+ static cpu_set_t entry_affinity;
- /* Be verbose about this */
- if (e->nodeID == 0) {
- if (e->policy & engine_policy_cosmology)
- message("Dumping snapshot at a=%e",
- exp(e->ti_current * e->time_base) * e->cosmology->a_begin);
- else
- message("Dumping snapshot at t=%e",
- e->ti_current * e->time_base + e->time_begin);
- }
-#else
- if (e->verbose) {
- if (e->policy & engine_policy_cosmology)
- message("Dumping snapshot at a=%e",
- exp(e->ti_current * e->time_base) * e->cosmology->a_begin);
- else
- message("Dumping snapshot at t=%e",
- e->ti_current * e->time_base + e->time_begin);
+ if (!use_entry_affinity) {
+ pthread_t engine = pthread_self();
+ pthread_getaffinity_np(engine, sizeof(entry_affinity), &entry_affinity);
+ use_entry_affinity = 1;
}
-#endif
-#ifdef DEBUG_INTERACTIONS_STARS
- engine_collect_stars_counter(e);
+ return &entry_affinity;
+}
#endif
- /* Get time-step since the last mesh kick */
- if ((e->policy & engine_policy_self_gravity) && e->s->periodic) {
- const int with_cosmology = e->policy & engine_policy_cosmology;
-
- e->dt_kick_grav_mesh_for_io =
- kick_get_grav_kick_dt(e->mesh->ti_beg_mesh_next, e->ti_current,
- e->time_base, with_cosmology, e->cosmology) -
- kick_get_grav_kick_dt(
- e->mesh->ti_beg_mesh_next,
- (e->mesh->ti_beg_mesh_next + e->mesh->ti_end_mesh_next) / 2,
- e->time_base, with_cosmology, e->cosmology);
- }
-
-/* Dump (depending on the chosen strategy) ... */
-#if defined(HAVE_HDF5)
-#if defined(WITH_MPI)
-
- if (e->snapshot_distributed) {
+/**
+ * @brief Ensure the NUMA node on which we initialise (first touch) everything
+ * doesn't change before engine_init allocates NUMA-local workers.
+ */
+void engine_pin(void) {
- write_output_distributed(e, e->internal_units, e->snapshot_units, e->nodeID,
- e->nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL);
- } else {
+#ifdef HAVE_SETAFFINITY
+ cpu_set_t *entry_affinity = engine_entry_affinity();
+ int pin;
+ for (pin = 0; pin < CPU_SETSIZE && !CPU_ISSET(pin, entry_affinity); ++pin)
+ ;
-#if defined(HAVE_PARALLEL_HDF5)
- write_output_parallel(e, e->internal_units, e->snapshot_units, e->nodeID,
- e->nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL);
-#else
- write_output_serial(e, e->internal_units, e->snapshot_units, e->nodeID,
- e->nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL);
-#endif
+ cpu_set_t affinity;
+ CPU_ZERO(&affinity);
+ CPU_SET(pin, &affinity);
+ if (sched_setaffinity(0, sizeof(affinity), &affinity) != 0) {
+ error("failed to set engine's affinity");
}
#else
- write_output_single(e, e->internal_units, e->snapshot_units);
-#endif
+ error("SWIFT was not compiled with support for pinning.");
#endif
-
- /* Flag that we dumped a snapshot */
- e->step_props |= engine_step_prop_snapshot;
-
- clocks_gettime(&time2);
- if (e->verbose)
- message("writing particle properties took %.3f %s.",
- (float)clocks_diff(&time1, &time2), clocks_getunit());
}
/**
- * @brief Writes an index file with the current state of the engine
- *
- * @param e The #engine.
- */
-void engine_dump_index(struct engine *e) {
-
-#if defined(WITH_LOGGER)
- struct clocks_time time1, time2;
- clocks_gettime(&time1);
-
- if (e->verbose) {
- if (e->policy & engine_policy_cosmology)
- message("Writing index at a=%e",
- exp(e->ti_current * e->time_base) * e->cosmology->a_begin);
- else
- message("Writing index at t=%e",
- e->ti_current * e->time_base + e->time_begin);
- }
-
- /* Dump... */
- logger_write_index_file(e->logger, e);
-
- /* Flag that we dumped a snapshot */
- e->step_props |= engine_step_prop_logger_index;
-
- clocks_gettime(&time2);
- if (e->verbose)
- message("writing particle indices took %.3f %s.",
- (float)clocks_diff(&time1, &time2), clocks_getunit());
-#else
- error("SWIFT was not compiled with the logger");
-#endif
-}
-
-#ifdef HAVE_SETAFFINITY
-/**
- * @brief Returns the initial affinity the main thread is using.
- */
-cpu_set_t *engine_entry_affinity(void) {
-
- static int use_entry_affinity = 0;
- static cpu_set_t entry_affinity;
-
- if (!use_entry_affinity) {
- pthread_t engine = pthread_self();
- pthread_getaffinity_np(engine, sizeof(entry_affinity), &entry_affinity);
- use_entry_affinity = 1;
- }
-
- return &entry_affinity;
-}
-#endif
-
-/**
- * @brief Ensure the NUMA node on which we initialise (first touch) everything
- * doesn't change before engine_init allocates NUMA-local workers.
- */
-void engine_pin(void) {
-
-#ifdef HAVE_SETAFFINITY
- cpu_set_t *entry_affinity = engine_entry_affinity();
- int pin;
- for (pin = 0; pin < CPU_SETSIZE && !CPU_ISSET(pin, entry_affinity); ++pin)
- ;
-
- cpu_set_t affinity;
- CPU_ZERO(&affinity);
- CPU_SET(pin, &affinity);
- if (sched_setaffinity(0, sizeof(affinity), &affinity) != 0) {
- error("failed to set engine's affinity");
- }
-#else
- error("SWIFT was not compiled with support for pinning.");
-#endif
-}
-
-/**
- * @brief Unpins the main thread.
+ * @brief Unpins the main thread.
*/
void engine_unpin(void) {
#ifdef HAVE_SETAFFINITY
@@ -4173,717 +2926,6 @@ void engine_init(struct engine *e, struct space *s, struct swift_params *params,
engine_init_output_lists(e, params);
}
-
-/**
- * @brief configure an engine with the given number of threads, queues
- * and core affinity. Also initialises the scheduler and opens various
- * output files, computes the next timestep and initialises the
- * threadpool.
- *
- * Assumes the engine is correctly initialised i.e. is restored from a restart
- * file or has been setup by engine_init(). When restarting any output log
- * files are positioned so that further output is appended. Note that
- * parameters are not read from the engine, just the parameter file, this
- * allows values derived in this function to be changed between runs.
- * When not restarting params should be the same as given to engine_init().
- *
- * @param restart true when restarting the application.
- * @param fof true when starting a stand-alone FOF call.
- * @param e The #engine.
- * @param params The parsed parameter file.
- * @param nr_nodes The number of MPI ranks.
- * @param nodeID The MPI rank of this node.
- * @param nr_threads The number of threads per MPI rank.
- * @param with_aff use processor affinity, if supported.
- * @param verbose Is this #engine talkative ?
- * @param restart_file The name of our restart file.
- */
-void engine_config(int restart, int fof, struct engine *e,
- struct swift_params *params, int nr_nodes, int nodeID,
- int nr_threads, int with_aff, int verbose,
- const char *restart_file) {
-
- /* Store the values and initialise global fields. */
- e->nodeID = nodeID;
- e->nr_threads = nr_threads;
- e->nr_nodes = nr_nodes;
- e->proxy_ind = NULL;
- e->nr_proxies = 0;
- e->forcerebuild = 1;
- e->forcerepart = 0;
- e->restarting = restart;
- e->step_props = engine_step_prop_none;
- e->links = NULL;
- e->nr_links = 0;
- e->file_stats = NULL;
- e->file_timesteps = NULL;
- e->sfh_logger = NULL;
- e->verbose = verbose;
- e->wallclock_time = 0.f;
- e->restart_dump = 0;
- e->restart_file = restart_file;
- e->restart_next = 0;
- e->restart_dt = 0;
- e->run_fof = 0;
- engine_rank = nodeID;
-
- if (restart && fof) {
- error(
- "Can't configure the engine to be a stand-alone FOF and restarting "
- "from a check-point at the same time!");
- }
-
- /* Welcome message */
- if (e->nodeID == 0) message("Running simulation '%s'.", e->run_name);
-
- /* Get the number of queues */
- int nr_queues =
- parser_get_opt_param_int(params, "Scheduler:nr_queues", nr_threads);
- if (nr_queues <= 0) nr_queues = e->nr_threads;
- if (nr_queues != nr_threads)
- message("Number of task queues set to %d", nr_queues);
- e->s->nr_queues = nr_queues;
-
-/* Deal with affinity. For now, just figure out the number of cores. */
-#if defined(HAVE_SETAFFINITY)
- const int nr_cores = sysconf(_SC_NPROCESSORS_ONLN);
- cpu_set_t *entry_affinity = engine_entry_affinity();
- const int nr_affinity_cores = CPU_COUNT(entry_affinity);
-
- if (nr_cores > CPU_SETSIZE) /* Unlikely, except on e.g. SGI UV. */
- error("must allocate dynamic cpu_set_t (too many cores per node)");
-
- if (verbose && with_aff) {
- char *buf = (char *)malloc((nr_cores + 1) * sizeof(char));
- buf[nr_cores] = '\0';
- for (int j = 0; j < nr_cores; ++j) {
- /* Reversed bit order from convention, but same as e.g. Intel MPI's
- * I_MPI_PIN_DOMAIN explicit mask: left-to-right, LSB-to-MSB. */
- buf[j] = CPU_ISSET(j, entry_affinity) ? '1' : '0';
- }
- message("Affinity at entry: %s", buf);
- free(buf);
- }
-
- int *cpuid = NULL;
- cpu_set_t cpuset;
-
- if (with_aff) {
-
- cpuid = (int *)malloc(nr_affinity_cores * sizeof(int));
-
- int skip = 0;
- for (int k = 0; k < nr_affinity_cores; k++) {
- int c;
- for (c = skip; c < CPU_SETSIZE && !CPU_ISSET(c, entry_affinity); ++c)
- ;
- cpuid[k] = c;
- skip = c + 1;
- }
-
-#if defined(HAVE_LIBNUMA) && defined(_GNU_SOURCE)
- if ((e->policy & engine_policy_cputight) != engine_policy_cputight) {
-
- if (numa_available() >= 0) {
- if (nodeID == 0) message("prefer NUMA-distant CPUs");
-
- /* Get list of numa nodes of all available cores. */
- int *nodes = (int *)malloc(nr_affinity_cores * sizeof(int));
- int nnodes = 0;
- for (int i = 0; i < nr_affinity_cores; i++) {
- nodes[i] = numa_node_of_cpu(cpuid[i]);
- if (nodes[i] > nnodes) nnodes = nodes[i];
- }
- nnodes += 1;
-
- /* Count cores per node. */
- int *core_counts = (int *)malloc(nnodes * sizeof(int));
- for (int i = 0; i < nr_affinity_cores; i++) {
- core_counts[nodes[i]] = 0;
- }
- for (int i = 0; i < nr_affinity_cores; i++) {
- core_counts[nodes[i]] += 1;
- }
-
- /* Index cores within each node. */
- int *core_indices = (int *)malloc(nr_affinity_cores * sizeof(int));
- for (int i = nr_affinity_cores - 1; i >= 0; i--) {
- core_indices[i] = core_counts[nodes[i]];
- core_counts[nodes[i]] -= 1;
- }
-
- /* Now sort so that we pick adjacent cpuids from different nodes
- * by sorting internal node core indices. */
- int done = 0;
- while (!done) {
- done = 1;
- for (int i = 1; i < nr_affinity_cores; i++) {
- if (core_indices[i] < core_indices[i - 1]) {
- int t = cpuid[i - 1];
- cpuid[i - 1] = cpuid[i];
- cpuid[i] = t;
-
- t = core_indices[i - 1];
- core_indices[i - 1] = core_indices[i];
- core_indices[i] = t;
- done = 0;
- }
- }
- }
-
- free(nodes);
- free(core_counts);
- free(core_indices);
- }
- }
-#endif
- } else {
- if (nodeID == 0) message("no processor affinity used");
-
- } /* with_aff */
-
- /* Avoid (unexpected) interference between engine and runner threads. We can
- * do this once we've made at least one call to engine_entry_affinity and
- * maybe numa_node_of_cpu(sched_getcpu()), even if the engine isn't already
- * pinned. */
- if (with_aff) engine_unpin();
-#endif
-
- if (with_aff && nodeID == 0) {
-#ifdef HAVE_SETAFFINITY
-#ifdef WITH_MPI
- printf("[%04i] %s engine_init: cpu map is [ ", nodeID,
- clocks_get_timesincestart());
-#else
- printf("%s engine_init: cpu map is [ ", clocks_get_timesincestart());
-#endif
- for (int i = 0; i < nr_affinity_cores; i++) printf("%i ", cpuid[i]);
- printf("].\n");
-#endif
- }
-
- /* Are we doing stuff in parallel? */
- if (nr_nodes > 1) {
-#ifndef WITH_MPI
- error("SWIFT was not compiled with MPI support.");
-#else
- e->policy |= engine_policy_mpi;
- if ((e->proxies = (struct proxy *)calloc(sizeof(struct proxy),
- engine_maxproxies)) == NULL)
- error("Failed to allocate memory for proxies.");
- e->nr_proxies = 0;
-
- /* Use synchronous MPI sends and receives when redistributing. */
- e->syncredist =
- parser_get_opt_param_int(params, "DomainDecomposition:synchronous", 0);
-
-#endif
- }
-
- /* Open some global files */
- if (!fof && e->nodeID == 0) {
-
- /* When restarting append to these files. */
- const char *mode;
- if (restart)
- mode = "a";
- else
- mode = "w";
-
- char energyfileName[200] = "";
- parser_get_opt_param_string(params, "Statistics:energy_file_name",
- energyfileName,
- engine_default_energy_file_name);
- sprintf(energyfileName + strlen(energyfileName), ".txt");
- e->file_stats = fopen(energyfileName, mode);
-
- if (!restart)
- stats_write_file_header(e->file_stats, e->internal_units,
- e->physical_constants);
-
- char timestepsfileName[200] = "";
- parser_get_opt_param_string(params, "Statistics:timestep_file_name",
- timestepsfileName,
- engine_default_timesteps_file_name);
-
- sprintf(timestepsfileName + strlen(timestepsfileName), "_%d.txt",
- nr_nodes * nr_threads);
- e->file_timesteps = fopen(timestepsfileName, mode);
-
- if (!restart) {
- fprintf(
- e->file_timesteps,
- "# Host: %s\n# Branch: %s\n# Revision: %s\n# Compiler: %s, "
- "Version: %s \n# "
- "Number of threads: %d\n# Number of MPI ranks: %d\n# Hydrodynamic "
- "scheme: %s\n# Hydrodynamic kernel: %s\n# No. of neighbours: %.2f "
- "+/- %.4f\n# Eta: %f\n# Config: %s\n# CFLAGS: %s\n",
- hostname(), git_branch(), git_revision(), compiler_name(),
- compiler_version(), e->nr_threads, e->nr_nodes, SPH_IMPLEMENTATION,
- kernel_name, e->hydro_properties->target_neighbours,
- e->hydro_properties->delta_neighbours,
- e->hydro_properties->eta_neighbours, configuration_options(),
- compilation_cflags());
-
- fprintf(
- e->file_timesteps,
- "# Step Properties: Rebuild=%d, Redistribute=%d, Repartition=%d, "
- "Statistics=%d, Snapshot=%d, Restarts=%d STF=%d, FOF=%d, mesh=%d, "
- "logger=%d\n",
- engine_step_prop_rebuild, engine_step_prop_redistribute,
- engine_step_prop_repartition, engine_step_prop_statistics,
- engine_step_prop_snapshot, engine_step_prop_restarts,
- engine_step_prop_stf, engine_step_prop_fof, engine_step_prop_mesh,
- engine_step_prop_logger_index);
-
- fprintf(e->file_timesteps,
- "# %6s %14s %12s %12s %14s %9s %12s %12s %12s %12s %12s %16s "
- "[%s] %6s\n",
- "Step", "Time", "Scale-factor", "Redshift", "Time-step",
- "Time-bins", "Updates", "g-Updates", "s-Updates", "Sink-Updates",
- "b-Updates", "Wall-clock time", clocks_getunit(), "Props");
- fflush(e->file_timesteps);
- }
-
- /* Initialize the SFH logger if running with star formation */
- if (e->policy & engine_policy_star_formation) {
- e->sfh_logger = fopen("SFR.txt", mode);
- if (!restart) {
- star_formation_logger_init_log_file(e->sfh_logger, e->internal_units,
- e->physical_constants);
- fflush(e->sfh_logger);
- }
- }
- }
-
- /* Print policy */
- engine_print_policy(e);
-
- if (!fof) {
-
- /* Print information about the hydro scheme */
- if (e->policy & engine_policy_hydro) {
- if (e->nodeID == 0) hydro_props_print(e->hydro_properties);
- if (e->nodeID == 0) entropy_floor_print(e->entropy_floor);
- }
-
- /* Print information about the gravity scheme */
- if (e->policy & engine_policy_self_gravity)
- if (e->nodeID == 0) gravity_props_print(e->gravity_properties);
-
- if (e->policy & engine_policy_stars)
- if (e->nodeID == 0) stars_props_print(e->stars_properties);
-
- /* Check we have sensible time bounds */
- if (e->time_begin >= e->time_end)
- error(
- "Final simulation time (t_end = %e) must be larger than the start "
- "time (t_beg = %e)",
- e->time_end, e->time_begin);
-
- /* Check we don't have inappropriate time labels */
- if ((e->snapshot_int_time_label_on == 1) && (e->time_end <= 1.f))
- error("Snapshot integer time labels enabled but end time <= 1");
-
- /* Check we have sensible time-step values */
- if (e->dt_min > e->dt_max)
- error(
- "Minimal time-step size (%e) must be smaller than maximal time-step "
- "size (%e)",
- e->dt_min, e->dt_max);
-
- /* Info about time-steps */
- if (e->nodeID == 0) {
- message("Absolute minimal timestep size: %e", e->time_base);
-
- float dt_min = e->time_end - e->time_begin;
- while (dt_min > e->dt_min) dt_min /= 2.f;
-
- message("Minimal timestep size (on time-line): %e", dt_min);
-
- float dt_max = e->time_end - e->time_begin;
- while (dt_max > e->dt_max) dt_max /= 2.f;
-
- message("Maximal timestep size (on time-line): %e", dt_max);
- }
-
- if (e->dt_min < e->time_base && e->nodeID == 0)
- error(
- "Minimal time-step size smaller than the absolute possible minimum "
- "dt=%e",
- e->time_base);
-
- if (!(e->policy & engine_policy_cosmology))
- if (e->dt_max > (e->time_end - e->time_begin) && e->nodeID == 0)
- error("Maximal time-step size larger than the simulation run time t=%e",
- e->time_end - e->time_begin);
-
- /* Deal with outputs */
- if (e->policy & engine_policy_cosmology) {
-
- if (e->delta_time_snapshot <= 1.)
- error("Time between snapshots (%e) must be > 1.",
- e->delta_time_snapshot);
-
- if (e->delta_time_statistics <= 1.)
- error("Time between statistics (%e) must be > 1.",
- e->delta_time_statistics);
-
- if (e->a_first_snapshot < e->cosmology->a_begin)
- error(
- "Scale-factor of first snapshot (%e) must be after the simulation "
- "start a=%e.",
- e->a_first_snapshot, e->cosmology->a_begin);
-
- if (e->a_first_statistics < e->cosmology->a_begin)
- error(
- "Scale-factor of first stats output (%e) must be after the "
- "simulation start a=%e.",
- e->a_first_statistics, e->cosmology->a_begin);
-
- if (e->policy & engine_policy_structure_finding) {
-
- if (e->delta_time_stf == -1. && !e->snapshot_invoke_stf)
- error("A value for `StructureFinding:delta_time` must be specified");
-
- if (e->a_first_stf_output < e->cosmology->a_begin)
- error(
- "Scale-factor of first stf output (%e) must be after the "
- "simulation start a=%e.",
- e->a_first_stf_output, e->cosmology->a_begin);
- }
-
- if (e->policy & engine_policy_fof) {
-
- if (e->delta_time_fof <= 1.)
- error("Time between FOF (%e) must be > 1.", e->delta_time_fof);
-
- if (e->a_first_fof_call < e->cosmology->a_begin)
- error(
- "Scale-factor of first fof call (%e) must be after the "
- "simulation start a=%e.",
- e->a_first_fof_call, e->cosmology->a_begin);
- }
-
- } else {
-
- if (e->delta_time_snapshot <= 0.)
- error("Time between snapshots (%e) must be positive.",
- e->delta_time_snapshot);
-
- if (e->delta_time_statistics <= 0.)
- error("Time between statistics (%e) must be positive.",
- e->delta_time_statistics);
-
- /* Find the time of the first output */
- if (e->time_first_snapshot < e->time_begin)
- error(
- "Time of first snapshot (%e) must be after the simulation start "
- "t=%e.",
- e->time_first_snapshot, e->time_begin);
-
- if (e->time_first_statistics < e->time_begin)
- error(
- "Time of first stats output (%e) must be after the simulation "
- "start t=%e.",
- e->time_first_statistics, e->time_begin);
-
- if (e->policy & engine_policy_structure_finding) {
-
- if (e->delta_time_stf == -1. && !e->snapshot_invoke_stf)
- error("A value for `StructureFinding:delta_time` must be specified");
-
- if (e->delta_time_stf <= 0. && e->delta_time_stf != -1.)
- error("Time between STF (%e) must be positive.", e->delta_time_stf);
-
- if (e->time_first_stf_output < e->time_begin)
- error(
- "Time of first STF (%e) must be after the simulation start t=%e.",
- e->time_first_stf_output, e->time_begin);
- }
- }
-
- /* Try to ensure the snapshot directory exists */
- if (e->nodeID == 0) io_make_snapshot_subdir(e->snapshot_subdir);
-
- /* Get the total mass */
- e->total_mass = 0.;
- for (size_t i = 0; i < e->s->nr_gparts; ++i)
- e->total_mass += e->s->gparts[i].mass;
-
-/* Reduce the total mass */
-#ifdef WITH_MPI
- MPI_Allreduce(MPI_IN_PLACE, &e->total_mass, 1, MPI_DOUBLE, MPI_SUM,
- MPI_COMM_WORLD);
-#endif
-
-#if defined(WITH_LOGGER)
- if ((e->policy & engine_policy_logger) && e->nodeID == 0)
- message(
- "WARNING: There is currently no way of predicting the output "
- "size, please use it carefully");
-#endif
-
- /* Find the time of the first snapshot output */
- engine_compute_next_snapshot_time(e);
-
- /* Find the time of the first statistics output */
- engine_compute_next_statistics_time(e);
-
- /* Find the time of the first line of sight output */
- if (e->policy & engine_policy_line_of_sight) {
- engine_compute_next_los_time(e);
- }
-
- /* Find the time of the first stf output */
- if (e->policy & engine_policy_structure_finding) {
- engine_compute_next_stf_time(e);
- }
-
- /* Find the time of the first stf output */
- if (e->policy & engine_policy_fof) {
- engine_compute_next_fof_time(e);
- }
-
- /* Check that the snapshot naming policy is valid */
- if (e->snapshot_invoke_stf && e->snapshot_int_time_label_on)
- error(
- "Cannot use snapshot time labels and VELOCIraptor invocations "
- "together!");
-
- /* Check that we are invoking VELOCIraptor only if we have it */
- if (e->snapshot_invoke_stf &&
- !(e->policy & engine_policy_structure_finding)) {
- error(
- "Invoking VELOCIraptor after snapshots but structure finding wasn't "
- "activated at runtime (Use --velociraptor).");
- }
-
- /* Whether restarts are enabled. Yes by default. Can be changed on restart.
- */
- e->restart_dump = parser_get_opt_param_int(params, "Restarts:enable", 1);
-
- /* Whether to save backup copies of the previous restart files. */
- e->restart_save = parser_get_opt_param_int(params, "Restarts:save", 1);
-
- /* Whether restarts should be dumped on exit. Not by default. Can be changed
- * on restart. */
- e->restart_onexit = parser_get_opt_param_int(params, "Restarts:onexit", 0);
-
- /* Hours between restart dumps. Can be changed on restart. */
- float dhours =
- parser_get_opt_param_float(params, "Restarts:delta_hours", 5.0f);
- if (e->nodeID == 0) {
- if (e->restart_dump)
- message("Restarts will be dumped every %f hours", dhours);
- else
- message("WARNING: restarts will not be dumped");
-
- if (e->verbose && e->restart_onexit)
- message("Restarts will be dumped after the final step");
- }
-
- /* Internally we use ticks, so convert into a delta ticks. Assumes we can
- * convert from ticks into milliseconds. */
- e->restart_dt = clocks_to_ticks(dhours * 60.0 * 60.0 * 1000.0);
-
- /* The first dump will happen no sooner than restart_dt ticks in the
- * future. */
- e->restart_next = getticks() + e->restart_dt;
- }
-
-/* Construct types for MPI communications */
-#ifdef WITH_MPI
- part_create_mpi_types();
- multipole_create_mpi_types();
- stats_create_mpi_type();
- proxy_create_mpi_type();
- task_create_mpi_comms();
-#ifdef WITH_FOF
- fof_create_mpi_types();
-#endif /* WITH_FOF */
-#endif /* WITH_MPI */
-
- if (!fof) {
-
- /* Initialise the collection group. */
- collectgroup_init();
- }
-
- /* Initialize the threadpool. */
- threadpool_init(&e->threadpool, e->nr_threads);
-
- /* First of all, init the barrier and lock it. */
- if (swift_barrier_init(&e->wait_barrier, NULL, e->nr_threads + 1) != 0 ||
- swift_barrier_init(&e->run_barrier, NULL, e->nr_threads + 1) != 0)
- error("Failed to initialize barrier.");
-
- /* Expected average for tasks per cell. If set to zero we use a heuristic
- * guess based on the numbers of cells and how many tasks per cell we expect.
- * On restart this number cannot be estimated (no cells yet), so we recover
- * from the end of the dumped run. Can be changed on restart. */
- e->tasks_per_cell =
- parser_get_opt_param_float(params, "Scheduler:tasks_per_cell", 0.0);
- e->tasks_per_cell_max = 0.0f;
-
- float maxtasks = 0;
- if (restart)
- maxtasks = e->restart_max_tasks;
- else
- maxtasks = engine_estimate_nr_tasks(e);
-
- /* Estimated number of links per tasks */
- e->links_per_tasks =
- parser_get_opt_param_float(params, "Scheduler:links_per_tasks", 25.);
-
- /* Init the scheduler. */
- scheduler_init(&e->sched, e->s, maxtasks, nr_queues,
- (e->policy & scheduler_flag_steal), e->nodeID, &e->threadpool);
-
- /* Maximum size of MPI task messages, in KB, that should not be buffered,
- * that is sent using MPI_Issend, not MPI_Isend. 4Mb by default. Can be
- * changed on restart.
- */
- e->sched.mpi_message_limit =
- parser_get_opt_param_int(params, "Scheduler:mpi_message_limit", 4) * 1024;
-
- if (restart) {
-
- /* Overwrite the constants for the scheduler */
- space_maxsize = parser_get_opt_param_int(params, "Scheduler:cell_max_size",
- space_maxsize);
- space_subsize_pair_hydro = parser_get_opt_param_int(
- params, "Scheduler:cell_sub_size_pair_hydro", space_subsize_pair_hydro);
- space_subsize_self_hydro = parser_get_opt_param_int(
- params, "Scheduler:cell_sub_size_self_hydro", space_subsize_self_hydro);
- space_subsize_pair_stars = parser_get_opt_param_int(
- params, "Scheduler:cell_sub_size_pair_stars", space_subsize_pair_stars);
- space_subsize_self_stars = parser_get_opt_param_int(
- params, "Scheduler:cell_sub_size_self_stars", space_subsize_self_stars);
- space_subsize_pair_grav = parser_get_opt_param_int(
- params, "Scheduler:cell_sub_size_pair_grav", space_subsize_pair_grav);
- space_subsize_self_grav = parser_get_opt_param_int(
- params, "Scheduler:cell_sub_size_self_grav", space_subsize_self_grav);
- space_splitsize = parser_get_opt_param_int(
- params, "Scheduler:cell_split_size", space_splitsize);
- space_subdepth_diff_grav =
- parser_get_opt_param_int(params, "Scheduler:cell_subdepth_diff_grav",
- space_subdepth_diff_grav_default);
- space_extra_parts = parser_get_opt_param_int(
- params, "Scheduler:cell_extra_parts", space_extra_parts);
- space_extra_sparts = parser_get_opt_param_int(
- params, "Scheduler:cell_extra_sparts", space_extra_sparts);
- space_extra_gparts = parser_get_opt_param_int(
- params, "Scheduler:cell_extra_gparts", space_extra_gparts);
- space_extra_bparts = parser_get_opt_param_int(
- params, "Scheduler:cell_extra_bparts", space_extra_bparts);
-
- engine_max_parts_per_ghost =
- parser_get_opt_param_int(params, "Scheduler:engine_max_parts_per_ghost",
- engine_max_parts_per_ghost);
- engine_max_sparts_per_ghost = parser_get_opt_param_int(
- params, "Scheduler:engine_max_sparts_per_ghost",
- engine_max_sparts_per_ghost);
-
- engine_max_parts_per_cooling = parser_get_opt_param_int(
- params, "Scheduler:engine_max_parts_per_cooling",
- engine_max_parts_per_cooling);
- }
-
- /* Allocate and init the threads. */
- if (swift_memalign("runners", (void **)&e->runners, SWIFT_CACHE_ALIGNMENT,
- e->nr_threads * sizeof(struct runner)) != 0)
- error("Failed to allocate threads array.");
-
- for (int k = 0; k < e->nr_threads; k++) {
- e->runners[k].id = k;
- e->runners[k].e = e;
- if (pthread_create(&e->runners[k].thread, NULL, &runner_main,
- &e->runners[k]) != 0)
- error("Failed to create runner thread.");
-
- /* Try to pin the runner to a given core */
- if (with_aff &&
- (e->policy & engine_policy_setaffinity) == engine_policy_setaffinity) {
-#if defined(HAVE_SETAFFINITY)
-
- /* Set a reasonable queue ID. */
- int coreid = k % nr_affinity_cores;
- e->runners[k].cpuid = cpuid[coreid];
-
- if (nr_queues < e->nr_threads)
- e->runners[k].qid = cpuid[coreid] * nr_queues / nr_affinity_cores;
- else
- e->runners[k].qid = k;
-
- /* Set the cpu mask to zero | e->id. */
- CPU_ZERO(&cpuset);
- CPU_SET(cpuid[coreid], &cpuset);
-
- /* Apply this mask to the runner's pthread. */
- if (pthread_setaffinity_np(e->runners[k].thread, sizeof(cpu_set_t),
- &cpuset) != 0)
- error("Failed to set thread affinity.");
-
-#else
- error("SWIFT was not compiled with affinity enabled.");
-#endif
- } else {
- e->runners[k].cpuid = k;
- e->runners[k].qid = k * nr_queues / e->nr_threads;
- }
-
- /* Allocate particle caches. */
- e->runners[k].ci_gravity_cache.count = 0;
- e->runners[k].cj_gravity_cache.count = 0;
- gravity_cache_init(&e->runners[k].ci_gravity_cache, space_splitsize);
- gravity_cache_init(&e->runners[k].cj_gravity_cache, space_splitsize);
-#ifdef WITH_VECTORIZATION
- e->runners[k].ci_cache.count = 0;
- e->runners[k].cj_cache.count = 0;
- cache_init(&e->runners[k].ci_cache, CACHE_SIZE);
- cache_init(&e->runners[k].cj_cache, CACHE_SIZE);
-#endif
-
- if (verbose) {
- if (with_aff)
- message("runner %i on cpuid=%i with qid=%i.", e->runners[k].id,
- e->runners[k].cpuid, e->runners[k].qid);
- else
- message("runner %i using qid=%i no cpuid.", e->runners[k].id,
- e->runners[k].qid);
- }
- }
-
-#ifdef WITH_LOGGER
- if ((e->policy & engine_policy_logger) && !restart) {
- /* Write the particle logger header */
- logger_write_file_header(e->logger);
- }
-#endif
-
- /* Initialise the structure finder */
-#ifdef HAVE_VELOCIRAPTOR
- if (e->policy & engine_policy_structure_finding) velociraptor_init(e);
-#endif
-
- /* Free the affinity stuff */
-#if defined(HAVE_SETAFFINITY)
- if (with_aff) {
- free(cpuid);
- }
-#endif
-
-#ifdef SWIFT_DUMPER_THREAD
-
- /* Start the dumper thread.*/
- engine_dumper_init(e);
-#endif
-
- /* Wait for the runner threads to be in place. */
- swift_barrier_wait(&e->wait_barrier);
-}
-
/**
* @brief Prints the current policy of an engine
*
@@ -4910,406 +2952,6 @@ void engine_print_policy(struct engine *e) {
#endif
}
-/**
- * @brief Computes the next time (on the time line) for a dump
- *
- * @param e The #engine.
- */
-void engine_compute_next_snapshot_time(struct engine *e) {
-
- /* Do output_list file case */
- if (e->output_list_snapshots) {
- output_list_read_next_time(e->output_list_snapshots, e, "snapshots",
- &e->ti_next_snapshot);
- return;
- }
-
- /* Find upper-bound on last output */
- double time_end;
- if (e->policy & engine_policy_cosmology)
- time_end = e->cosmology->a_end * e->delta_time_snapshot;
- else
- time_end = e->time_end + e->delta_time_snapshot;
-
- /* Find next snasphot above current time */
- double time;
- if (e->policy & engine_policy_cosmology)
- time = e->a_first_snapshot;
- else
- time = e->time_first_snapshot;
-
- int found_snapshot_time = 0;
- while (time < time_end) {
-
- /* Output time on the integer timeline */
- if (e->policy & engine_policy_cosmology)
- e->ti_next_snapshot = log(time / e->cosmology->a_begin) / e->time_base;
- else
- e->ti_next_snapshot = (time - e->time_begin) / e->time_base;
-
- /* Found it? */
- if (e->ti_next_snapshot > e->ti_current) {
- found_snapshot_time = 1;
- break;
- }
-
- if (e->policy & engine_policy_cosmology)
- time *= e->delta_time_snapshot;
- else
- time += e->delta_time_snapshot;
- }
-
- /* Deal with last snapshot */
- if (!found_snapshot_time) {
- e->ti_next_snapshot = -1;
- if (e->verbose) message("No further output time.");
- } else {
-
- /* Be nice, talk... */
- if (e->policy & engine_policy_cosmology) {
- const double next_snapshot_time =
- exp(e->ti_next_snapshot * e->time_base) * e->cosmology->a_begin;
- if (e->verbose)
- message("Next snapshot time set to a=%e.", next_snapshot_time);
- } else {
- const double next_snapshot_time =
- e->ti_next_snapshot * e->time_base + e->time_begin;
- if (e->verbose)
- message("Next snapshot time set to t=%e.", next_snapshot_time);
- }
- }
-}
-
-/**
- * @brief Computes the next time (on the time line) for a statistics dump
- *
- * @param e The #engine.
- */
-void engine_compute_next_statistics_time(struct engine *e) {
- /* Do output_list file case */
- if (e->output_list_stats) {
- output_list_read_next_time(e->output_list_stats, e, "stats",
- &e->ti_next_stats);
- return;
- }
-
- /* Find upper-bound on last output */
- double time_end;
- if (e->policy & engine_policy_cosmology)
- time_end = e->cosmology->a_end * e->delta_time_statistics;
- else
- time_end = e->time_end + e->delta_time_statistics;
-
- /* Find next snasphot above current time */
- double time;
- if (e->policy & engine_policy_cosmology)
- time = e->a_first_statistics;
- else
- time = e->time_first_statistics;
-
- int found_stats_time = 0;
- while (time < time_end) {
-
- /* Output time on the integer timeline */
- if (e->policy & engine_policy_cosmology)
- e->ti_next_stats = log(time / e->cosmology->a_begin) / e->time_base;
- else
- e->ti_next_stats = (time - e->time_begin) / e->time_base;
-
- /* Found it? */
- if (e->ti_next_stats > e->ti_current) {
- found_stats_time = 1;
- break;
- }
-
- if (e->policy & engine_policy_cosmology)
- time *= e->delta_time_statistics;
- else
- time += e->delta_time_statistics;
- }
-
- /* Deal with last statistics */
- if (!found_stats_time) {
- e->ti_next_stats = -1;
- if (e->verbose) message("No further output time.");
- } else {
-
- /* Be nice, talk... */
- if (e->policy & engine_policy_cosmology) {
- const double next_statistics_time =
- exp(e->ti_next_stats * e->time_base) * e->cosmology->a_begin;
- if (e->verbose)
- message("Next output time for stats set to a=%e.",
- next_statistics_time);
- } else {
- const double next_statistics_time =
- e->ti_next_stats * e->time_base + e->time_begin;
- if (e->verbose)
- message("Next output time for stats set to t=%e.",
- next_statistics_time);
- }
- }
-}
-
-/**
- * @brief Computes the next time (on the time line) for a line of sight dump
- *
- * @param e The #engine.
- */
-void engine_compute_next_los_time(struct engine *e) {
- /* Do output_list file case */
- if (e->output_list_los) {
- output_list_read_next_time(e->output_list_los, e, "line of sights",
- &e->ti_next_los);
- return;
- }
-
- /* Find upper-bound on last output */
- double time_end;
- if (e->policy & engine_policy_cosmology)
- time_end = e->cosmology->a_end * e->delta_time_los;
- else
- time_end = e->time_end + e->delta_time_los;
-
- /* Find next los above current time */
- double time;
- if (e->policy & engine_policy_cosmology)
- time = e->a_first_los;
- else
- time = e->time_first_los;
-
- int found_los_time = 0;
- while (time < time_end) {
-
- /* Output time on the integer timeline */
- if (e->policy & engine_policy_cosmology)
- e->ti_next_los = log(time / e->cosmology->a_begin) / e->time_base;
- else
- e->ti_next_los = (time - e->time_begin) / e->time_base;
-
- /* Found it? */
- if (e->ti_next_los > e->ti_current) {
- found_los_time = 1;
- break;
- }
-
- if (e->policy & engine_policy_cosmology)
- time *= e->delta_time_los;
- else
- time += e->delta_time_los;
- }
-
- /* Deal with last line of sight */
- if (!found_los_time) {
- e->ti_next_los = -1;
- if (e->verbose) message("No further LOS output time.");
- } else {
-
- /* Be nice, talk... */
- if (e->policy & engine_policy_cosmology) {
- const double next_los_time =
- exp(e->ti_next_los * e->time_base) * e->cosmology->a_begin;
- if (e->verbose)
- message("Next output time for line of sight set to a=%e.",
- next_los_time);
- } else {
- const double next_los_time =
- e->ti_next_los * e->time_base + e->time_begin;
- if (e->verbose)
- message("Next output time for line of sight set to t=%e.",
- next_los_time);
- }
- }
-}
-
-/**
- * @brief Computes the next time (on the time line) for structure finding
- *
- * @param e The #engine.
- */
-void engine_compute_next_stf_time(struct engine *e) {
- /* Do output_list file case */
- if (e->output_list_stf) {
- output_list_read_next_time(e->output_list_stf, e, "stf", &e->ti_next_stf);
- return;
- }
-
- /* Find upper-bound on last output */
- double time_end;
- if (e->policy & engine_policy_cosmology)
- time_end = e->cosmology->a_end * e->delta_time_stf;
- else
- time_end = e->time_end + e->delta_time_stf;
-
- /* Find next snasphot above current time */
- double time;
- if (e->policy & engine_policy_cosmology)
- time = e->a_first_stf_output;
- else
- time = e->time_first_stf_output;
-
- int found_stf_time = 0;
- while (time < time_end) {
-
- /* Output time on the integer timeline */
- if (e->policy & engine_policy_cosmology)
- e->ti_next_stf = log(time / e->cosmology->a_begin) / e->time_base;
- else
- e->ti_next_stf = (time - e->time_begin) / e->time_base;
-
- /* Found it? */
- if (e->ti_next_stf > e->ti_current) {
- found_stf_time = 1;
- break;
- }
-
- if (e->policy & engine_policy_cosmology)
- time *= e->delta_time_stf;
- else
- time += e->delta_time_stf;
- }
-
- /* Deal with last snapshot */
- if (!found_stf_time) {
- e->ti_next_stf = -1;
- if (e->verbose) message("No further output time.");
- } else {
-
- /* Be nice, talk... */
- if (e->policy & engine_policy_cosmology) {
- const float next_stf_time =
- exp(e->ti_next_stf * e->time_base) * e->cosmology->a_begin;
- if (e->verbose)
- message("Next VELOCIraptor time set to a=%e.", next_stf_time);
- } else {
- const float next_stf_time = e->ti_next_stf * e->time_base + e->time_begin;
- if (e->verbose)
- message("Next VELOCIraptor time set to t=%e.", next_stf_time);
- }
- }
-}
-
-/**
- * @brief Computes the next time (on the time line) for FoF black holes seeding
- *
- * @param e The #engine.
- */
-void engine_compute_next_fof_time(struct engine *e) {
-
- /* Find upper-bound on last output */
- double time_end;
- if (e->policy & engine_policy_cosmology)
- time_end = e->cosmology->a_end * e->delta_time_fof;
- else
- time_end = e->time_end + e->delta_time_fof;
-
- /* Find next snasphot above current time */
- double time;
- if (e->policy & engine_policy_cosmology)
- time = e->a_first_fof_call;
- else
- time = e->time_first_fof_call;
-
- int found_fof_time = 0;
- while (time < time_end) {
-
- /* Output time on the integer timeline */
- if (e->policy & engine_policy_cosmology)
- e->ti_next_fof = log(time / e->cosmology->a_begin) / e->time_base;
- else
- e->ti_next_fof = (time - e->time_begin) / e->time_base;
-
- /* Found it? */
- if (e->ti_next_fof > e->ti_current) {
- found_fof_time = 1;
- break;
- }
-
- if (e->policy & engine_policy_cosmology)
- time *= e->delta_time_fof;
- else
- time += e->delta_time_fof;
- }
-
- /* Deal with last snapshot */
- if (!found_fof_time) {
- e->ti_next_fof = -1;
- if (e->verbose) message("No further FoF time.");
- } else {
-
- /* Be nice, talk... */
- if (e->policy & engine_policy_cosmology) {
- const float next_fof_time =
- exp(e->ti_next_fof * e->time_base) * e->cosmology->a_begin;
- // if (e->verbose)
- message("Next FoF time set to a=%e.", next_fof_time);
- } else {
- const float next_fof_time = e->ti_next_fof * e->time_base + e->time_begin;
- if (e->verbose) message("Next FoF time set to t=%e.", next_fof_time);
- }
- }
-}
-
-/**
- * @brief Initialize all the output_list required by the engine
- *
- * @param e The #engine.
- * @param params The #swift_params.
- */
-void engine_init_output_lists(struct engine *e, struct swift_params *params) {
- /* Deal with snapshots */
- double snaps_time_first;
- e->output_list_snapshots = NULL;
- output_list_init(&e->output_list_snapshots, e, "Snapshots",
- &e->delta_time_snapshot, &snaps_time_first);
-
- if (e->output_list_snapshots) {
- if (e->policy & engine_policy_cosmology)
- e->a_first_snapshot = snaps_time_first;
- else
- e->time_first_snapshot = snaps_time_first;
- }
-
- /* Deal with stats */
- double stats_time_first;
- e->output_list_stats = NULL;
- output_list_init(&e->output_list_stats, e, "Statistics",
- &e->delta_time_statistics, &stats_time_first);
-
- if (e->output_list_stats) {
- if (e->policy & engine_policy_cosmology)
- e->a_first_statistics = stats_time_first;
- else
- e->time_first_statistics = stats_time_first;
- }
-
- /* Deal with stf */
- double stf_time_first;
- e->output_list_stf = NULL;
- output_list_init(&e->output_list_stf, e, "StructureFinding",
- &e->delta_time_stf, &stf_time_first);
-
- if (e->output_list_stf) {
- if (e->policy & engine_policy_cosmology)
- e->a_first_stf_output = stf_time_first;
- else
- e->time_first_stf_output = stf_time_first;
- }
-
- /* Deal with line of sight */
- double los_time_first;
- e->output_list_los = NULL;
- output_list_init(&e->output_list_los, e, "LineOfSight", &e->delta_time_los,
- &los_time_first);
-
- if (e->output_list_los) {
- if (e->policy & engine_policy_cosmology)
- e->a_first_los = los_time_first;
- else
- e->time_first_los = los_time_first;
- }
-}
-
/**
* @brief Computes the maximal time-step allowed by the max RMS displacement
* condition.
diff --git a/src/engine_config.c b/src/engine_config.c
new file mode 100644
index 0000000000000000000000000000000000000000..d769f99cf5a9a42f4af97006c80618d711fbbf55
--- /dev/null
+++ b/src/engine_config.c
@@ -0,0 +1,760 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#ifdef HAVE_LIBNUMA
+#include <numa.h>
+#endif
+
+/* This object's header. */
+#include "engine.h"
+
+/* Local headers. */
+#include "fof.h"
+#include "part.h"
+#include "proxy.h"
+#include "star_formation_logger.h"
+#include "stars_io.h"
+#include "statistics.h"
+#include "version.h"
+
+extern int engine_max_parts_per_ghost;
+extern int engine_max_sparts_per_ghost;
+extern int engine_max_parts_per_cooling;
+
+/* Particle cache size. */
+#define CACHE_SIZE 512
+
+/**
+ * @brief configure an engine with the given number of threads, queues
+ * and core affinity. Also initialises the scheduler and opens various
+ * output files, computes the next timestep and initialises the
+ * threadpool.
+ *
+ * Assumes the engine is correctly initialised i.e. is restored from a restart
+ * file or has been setup by engine_init(). When restarting any output log
+ * files are positioned so that further output is appended. Note that
+ * parameters are not read from the engine, just the parameter file, this
+ * allows values derived in this function to be changed between runs.
+ * When not restarting params should be the same as given to engine_init().
+ *
+ * @param restart true when restarting the application.
+ * @param fof true when starting a stand-alone FOF call.
+ * @param e The #engine.
+ * @param params The parsed parameter file.
+ * @param nr_nodes The number of MPI ranks.
+ * @param nodeID The MPI rank of this node.
+ * @param nr_threads The number of threads per MPI rank.
+ * @param with_aff use processor affinity, if supported.
+ * @param verbose Is this #engine talkative ?
+ * @param restart_file The name of our restart file.
+ */
+void engine_config(int restart, int fof, struct engine *e,
+ struct swift_params *params, int nr_nodes, int nodeID,
+ int nr_threads, int with_aff, int verbose,
+ const char *restart_file) {
+
+ /* Store the values and initialise global fields. */
+ e->nodeID = nodeID;
+ e->nr_threads = nr_threads;
+ e->nr_nodes = nr_nodes;
+ e->proxy_ind = NULL;
+ e->nr_proxies = 0;
+ e->forcerebuild = 1;
+ e->forcerepart = 0;
+ e->restarting = restart;
+ e->step_props = engine_step_prop_none;
+ e->links = NULL;
+ e->nr_links = 0;
+ e->file_stats = NULL;
+ e->file_timesteps = NULL;
+ e->sfh_logger = NULL;
+ e->verbose = verbose;
+ e->wallclock_time = 0.f;
+ e->restart_dump = 0;
+ e->restart_file = restart_file;
+ e->restart_next = 0;
+ e->restart_dt = 0;
+ e->run_fof = 0;
+ engine_rank = nodeID;
+
+ if (restart && fof) {
+ error(
+ "Can't configure the engine to be a stand-alone FOF and restarting "
+ "from a check-point at the same time!");
+ }
+
+ /* Welcome message */
+ if (e->nodeID == 0) message("Running simulation '%s'.", e->run_name);
+
+ /* Get the number of queues */
+ int nr_queues =
+ parser_get_opt_param_int(params, "Scheduler:nr_queues", nr_threads);
+ if (nr_queues <= 0) nr_queues = e->nr_threads;
+ if (nr_queues != nr_threads)
+ message("Number of task queues set to %d", nr_queues);
+ e->s->nr_queues = nr_queues;
+
+/* Deal with affinity. For now, just figure out the number of cores. */
+#if defined(HAVE_SETAFFINITY)
+ const int nr_cores = sysconf(_SC_NPROCESSORS_ONLN);
+ cpu_set_t *entry_affinity = engine_entry_affinity();
+ const int nr_affinity_cores = CPU_COUNT(entry_affinity);
+
+ if (nr_cores > CPU_SETSIZE) /* Unlikely, except on e.g. SGI UV. */
+ error("must allocate dynamic cpu_set_t (too many cores per node)");
+
+ if (verbose && with_aff) {
+ char *buf = (char *)malloc((nr_cores + 1) * sizeof(char));
+ buf[nr_cores] = '\0';
+ for (int j = 0; j < nr_cores; ++j) {
+ /* Reversed bit order from convention, but same as e.g. Intel MPI's
+ * I_MPI_PIN_DOMAIN explicit mask: left-to-right, LSB-to-MSB. */
+ buf[j] = CPU_ISSET(j, entry_affinity) ? '1' : '0';
+ }
+ message("Affinity at entry: %s", buf);
+ free(buf);
+ }
+
+ int *cpuid = NULL;
+ cpu_set_t cpuset;
+
+ if (with_aff) {
+
+ cpuid = (int *)malloc(nr_affinity_cores * sizeof(int));
+
+ int skip = 0;
+ for (int k = 0; k < nr_affinity_cores; k++) {
+ int c;
+ for (c = skip; c < CPU_SETSIZE && !CPU_ISSET(c, entry_affinity); ++c)
+ ;
+ cpuid[k] = c;
+ skip = c + 1;
+ }
+
+#if defined(HAVE_LIBNUMA) && defined(_GNU_SOURCE)
+ if ((e->policy & engine_policy_cputight) != engine_policy_cputight) {
+
+ if (numa_available() >= 0) {
+ if (nodeID == 0) message("prefer NUMA-distant CPUs");
+
+ /* Get list of numa nodes of all available cores. */
+ int *nodes = (int *)malloc(nr_affinity_cores * sizeof(int));
+ int nnodes = 0;
+ for (int i = 0; i < nr_affinity_cores; i++) {
+ nodes[i] = numa_node_of_cpu(cpuid[i]);
+ if (nodes[i] > nnodes) nnodes = nodes[i];
+ }
+ nnodes += 1;
+
+ /* Count cores per node. */
+ int *core_counts = (int *)malloc(nnodes * sizeof(int));
+ for (int i = 0; i < nr_affinity_cores; i++) {
+ core_counts[nodes[i]] = 0;
+ }
+ for (int i = 0; i < nr_affinity_cores; i++) {
+ core_counts[nodes[i]] += 1;
+ }
+
+ /* Index cores within each node. */
+ int *core_indices = (int *)malloc(nr_affinity_cores * sizeof(int));
+ for (int i = nr_affinity_cores - 1; i >= 0; i--) {
+ core_indices[i] = core_counts[nodes[i]];
+ core_counts[nodes[i]] -= 1;
+ }
+
+ /* Now sort so that we pick adjacent cpuids from different nodes
+ * by sorting internal node core indices. */
+ int done = 0;
+ while (!done) {
+ done = 1;
+ for (int i = 1; i < nr_affinity_cores; i++) {
+ if (core_indices[i] < core_indices[i - 1]) {
+ int t = cpuid[i - 1];
+ cpuid[i - 1] = cpuid[i];
+ cpuid[i] = t;
+
+ t = core_indices[i - 1];
+ core_indices[i - 1] = core_indices[i];
+ core_indices[i] = t;
+ done = 0;
+ }
+ }
+ }
+
+ free(nodes);
+ free(core_counts);
+ free(core_indices);
+ }
+ }
+#endif
+ } else {
+ if (nodeID == 0) message("no processor affinity used");
+
+ } /* with_aff */
+
+ /* Avoid (unexpected) interference between engine and runner threads. We can
+ * do this once we've made at least one call to engine_entry_affinity and
+ * maybe numa_node_of_cpu(sched_getcpu()), even if the engine isn't already
+ * pinned. */
+ if (with_aff) engine_unpin();
+#endif
+
+ if (with_aff && nodeID == 0) {
+#ifdef HAVE_SETAFFINITY
+#ifdef WITH_MPI
+ printf("[%04i] %s engine_init: cpu map is [ ", nodeID,
+ clocks_get_timesincestart());
+#else
+ printf("%s engine_init: cpu map is [ ", clocks_get_timesincestart());
+#endif
+ for (int i = 0; i < nr_affinity_cores; i++) printf("%i ", cpuid[i]);
+ printf("].\n");
+#endif
+ }
+
+ /* Are we doing stuff in parallel? */
+ if (nr_nodes > 1) {
+#ifndef WITH_MPI
+ error("SWIFT was not compiled with MPI support.");
+#else
+ e->policy |= engine_policy_mpi;
+ if ((e->proxies = (struct proxy *)calloc(sizeof(struct proxy),
+ engine_maxproxies)) == NULL)
+ error("Failed to allocate memory for proxies.");
+ e->nr_proxies = 0;
+
+ /* Use synchronous MPI sends and receives when redistributing. */
+ e->syncredist =
+ parser_get_opt_param_int(params, "DomainDecomposition:synchronous", 0);
+
+#endif
+ }
+
+ /* Open some global files */
+ if (!fof && e->nodeID == 0) {
+
+ /* When restarting append to these files. */
+ const char *mode;
+ if (restart)
+ mode = "a";
+ else
+ mode = "w";
+
+ char energyfileName[200] = "";
+ parser_get_opt_param_string(params, "Statistics:energy_file_name",
+ energyfileName,
+ engine_default_energy_file_name);
+ sprintf(energyfileName + strlen(energyfileName), ".txt");
+ e->file_stats = fopen(energyfileName, mode);
+
+ if (!restart)
+ stats_write_file_header(e->file_stats, e->internal_units,
+ e->physical_constants);
+
+ char timestepsfileName[200] = "";
+ parser_get_opt_param_string(params, "Statistics:timestep_file_name",
+ timestepsfileName,
+ engine_default_timesteps_file_name);
+
+ sprintf(timestepsfileName + strlen(timestepsfileName), "_%d.txt",
+ nr_nodes * nr_threads);
+ e->file_timesteps = fopen(timestepsfileName, mode);
+
+ if (!restart) {
+ fprintf(
+ e->file_timesteps,
+ "# Host: %s\n# Branch: %s\n# Revision: %s\n# Compiler: %s, "
+ "Version: %s \n# "
+ "Number of threads: %d\n# Number of MPI ranks: %d\n# Hydrodynamic "
+ "scheme: %s\n# Hydrodynamic kernel: %s\n# No. of neighbours: %.2f "
+ "+/- %.4f\n# Eta: %f\n# Config: %s\n# CFLAGS: %s\n",
+ hostname(), git_branch(), git_revision(), compiler_name(),
+ compiler_version(), e->nr_threads, e->nr_nodes, SPH_IMPLEMENTATION,
+ kernel_name, e->hydro_properties->target_neighbours,
+ e->hydro_properties->delta_neighbours,
+ e->hydro_properties->eta_neighbours, configuration_options(),
+ compilation_cflags());
+
+ fprintf(
+ e->file_timesteps,
+ "# Step Properties: Rebuild=%d, Redistribute=%d, Repartition=%d, "
+ "Statistics=%d, Snapshot=%d, Restarts=%d STF=%d, FOF=%d, mesh=%d, "
+ "logger=%d\n",
+ engine_step_prop_rebuild, engine_step_prop_redistribute,
+ engine_step_prop_repartition, engine_step_prop_statistics,
+ engine_step_prop_snapshot, engine_step_prop_restarts,
+ engine_step_prop_stf, engine_step_prop_fof, engine_step_prop_mesh,
+ engine_step_prop_logger_index);
+
+ fprintf(e->file_timesteps,
+ "# %6s %14s %12s %12s %14s %9s %12s %12s %12s %12s %12s %16s "
+ "[%s] %6s\n",
+ "Step", "Time", "Scale-factor", "Redshift", "Time-step",
+ "Time-bins", "Updates", "g-Updates", "s-Updates", "Sink-Updates",
+ "b-Updates", "Wall-clock time", clocks_getunit(), "Props");
+ fflush(e->file_timesteps);
+ }
+
+ /* Initialize the SFH logger if running with star formation */
+ if (e->policy & engine_policy_star_formation) {
+ e->sfh_logger = fopen("SFR.txt", mode);
+ if (!restart) {
+ star_formation_logger_init_log_file(e->sfh_logger, e->internal_units,
+ e->physical_constants);
+ fflush(e->sfh_logger);
+ }
+ }
+ }
+
+ /* Print policy */
+ engine_print_policy(e);
+
+ if (!fof) {
+
+ /* Print information about the hydro scheme */
+ if (e->policy & engine_policy_hydro) {
+ if (e->nodeID == 0) hydro_props_print(e->hydro_properties);
+ if (e->nodeID == 0) entropy_floor_print(e->entropy_floor);
+ }
+
+ /* Print information about the gravity scheme */
+ if (e->policy & engine_policy_self_gravity)
+ if (e->nodeID == 0) gravity_props_print(e->gravity_properties);
+
+ if (e->policy & engine_policy_stars)
+ if (e->nodeID == 0) stars_props_print(e->stars_properties);
+
+ /* Check we have sensible time bounds */
+ if (e->time_begin >= e->time_end)
+ error(
+ "Final simulation time (t_end = %e) must be larger than the start "
+ "time (t_beg = %e)",
+ e->time_end, e->time_begin);
+
+ /* Check we don't have inappropriate time labels */
+ if ((e->snapshot_int_time_label_on == 1) && (e->time_end <= 1.f))
+ error("Snapshot integer time labels enabled but end time <= 1");
+
+ /* Check we have sensible time-step values */
+ if (e->dt_min > e->dt_max)
+ error(
+ "Minimal time-step size (%e) must be smaller than maximal time-step "
+ "size (%e)",
+ e->dt_min, e->dt_max);
+
+ /* Info about time-steps */
+ if (e->nodeID == 0) {
+ message("Absolute minimal timestep size: %e", e->time_base);
+
+ float dt_min = e->time_end - e->time_begin;
+ while (dt_min > e->dt_min) dt_min /= 2.f;
+
+ message("Minimal timestep size (on time-line): %e", dt_min);
+
+ float dt_max = e->time_end - e->time_begin;
+ while (dt_max > e->dt_max) dt_max /= 2.f;
+
+ message("Maximal timestep size (on time-line): %e", dt_max);
+ }
+
+ if (e->dt_min < e->time_base && e->nodeID == 0)
+ error(
+ "Minimal time-step size smaller than the absolute possible minimum "
+ "dt=%e",
+ e->time_base);
+
+ if (!(e->policy & engine_policy_cosmology))
+ if (e->dt_max > (e->time_end - e->time_begin) && e->nodeID == 0)
+ error("Maximal time-step size larger than the simulation run time t=%e",
+ e->time_end - e->time_begin);
+
+ /* Deal with outputs */
+ if (e->policy & engine_policy_cosmology) {
+
+ if (e->delta_time_snapshot <= 1.)
+ error("Time between snapshots (%e) must be > 1.",
+ e->delta_time_snapshot);
+
+ if (e->delta_time_statistics <= 1.)
+ error("Time between statistics (%e) must be > 1.",
+ e->delta_time_statistics);
+
+ if (e->a_first_snapshot < e->cosmology->a_begin)
+ error(
+ "Scale-factor of first snapshot (%e) must be after the simulation "
+ "start a=%e.",
+ e->a_first_snapshot, e->cosmology->a_begin);
+
+ if (e->a_first_statistics < e->cosmology->a_begin)
+ error(
+ "Scale-factor of first stats output (%e) must be after the "
+ "simulation start a=%e.",
+ e->a_first_statistics, e->cosmology->a_begin);
+
+ if (e->policy & engine_policy_structure_finding) {
+
+ if (e->delta_time_stf == -1. && !e->snapshot_invoke_stf)
+ error("A value for `StructureFinding:delta_time` must be specified");
+
+ if (e->a_first_stf_output < e->cosmology->a_begin)
+ error(
+ "Scale-factor of first stf output (%e) must be after the "
+ "simulation start a=%e.",
+ e->a_first_stf_output, e->cosmology->a_begin);
+ }
+
+ if (e->policy & engine_policy_fof) {
+
+ if (e->delta_time_fof <= 1.)
+ error("Time between FOF (%e) must be > 1.", e->delta_time_fof);
+
+ if (e->a_first_fof_call < e->cosmology->a_begin)
+ error(
+ "Scale-factor of first fof call (%e) must be after the "
+ "simulation start a=%e.",
+ e->a_first_fof_call, e->cosmology->a_begin);
+ }
+
+ } else {
+
+ if (e->delta_time_snapshot <= 0.)
+ error("Time between snapshots (%e) must be positive.",
+ e->delta_time_snapshot);
+
+ if (e->delta_time_statistics <= 0.)
+ error("Time between statistics (%e) must be positive.",
+ e->delta_time_statistics);
+
+ /* Find the time of the first output */
+ if (e->time_first_snapshot < e->time_begin)
+ error(
+ "Time of first snapshot (%e) must be after the simulation start "
+ "t=%e.",
+ e->time_first_snapshot, e->time_begin);
+
+ if (e->time_first_statistics < e->time_begin)
+ error(
+ "Time of first stats output (%e) must be after the simulation "
+ "start t=%e.",
+ e->time_first_statistics, e->time_begin);
+
+ if (e->policy & engine_policy_structure_finding) {
+
+ if (e->delta_time_stf == -1. && !e->snapshot_invoke_stf)
+ error("A value for `StructureFinding:delta_time` must be specified");
+
+ if (e->delta_time_stf <= 0. && e->delta_time_stf != -1.)
+ error("Time between STF (%e) must be positive.", e->delta_time_stf);
+
+ if (e->time_first_stf_output < e->time_begin)
+ error(
+ "Time of first STF (%e) must be after the simulation start t=%e.",
+ e->time_first_stf_output, e->time_begin);
+ }
+ }
+
+ /* Try to ensure the snapshot directory exists */
+ if (e->nodeID == 0) io_make_snapshot_subdir(e->snapshot_subdir);
+
+ /* Get the total mass */
+ e->total_mass = 0.;
+ for (size_t i = 0; i < e->s->nr_gparts; ++i)
+ e->total_mass += e->s->gparts[i].mass;
+
+/* Reduce the total mass */
+#ifdef WITH_MPI
+ MPI_Allreduce(MPI_IN_PLACE, &e->total_mass, 1, MPI_DOUBLE, MPI_SUM,
+ MPI_COMM_WORLD);
+#endif
+
+#if defined(WITH_LOGGER)
+ if ((e->policy & engine_policy_logger) && e->nodeID == 0)
+ message(
+ "WARNING: There is currently no way of predicting the output "
+ "size, please use it carefully");
+#endif
+
+ /* Find the time of the first snapshot output */
+ engine_compute_next_snapshot_time(e);
+
+ /* Find the time of the first statistics output */
+ engine_compute_next_statistics_time(e);
+
+ /* Find the time of the first line of sight output */
+ if (e->policy & engine_policy_line_of_sight) {
+ engine_compute_next_los_time(e);
+ }
+
+ /* Find the time of the first stf output */
+ if (e->policy & engine_policy_structure_finding) {
+ engine_compute_next_stf_time(e);
+ }
+
+ /* Find the time of the first stf output */
+ if (e->policy & engine_policy_fof) {
+ engine_compute_next_fof_time(e);
+ }
+
+ /* Check that the snapshot naming policy is valid */
+ if (e->snapshot_invoke_stf && e->snapshot_int_time_label_on)
+ error(
+ "Cannot use snapshot time labels and VELOCIraptor invocations "
+ "together!");
+
+ /* Check that we are invoking VELOCIraptor only if we have it */
+ if (e->snapshot_invoke_stf &&
+ !(e->policy & engine_policy_structure_finding)) {
+ error(
+ "Invoking VELOCIraptor after snapshots but structure finding wasn't "
+ "activated at runtime (Use --velociraptor).");
+ }
+
+ /* Whether restarts are enabled. Yes by default. Can be changed on restart.
+ */
+ e->restart_dump = parser_get_opt_param_int(params, "Restarts:enable", 1);
+
+ /* Whether to save backup copies of the previous restart files. */
+ e->restart_save = parser_get_opt_param_int(params, "Restarts:save", 1);
+
+ /* Whether restarts should be dumped on exit. Not by default. Can be changed
+ * on restart. */
+ e->restart_onexit = parser_get_opt_param_int(params, "Restarts:onexit", 0);
+
+ /* Hours between restart dumps. Can be changed on restart. */
+ float dhours =
+ parser_get_opt_param_float(params, "Restarts:delta_hours", 5.0f);
+ if (e->nodeID == 0) {
+ if (e->restart_dump)
+ message("Restarts will be dumped every %f hours", dhours);
+ else
+ message("WARNING: restarts will not be dumped");
+
+ if (e->verbose && e->restart_onexit)
+ message("Restarts will be dumped after the final step");
+ }
+
+ /* Internally we use ticks, so convert into a delta ticks. Assumes we can
+ * convert from ticks into milliseconds. */
+ e->restart_dt = clocks_to_ticks(dhours * 60.0 * 60.0 * 1000.0);
+
+ /* The first dump will happen no sooner than restart_dt ticks in the
+ * future. */
+ e->restart_next = getticks() + e->restart_dt;
+ }
+
+/* Construct types for MPI communications */
+#ifdef WITH_MPI
+ part_create_mpi_types();
+ multipole_create_mpi_types();
+ stats_create_mpi_type();
+ proxy_create_mpi_type();
+ task_create_mpi_comms();
+#ifdef WITH_FOF
+ fof_create_mpi_types();
+#endif /* WITH_FOF */
+#endif /* WITH_MPI */
+
+ if (!fof) {
+
+ /* Initialise the collection group. */
+ collectgroup_init();
+ }
+
+ /* Initialize the threadpool. */
+ threadpool_init(&e->threadpool, e->nr_threads);
+
+ /* First of all, init the barrier and lock it. */
+ if (swift_barrier_init(&e->wait_barrier, NULL, e->nr_threads + 1) != 0 ||
+ swift_barrier_init(&e->run_barrier, NULL, e->nr_threads + 1) != 0)
+ error("Failed to initialize barrier.");
+
+ /* Expected average for tasks per cell. If set to zero we use a heuristic
+ * guess based on the numbers of cells and how many tasks per cell we expect.
+ * On restart this number cannot be estimated (no cells yet), so we recover
+ * from the end of the dumped run. Can be changed on restart. */
+ e->tasks_per_cell =
+ parser_get_opt_param_float(params, "Scheduler:tasks_per_cell", 0.0);
+ e->tasks_per_cell_max = 0.0f;
+
+ float maxtasks = 0;
+ if (restart)
+ maxtasks = e->restart_max_tasks;
+ else
+ maxtasks = engine_estimate_nr_tasks(e);
+
+ /* Estimated number of links per tasks */
+ e->links_per_tasks =
+ parser_get_opt_param_float(params, "Scheduler:links_per_tasks", 25.);
+
+ /* Init the scheduler. */
+ scheduler_init(&e->sched, e->s, maxtasks, nr_queues,
+ (e->policy & scheduler_flag_steal), e->nodeID, &e->threadpool);
+
+ /* Maximum size of MPI task messages, in KB, that should not be buffered,
+ * that is sent using MPI_Issend, not MPI_Isend. 4Mb by default. Can be
+ * changed on restart.
+ */
+ e->sched.mpi_message_limit =
+ parser_get_opt_param_int(params, "Scheduler:mpi_message_limit", 4) * 1024;
+
+ if (restart) {
+
+ /* Overwrite the constants for the scheduler */
+ space_maxsize = parser_get_opt_param_int(params, "Scheduler:cell_max_size",
+ space_maxsize);
+ space_subsize_pair_hydro = parser_get_opt_param_int(
+ params, "Scheduler:cell_sub_size_pair_hydro", space_subsize_pair_hydro);
+ space_subsize_self_hydro = parser_get_opt_param_int(
+ params, "Scheduler:cell_sub_size_self_hydro", space_subsize_self_hydro);
+ space_subsize_pair_stars = parser_get_opt_param_int(
+ params, "Scheduler:cell_sub_size_pair_stars", space_subsize_pair_stars);
+ space_subsize_self_stars = parser_get_opt_param_int(
+ params, "Scheduler:cell_sub_size_self_stars", space_subsize_self_stars);
+ space_subsize_pair_grav = parser_get_opt_param_int(
+ params, "Scheduler:cell_sub_size_pair_grav", space_subsize_pair_grav);
+ space_subsize_self_grav = parser_get_opt_param_int(
+ params, "Scheduler:cell_sub_size_self_grav", space_subsize_self_grav);
+ space_splitsize = parser_get_opt_param_int(
+ params, "Scheduler:cell_split_size", space_splitsize);
+ space_subdepth_diff_grav =
+ parser_get_opt_param_int(params, "Scheduler:cell_subdepth_diff_grav",
+ space_subdepth_diff_grav_default);
+ space_extra_parts = parser_get_opt_param_int(
+ params, "Scheduler:cell_extra_parts", space_extra_parts);
+ space_extra_sparts = parser_get_opt_param_int(
+ params, "Scheduler:cell_extra_sparts", space_extra_sparts);
+ space_extra_gparts = parser_get_opt_param_int(
+ params, "Scheduler:cell_extra_gparts", space_extra_gparts);
+ space_extra_bparts = parser_get_opt_param_int(
+ params, "Scheduler:cell_extra_bparts", space_extra_bparts);
+
+ engine_max_parts_per_ghost =
+ parser_get_opt_param_int(params, "Scheduler:engine_max_parts_per_ghost",
+ engine_max_parts_per_ghost);
+ engine_max_sparts_per_ghost = parser_get_opt_param_int(
+ params, "Scheduler:engine_max_sparts_per_ghost",
+ engine_max_sparts_per_ghost);
+
+ engine_max_parts_per_cooling = parser_get_opt_param_int(
+ params, "Scheduler:engine_max_parts_per_cooling",
+ engine_max_parts_per_cooling);
+ }
+
+ /* Allocate and init the threads. */
+ if (swift_memalign("runners", (void **)&e->runners, SWIFT_CACHE_ALIGNMENT,
+ e->nr_threads * sizeof(struct runner)) != 0)
+ error("Failed to allocate threads array.");
+
+ for (int k = 0; k < e->nr_threads; k++) {
+ e->runners[k].id = k;
+ e->runners[k].e = e;
+ if (pthread_create(&e->runners[k].thread, NULL, &runner_main,
+ &e->runners[k]) != 0)
+ error("Failed to create runner thread.");
+
+ /* Try to pin the runner to a given core */
+ if (with_aff &&
+ (e->policy & engine_policy_setaffinity) == engine_policy_setaffinity) {
+#if defined(HAVE_SETAFFINITY)
+
+ /* Set a reasonable queue ID. */
+ int coreid = k % nr_affinity_cores;
+ e->runners[k].cpuid = cpuid[coreid];
+
+ if (nr_queues < e->nr_threads)
+ e->runners[k].qid = cpuid[coreid] * nr_queues / nr_affinity_cores;
+ else
+ e->runners[k].qid = k;
+
+ /* Set the cpu mask to zero | e->id. */
+ CPU_ZERO(&cpuset);
+ CPU_SET(cpuid[coreid], &cpuset);
+
+ /* Apply this mask to the runner's pthread. */
+ if (pthread_setaffinity_np(e->runners[k].thread, sizeof(cpu_set_t),
+ &cpuset) != 0)
+ error("Failed to set thread affinity.");
+
+#else
+ error("SWIFT was not compiled with affinity enabled.");
+#endif
+ } else {
+ e->runners[k].cpuid = k;
+ e->runners[k].qid = k * nr_queues / e->nr_threads;
+ }
+
+ /* Allocate particle caches. */
+ e->runners[k].ci_gravity_cache.count = 0;
+ e->runners[k].cj_gravity_cache.count = 0;
+ gravity_cache_init(&e->runners[k].ci_gravity_cache, space_splitsize);
+ gravity_cache_init(&e->runners[k].cj_gravity_cache, space_splitsize);
+#ifdef WITH_VECTORIZATION
+ e->runners[k].ci_cache.count = 0;
+ e->runners[k].cj_cache.count = 0;
+ cache_init(&e->runners[k].ci_cache, CACHE_SIZE);
+ cache_init(&e->runners[k].cj_cache, CACHE_SIZE);
+#endif
+
+ if (verbose) {
+ if (with_aff)
+ message("runner %i on cpuid=%i with qid=%i.", e->runners[k].id,
+ e->runners[k].cpuid, e->runners[k].qid);
+ else
+ message("runner %i using qid=%i no cpuid.", e->runners[k].id,
+ e->runners[k].qid);
+ }
+ }
+
+#ifdef WITH_LOGGER
+ if ((e->policy & engine_policy_logger) && !restart) {
+ /* Write the particle logger header */
+ logger_write_file_header(e->logger);
+ }
+#endif
+
+ /* Initialise the structure finder */
+#ifdef HAVE_VELOCIRAPTOR
+ if (e->policy & engine_policy_structure_finding) velociraptor_init(e);
+#endif
+
+ /* Free the affinity stuff */
+#if defined(HAVE_SETAFFINITY)
+ if (with_aff) {
+ free(cpuid);
+ }
+#endif
+
+#ifdef SWIFT_DUMPER_THREAD
+
+ /* Start the dumper thread.*/
+ engine_dumper_init(e);
+#endif
+
+ /* Wait for the runner threads to be in place. */
+ swift_barrier_wait(&e->wait_barrier);
+}
diff --git a/src/engine_io.c b/src/engine_io.c
new file mode 100644
index 0000000000000000000000000000000000000000..7d961a96374fa4bec8d0ffaea5ea6a90bbd779aa
--- /dev/null
+++ b/src/engine_io.c
@@ -0,0 +1,868 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* MPI headers. */
+#ifdef WITH_MPI
+#include <mpi.h>
+#endif
+
+/* This object's header. */
+#include "engine.h"
+
+/* Local headers. */
+#include "distributed_io.h"
+#include "kick.h"
+#include "line_of_sight.h"
+#include "logger_io.h"
+#include "parallel_io.h"
+#include "serial_io.h"
+#include "single_io.h"
+
+/**
+ * @brief Check whether an index file has to be written during this
+ * step.
+ *
+ * @param e The #engine.
+ */
+void engine_check_for_index_dump(struct engine *e) {
+#ifdef WITH_LOGGER
+ /* Get a few variables */
+ struct logger_writer *log = e->logger;
+ const size_t dump_size = log->dump.count;
+ const size_t old_dump_size = log->index.dump_size_last_output;
+ const float mem_frac = log->index.mem_frac;
+ const size_t total_nr_parts =
+ (e->total_nr_parts + e->total_nr_gparts + e->total_nr_sparts +
+ e->total_nr_bparts + e->total_nr_DM_background_gparts);
+ const size_t index_file_size =
+ total_nr_parts * sizeof(struct logger_part_data);
+
+ /* Check if we should write a file */
+ if (mem_frac * (dump_size - old_dump_size) > index_file_size) {
+ /* Write an index file */
+ engine_dump_index(e);
+
+ /* Update the dump size for last output */
+ log->index.dump_size_last_output = dump_size;
+ }
+#else
+ error("This function should not be called without the logger.");
+#endif
+}
+
+/**
+ * @brief dump restart files if it is time to do so and dumps are enabled.
+ *
+ * @param e the engine.
+ * @param drifted_all true if a drift_all has just been performed.
+ * @param force force a dump, if dumping is enabled.
+ */
+void engine_dump_restarts(struct engine *e, int drifted_all, int force) {
+
+ if (e->restart_dump) {
+ ticks tic = getticks();
+
+ /* Dump when the time has arrived, or we are told to. */
+ int dump = ((tic > e->restart_next) || force);
+
+#ifdef WITH_MPI
+ /* Synchronize this action from rank 0 (ticks may differ between
+ * machines). */
+ MPI_Bcast(&dump, 1, MPI_INT, 0, MPI_COMM_WORLD);
+#endif
+ if (dump) {
+
+ if (e->nodeID == 0) message("Writing restart files");
+
+ /* Clean out the previous saved files, if found. Do this now as we are
+ * MPI synchronized. */
+ restart_remove_previous(e->restart_file);
+
+ /* Drift all particles first (may have just been done). */
+ if (!drifted_all) engine_drift_all(e, /*drift_mpole=*/1);
+ restart_write(e, e->restart_file);
+
+#ifdef WITH_MPI
+ /* Make sure all ranks finished writing to avoid having incomplete
+ * sets of restart files should the code crash before all the ranks
+ * are done */
+ MPI_Barrier(MPI_COMM_WORLD);
+#endif
+
+ if (e->verbose)
+ message("Dumping restart files took %.3f %s",
+ clocks_from_ticks(getticks() - tic), clocks_getunit());
+
+ /* Time after which next dump will occur. */
+ e->restart_next += e->restart_dt;
+
+ /* Flag that we dumped the restarts */
+ e->step_props |= engine_step_prop_restarts;
+ }
+ }
+}
+
+/**
+ * @brief Writes a snapshot with the current state of the engine
+ *
+ * @param e The #engine.
+ */
+void engine_dump_snapshot(struct engine *e) {
+
+ struct clocks_time time1, time2;
+ clocks_gettime(&time1);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Check that all cells have been drifted to the current time.
+ * That can include cells that have not
+ * previously been active on this rank. */
+ space_check_drift_point(e->s, e->ti_current, /* check_mpole=*/0);
+
+ /* Be verbose about this */
+ if (e->nodeID == 0) {
+ if (e->policy & engine_policy_cosmology)
+ message("Dumping snapshot at a=%e",
+ exp(e->ti_current * e->time_base) * e->cosmology->a_begin);
+ else
+ message("Dumping snapshot at t=%e",
+ e->ti_current * e->time_base + e->time_begin);
+ }
+#else
+ if (e->verbose) {
+ if (e->policy & engine_policy_cosmology)
+ message("Dumping snapshot at a=%e",
+ exp(e->ti_current * e->time_base) * e->cosmology->a_begin);
+ else
+ message("Dumping snapshot at t=%e",
+ e->ti_current * e->time_base + e->time_begin);
+ }
+#endif
+
+#ifdef DEBUG_INTERACTIONS_STARS
+ engine_collect_stars_counter(e);
+#endif
+
+ /* Get time-step since the last mesh kick */
+ if ((e->policy & engine_policy_self_gravity) && e->s->periodic) {
+ const int with_cosmology = e->policy & engine_policy_cosmology;
+
+ e->dt_kick_grav_mesh_for_io =
+ kick_get_grav_kick_dt(e->mesh->ti_beg_mesh_next, e->ti_current,
+ e->time_base, with_cosmology, e->cosmology) -
+ kick_get_grav_kick_dt(
+ e->mesh->ti_beg_mesh_next,
+ (e->mesh->ti_beg_mesh_next + e->mesh->ti_end_mesh_next) / 2,
+ e->time_base, with_cosmology, e->cosmology);
+ }
+
+/* Dump (depending on the chosen strategy) ... */
+#if defined(HAVE_HDF5)
+#if defined(WITH_MPI)
+
+ if (e->snapshot_distributed) {
+
+ write_output_distributed(e, e->internal_units, e->snapshot_units, e->nodeID,
+ e->nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL);
+ } else {
+
+#if defined(HAVE_PARALLEL_HDF5)
+ write_output_parallel(e, e->internal_units, e->snapshot_units, e->nodeID,
+ e->nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL);
+#else
+ write_output_serial(e, e->internal_units, e->snapshot_units, e->nodeID,
+ e->nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL);
+#endif
+ }
+#else
+ write_output_single(e, e->internal_units, e->snapshot_units);
+#endif
+#endif
+
+ /* Flag that we dumped a snapshot */
+ e->step_props |= engine_step_prop_snapshot;
+
+ clocks_gettime(&time2);
+ if (e->verbose)
+ message("writing particle properties took %.3f %s.",
+ (float)clocks_diff(&time1, &time2), clocks_getunit());
+}
+
+/**
+ * @brief Writes an index file with the current state of the engine
+ *
+ * @param e The #engine.
+ */
+void engine_dump_index(struct engine *e) {
+
+#if defined(WITH_LOGGER)
+ struct clocks_time time1, time2;
+ clocks_gettime(&time1);
+
+ if (e->verbose) {
+ if (e->policy & engine_policy_cosmology)
+ message("Writing index at a=%e",
+ exp(e->ti_current * e->time_base) * e->cosmology->a_begin);
+ else
+ message("Writing index at t=%e",
+ e->ti_current * e->time_base + e->time_begin);
+ }
+
+ /* Dump... */
+ logger_write_index_file(e->logger, e);
+
+ /* Flag that we dumped a snapshot */
+ e->step_props |= engine_step_prop_logger_index;
+
+ clocks_gettime(&time2);
+ if (e->verbose)
+ message("writing particle indices took %.3f %s.",
+ (float)clocks_diff(&time1, &time2), clocks_getunit());
+#else
+ error("SWIFT was not compiled with the logger");
+#endif
+}
+
+/**
+ * @brief Check whether any kind of i/o has to be performed during this
+ * step.
+ *
+ * This includes snapshots, stats and halo finder. We also handle the case
+ * of multiple outputs between two steps.
+ *
+ * @param e The #engine.
+ */
+void engine_check_for_dumps(struct engine *e) {
+ const int with_cosmology = (e->policy & engine_policy_cosmology);
+ const int with_stf = (e->policy & engine_policy_structure_finding);
+ const int with_los = (e->policy & engine_policy_line_of_sight);
+ const int with_fof = (e->policy & engine_policy_fof);
+
+ /* What kind of output are we getting? */
+ enum output_type {
+ output_none,
+ output_snapshot,
+ output_statistics,
+ output_stf,
+ output_los,
+ };
+
+ /* What kind of output do we want? And at which time ?
+ * Find the earliest output (amongst all kinds) that takes place
+ * before the next time-step */
+ enum output_type type = output_none;
+ integertime_t ti_output = max_nr_timesteps;
+ e->stf_this_timestep = 0;
+
+ /* Save some statistics ? */
+ if (e->ti_end_min > e->ti_next_stats && e->ti_next_stats > 0) {
+ if (e->ti_next_stats < ti_output) {
+ ti_output = e->ti_next_stats;
+ type = output_statistics;
+ }
+ }
+
+ /* Do we want a snapshot? */
+ if (e->ti_end_min > e->ti_next_snapshot && e->ti_next_snapshot > 0) {
+ if (e->ti_next_snapshot < ti_output) {
+ ti_output = e->ti_next_snapshot;
+ type = output_snapshot;
+ }
+ }
+
+ /* Do we want to perform structure finding? */
+ if (with_stf) {
+ if (e->ti_end_min > e->ti_next_stf && e->ti_next_stf > 0) {
+ if (e->ti_next_stf < ti_output) {
+ ti_output = e->ti_next_stf;
+ type = output_stf;
+ }
+ }
+ }
+
+ /* Do we want to write a line of sight file? */
+ if (with_los) {
+ if (e->ti_end_min > e->ti_next_los && e->ti_next_los > 0) {
+ if (e->ti_next_los < ti_output) {
+ ti_output = e->ti_next_los;
+ type = output_los;
+ }
+ }
+ }
+
+ /* Store information before attempting extra dump-related drifts */
+ const integertime_t ti_current = e->ti_current;
+ const timebin_t max_active_bin = e->max_active_bin;
+ const double time = e->time;
+
+ while (type != output_none) {
+
+ /* Let's fake that we are at the dump time */
+ e->ti_current = ti_output;
+ e->max_active_bin = 0;
+ if (with_cosmology) {
+ cosmology_update(e->cosmology, e->physical_constants, e->ti_current);
+ e->time = e->cosmology->time;
+ } else {
+ e->time = ti_output * e->time_base + e->time_begin;
+ }
+
+ /* Drift everyone */
+ engine_drift_all(e, /*drift_mpole=*/0);
+
+ /* Write some form of output */
+ switch (type) {
+
+ case output_snapshot:
+
+#ifdef SWIFT_GRAVITY_FORCE_CHECKS
+ /* Indicate we are allowed to do a brute force calculation now */
+ e->force_checks_snapshot_flag = 1;
+#endif
+
+ /* Do we want FoF group IDs in the snapshot? */
+ if(with_fof && e->snapshot_invoke_fof) {
+ engine_fof(e, /*dump_results=*/0, /*seed_black_holes=*/0);
+ }
+
+ /* Do we want a corresponding VELOCIraptor output? */
+ if (with_stf && e->snapshot_invoke_stf && !e->stf_this_timestep) {
+
+#ifdef HAVE_VELOCIRAPTOR
+ velociraptor_invoke(e, /*linked_with_snap=*/1);
+ e->step_props |= engine_step_prop_stf;
+#else
+ error(
+ "Asking for a VELOCIraptor output but SWIFT was compiled without "
+ "the interface!");
+#endif
+ }
+
+ /* Dump... */
+ engine_dump_snapshot(e);
+
+ /* Free the memory allocated for VELOCIraptor i/o. */
+ if (with_stf && e->snapshot_invoke_stf && e->s->gpart_group_data) {
+#ifdef HAVE_VELOCIRAPTOR
+ swift_free("gpart_group_data", e->s->gpart_group_data);
+ e->s->gpart_group_data = NULL;
+#endif
+ }
+
+ /* ... and find the next output time */
+ engine_compute_next_snapshot_time(e);
+ break;
+
+ case output_statistics:
+
+ /* Dump */
+ engine_print_stats(e);
+
+ /* and move on */
+ engine_compute_next_statistics_time(e);
+
+ break;
+
+ case output_stf:
+
+#ifdef HAVE_VELOCIRAPTOR
+ /* Unleash the raptor! */
+ if (!e->stf_this_timestep) {
+ velociraptor_invoke(e, /*linked_with_snap=*/0);
+ e->step_props |= engine_step_prop_stf;
+ }
+
+ /* ... and find the next output time */
+ engine_compute_next_stf_time(e);
+#else
+ error(
+ "Asking for a VELOCIraptor output but SWIFT was compiled without "
+ "the interface!");
+#endif
+ break;
+
+ case output_los:
+
+ /* Compute the LoS */
+ do_line_of_sight(e);
+
+ /* Move on */
+ engine_compute_next_los_time(e);
+
+ break;
+
+ default:
+ error("Invalid dump type");
+ }
+
+ /* We need to see whether whether we are in the pathological case
+ * where there can be another dump before the next step. */
+
+ type = output_none;
+ ti_output = max_nr_timesteps;
+
+ /* Save some statistics ? */
+ if (e->ti_end_min > e->ti_next_stats && e->ti_next_stats > 0) {
+ if (e->ti_next_stats < ti_output) {
+ ti_output = e->ti_next_stats;
+ type = output_statistics;
+ }
+ }
+
+ /* Do we want a snapshot? */
+ if (e->ti_end_min > e->ti_next_snapshot && e->ti_next_snapshot > 0) {
+ if (e->ti_next_snapshot < ti_output) {
+ ti_output = e->ti_next_snapshot;
+ type = output_snapshot;
+ }
+ }
+
+ /* Do we want to perform structure finding? */
+ if (with_stf) {
+ if (e->ti_end_min > e->ti_next_stf && e->ti_next_stf > 0) {
+ if (e->ti_next_stf < ti_output) {
+ ti_output = e->ti_next_stf;
+ type = output_stf;
+ }
+ }
+ }
+
+ /* Do line of sight ? */
+ if (with_los) {
+ if (e->ti_end_min > e->ti_next_los && e->ti_next_los > 0) {
+ if (e->ti_next_los < ti_output) {
+ ti_output = e->ti_next_los;
+ type = output_los;
+ }
+ }
+ }
+
+ } /* While loop over output types */
+
+ /* Restore the information we stored */
+ e->ti_current = ti_current;
+ if (e->policy & engine_policy_cosmology)
+ cosmology_update(e->cosmology, e->physical_constants, e->ti_current);
+ e->max_active_bin = max_active_bin;
+ e->time = time;
+}
+
+/**
+ * @brief Computes the next time (on the time line) for a dump
+ *
+ * @param e The #engine.
+ */
+void engine_compute_next_snapshot_time(struct engine *e) {
+
+ /* Do output_list file case */
+ if (e->output_list_snapshots) {
+ output_list_read_next_time(e->output_list_snapshots, e, "snapshots",
+ &e->ti_next_snapshot);
+ return;
+ }
+
+ /* Find upper-bound on last output */
+ double time_end;
+ if (e->policy & engine_policy_cosmology)
+ time_end = e->cosmology->a_end * e->delta_time_snapshot;
+ else
+ time_end = e->time_end + e->delta_time_snapshot;
+
+ /* Find next snasphot above current time */
+ double time;
+ if (e->policy & engine_policy_cosmology)
+ time = e->a_first_snapshot;
+ else
+ time = e->time_first_snapshot;
+
+ int found_snapshot_time = 0;
+ while (time < time_end) {
+
+ /* Output time on the integer timeline */
+ if (e->policy & engine_policy_cosmology)
+ e->ti_next_snapshot = log(time / e->cosmology->a_begin) / e->time_base;
+ else
+ e->ti_next_snapshot = (time - e->time_begin) / e->time_base;
+
+ /* Found it? */
+ if (e->ti_next_snapshot > e->ti_current) {
+ found_snapshot_time = 1;
+ break;
+ }
+
+ if (e->policy & engine_policy_cosmology)
+ time *= e->delta_time_snapshot;
+ else
+ time += e->delta_time_snapshot;
+ }
+
+ /* Deal with last snapshot */
+ if (!found_snapshot_time) {
+ e->ti_next_snapshot = -1;
+ if (e->verbose) message("No further output time.");
+ } else {
+
+ /* Be nice, talk... */
+ if (e->policy & engine_policy_cosmology) {
+ const double next_snapshot_time =
+ exp(e->ti_next_snapshot * e->time_base) * e->cosmology->a_begin;
+ if (e->verbose)
+ message("Next snapshot time set to a=%e.", next_snapshot_time);
+ } else {
+ const double next_snapshot_time =
+ e->ti_next_snapshot * e->time_base + e->time_begin;
+ if (e->verbose)
+ message("Next snapshot time set to t=%e.", next_snapshot_time);
+ }
+ }
+}
+
+/**
+ * @brief Computes the next time (on the time line) for a statistics dump
+ *
+ * @param e The #engine.
+ */
+void engine_compute_next_statistics_time(struct engine *e) {
+ /* Do output_list file case */
+ if (e->output_list_stats) {
+ output_list_read_next_time(e->output_list_stats, e, "stats",
+ &e->ti_next_stats);
+ return;
+ }
+
+ /* Find upper-bound on last output */
+ double time_end;
+ if (e->policy & engine_policy_cosmology)
+ time_end = e->cosmology->a_end * e->delta_time_statistics;
+ else
+ time_end = e->time_end + e->delta_time_statistics;
+
+ /* Find next snasphot above current time */
+ double time;
+ if (e->policy & engine_policy_cosmology)
+ time = e->a_first_statistics;
+ else
+ time = e->time_first_statistics;
+
+ int found_stats_time = 0;
+ while (time < time_end) {
+
+ /* Output time on the integer timeline */
+ if (e->policy & engine_policy_cosmology)
+ e->ti_next_stats = log(time / e->cosmology->a_begin) / e->time_base;
+ else
+ e->ti_next_stats = (time - e->time_begin) / e->time_base;
+
+ /* Found it? */
+ if (e->ti_next_stats > e->ti_current) {
+ found_stats_time = 1;
+ break;
+ }
+
+ if (e->policy & engine_policy_cosmology)
+ time *= e->delta_time_statistics;
+ else
+ time += e->delta_time_statistics;
+ }
+
+ /* Deal with last statistics */
+ if (!found_stats_time) {
+ e->ti_next_stats = -1;
+ if (e->verbose) message("No further output time.");
+ } else {
+
+ /* Be nice, talk... */
+ if (e->policy & engine_policy_cosmology) {
+ const double next_statistics_time =
+ exp(e->ti_next_stats * e->time_base) * e->cosmology->a_begin;
+ if (e->verbose)
+ message("Next output time for stats set to a=%e.",
+ next_statistics_time);
+ } else {
+ const double next_statistics_time =
+ e->ti_next_stats * e->time_base + e->time_begin;
+ if (e->verbose)
+ message("Next output time for stats set to t=%e.",
+ next_statistics_time);
+ }
+ }
+}
+
+/**
+ * @brief Computes the next time (on the time line) for a line of sight dump
+ *
+ * @param e The #engine.
+ */
+void engine_compute_next_los_time(struct engine *e) {
+ /* Do output_list file case */
+ if (e->output_list_los) {
+ output_list_read_next_time(e->output_list_los, e, "line of sights",
+ &e->ti_next_los);
+ return;
+ }
+
+ /* Find upper-bound on last output */
+ double time_end;
+ if (e->policy & engine_policy_cosmology)
+ time_end = e->cosmology->a_end * e->delta_time_los;
+ else
+ time_end = e->time_end + e->delta_time_los;
+
+ /* Find next los above current time */
+ double time;
+ if (e->policy & engine_policy_cosmology)
+ time = e->a_first_los;
+ else
+ time = e->time_first_los;
+
+ int found_los_time = 0;
+ while (time < time_end) {
+
+ /* Output time on the integer timeline */
+ if (e->policy & engine_policy_cosmology)
+ e->ti_next_los = log(time / e->cosmology->a_begin) / e->time_base;
+ else
+ e->ti_next_los = (time - e->time_begin) / e->time_base;
+
+ /* Found it? */
+ if (e->ti_next_los > e->ti_current) {
+ found_los_time = 1;
+ break;
+ }
+
+ if (e->policy & engine_policy_cosmology)
+ time *= e->delta_time_los;
+ else
+ time += e->delta_time_los;
+ }
+
+ /* Deal with last line of sight */
+ if (!found_los_time) {
+ e->ti_next_los = -1;
+ if (e->verbose) message("No further LOS output time.");
+ } else {
+
+ /* Be nice, talk... */
+ if (e->policy & engine_policy_cosmology) {
+ const double next_los_time =
+ exp(e->ti_next_los * e->time_base) * e->cosmology->a_begin;
+ if (e->verbose)
+ message("Next output time for line of sight set to a=%e.",
+ next_los_time);
+ } else {
+ const double next_los_time =
+ e->ti_next_los * e->time_base + e->time_begin;
+ if (e->verbose)
+ message("Next output time for line of sight set to t=%e.",
+ next_los_time);
+ }
+ }
+}
+
+/**
+ * @brief Computes the next time (on the time line) for structure finding
+ *
+ * @param e The #engine.
+ */
+void engine_compute_next_stf_time(struct engine *e) {
+ /* Do output_list file case */
+ if (e->output_list_stf) {
+ output_list_read_next_time(e->output_list_stf, e, "stf", &e->ti_next_stf);
+ return;
+ }
+
+ /* Find upper-bound on last output */
+ double time_end;
+ if (e->policy & engine_policy_cosmology)
+ time_end = e->cosmology->a_end * e->delta_time_stf;
+ else
+ time_end = e->time_end + e->delta_time_stf;
+
+ /* Find next snasphot above current time */
+ double time;
+ if (e->policy & engine_policy_cosmology)
+ time = e->a_first_stf_output;
+ else
+ time = e->time_first_stf_output;
+
+ int found_stf_time = 0;
+ while (time < time_end) {
+
+ /* Output time on the integer timeline */
+ if (e->policy & engine_policy_cosmology)
+ e->ti_next_stf = log(time / e->cosmology->a_begin) / e->time_base;
+ else
+ e->ti_next_stf = (time - e->time_begin) / e->time_base;
+
+ /* Found it? */
+ if (e->ti_next_stf > e->ti_current) {
+ found_stf_time = 1;
+ break;
+ }
+
+ if (e->policy & engine_policy_cosmology)
+ time *= e->delta_time_stf;
+ else
+ time += e->delta_time_stf;
+ }
+
+ /* Deal with last snapshot */
+ if (!found_stf_time) {
+ e->ti_next_stf = -1;
+ if (e->verbose) message("No further output time.");
+ } else {
+
+ /* Be nice, talk... */
+ if (e->policy & engine_policy_cosmology) {
+ const float next_stf_time =
+ exp(e->ti_next_stf * e->time_base) * e->cosmology->a_begin;
+ if (e->verbose)
+ message("Next VELOCIraptor time set to a=%e.", next_stf_time);
+ } else {
+ const float next_stf_time = e->ti_next_stf * e->time_base + e->time_begin;
+ if (e->verbose)
+ message("Next VELOCIraptor time set to t=%e.", next_stf_time);
+ }
+ }
+}
+
+/**
+ * @brief Computes the next time (on the time line) for FoF black holes seeding
+ *
+ * @param e The #engine.
+ */
+void engine_compute_next_fof_time(struct engine *e) {
+
+ /* Find upper-bound on last output */
+ double time_end;
+ if (e->policy & engine_policy_cosmology)
+ time_end = e->cosmology->a_end * e->delta_time_fof;
+ else
+ time_end = e->time_end + e->delta_time_fof;
+
+ /* Find next snasphot above current time */
+ double time;
+ if (e->policy & engine_policy_cosmology)
+ time = e->a_first_fof_call;
+ else
+ time = e->time_first_fof_call;
+
+ int found_fof_time = 0;
+ while (time < time_end) {
+
+ /* Output time on the integer timeline */
+ if (e->policy & engine_policy_cosmology)
+ e->ti_next_fof = log(time / e->cosmology->a_begin) / e->time_base;
+ else
+ e->ti_next_fof = (time - e->time_begin) / e->time_base;
+
+ /* Found it? */
+ if (e->ti_next_fof > e->ti_current) {
+ found_fof_time = 1;
+ break;
+ }
+
+ if (e->policy & engine_policy_cosmology)
+ time *= e->delta_time_fof;
+ else
+ time += e->delta_time_fof;
+ }
+
+ /* Deal with last snapshot */
+ if (!found_fof_time) {
+ e->ti_next_fof = -1;
+ if (e->verbose) message("No further FoF time.");
+ } else {
+
+ /* Be nice, talk... */
+ if (e->policy & engine_policy_cosmology) {
+ const float next_fof_time =
+ exp(e->ti_next_fof * e->time_base) * e->cosmology->a_begin;
+ // if (e->verbose)
+ message("Next FoF time set to a=%e.", next_fof_time);
+ } else {
+ const float next_fof_time = e->ti_next_fof * e->time_base + e->time_begin;
+ if (e->verbose) message("Next FoF time set to t=%e.", next_fof_time);
+ }
+ }
+}
+
+/**
+ * @brief Initialize all the output_list required by the engine
+ *
+ * @param e The #engine.
+ * @param params The #swift_params.
+ */
+void engine_init_output_lists(struct engine *e, struct swift_params *params) {
+ /* Deal with snapshots */
+ double snaps_time_first;
+ e->output_list_snapshots = NULL;
+ output_list_init(&e->output_list_snapshots, e, "Snapshots",
+ &e->delta_time_snapshot, &snaps_time_first);
+
+ if (e->output_list_snapshots) {
+ if (e->policy & engine_policy_cosmology)
+ e->a_first_snapshot = snaps_time_first;
+ else
+ e->time_first_snapshot = snaps_time_first;
+ }
+
+ /* Deal with stats */
+ double stats_time_first;
+ e->output_list_stats = NULL;
+ output_list_init(&e->output_list_stats, e, "Statistics",
+ &e->delta_time_statistics, &stats_time_first);
+
+ if (e->output_list_stats) {
+ if (e->policy & engine_policy_cosmology)
+ e->a_first_statistics = stats_time_first;
+ else
+ e->time_first_statistics = stats_time_first;
+ }
+
+ /* Deal with stf */
+ double stf_time_first;
+ e->output_list_stf = NULL;
+ output_list_init(&e->output_list_stf, e, "StructureFinding",
+ &e->delta_time_stf, &stf_time_first);
+
+ if (e->output_list_stf) {
+ if (e->policy & engine_policy_cosmology)
+ e->a_first_stf_output = stf_time_first;
+ else
+ e->time_first_stf_output = stf_time_first;
+ }
+
+ /* Deal with line of sight */
+ double los_time_first;
+ e->output_list_los = NULL;
+ output_list_init(&e->output_list_los, e, "LineOfSight", &e->delta_time_los,
+ &los_time_first);
+
+ if (e->output_list_los) {
+ if (e->policy & engine_policy_cosmology)
+ e->a_first_los = los_time_first;
+ else
+ e->time_first_los = los_time_first;
+ }
+}
diff --git a/src/engine_proxy.c b/src/engine_proxy.c
new file mode 100644
index 0000000000000000000000000000000000000000..2fb1d3e75d597f3d4aa1a09ffc67cf453d50228b
--- /dev/null
+++ b/src/engine_proxy.c
@@ -0,0 +1,306 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* MPI headers. */
+#ifdef WITH_MPI
+#include <mpi.h>
+#endif
+
+/* This object's header. */
+#include "engine.h"
+
+/* Local headers. */
+#include "proxy.h"
+
+/**
+ * @brief Create and fill the proxies.
+ *
+ * @param e The #engine.
+ */
+void engine_makeproxies(struct engine *e) {
+
+#ifdef WITH_MPI
+ /* Let's time this */
+ const ticks tic = getticks();
+
+ /* Useful local information */
+ const int nodeID = e->nodeID;
+ const struct space *s = e->s;
+
+ /* Handle on the cells and proxies */
+ struct cell *cells = s->cells_top;
+ struct proxy *proxies = e->proxies;
+
+ /* Some info about the domain */
+ const int cdim[3] = {s->cdim[0], s->cdim[1], s->cdim[2]};
+ const double dim[3] = {s->dim[0], s->dim[1], s->dim[2]};
+ const int periodic = s->periodic;
+ const double cell_width[3] = {cells[0].width[0], cells[0].width[1],
+ cells[0].width[2]};
+
+ /* Get some info about the physics */
+ const int with_hydro = (e->policy & engine_policy_hydro);
+ const int with_gravity = (e->policy & engine_policy_self_gravity);
+ const double theta_crit = e->gravity_properties->theta_crit;
+ const double theta_crit_inv = 1. / e->gravity_properties->theta_crit;
+ const double max_mesh_dist = e->mesh->r_cut_max;
+ const double max_mesh_dist2 = max_mesh_dist * max_mesh_dist;
+
+ /* Distance between centre of the cell and corners */
+ const double r_diag2 = cell_width[0] * cell_width[0] +
+ cell_width[1] * cell_width[1] +
+ cell_width[2] * cell_width[2];
+ const double r_diag = 0.5 * sqrt(r_diag2);
+
+ /* Maximal distance from shifted CoM to any corner */
+ const double r_max = 2 * r_diag;
+
+ /* Prepare the proxies and the proxy index. */
+ if (e->proxy_ind == NULL)
+ if ((e->proxy_ind = (int *)malloc(sizeof(int) * e->nr_nodes)) == NULL)
+ error("Failed to allocate proxy index.");
+ for (int k = 0; k < e->nr_nodes; k++) e->proxy_ind[k] = -1;
+ e->nr_proxies = 0;
+
+ /* Compute how many cells away we need to walk */
+ int delta_cells = 1; /*hydro case */
+
+ /* Gravity needs to take the opening angle into account */
+ if (with_gravity) {
+ const double distance = 2. * r_max * theta_crit_inv;
+ delta_cells = (int)(distance / cells[0].dmin) + 1;
+ }
+
+ /* Turn this into upper and lower bounds for loops */
+ int delta_m = delta_cells;
+ int delta_p = delta_cells;
+
+ /* Special case where every cell is in range of every other one */
+ if (delta_cells >= cdim[0] / 2) {
+ if (cdim[0] % 2 == 0) {
+ delta_m = cdim[0] / 2;
+ delta_p = cdim[0] / 2 - 1;
+ } else {
+ delta_m = cdim[0] / 2;
+ delta_p = cdim[0] / 2;
+ }
+ }
+
+ /* Let's be verbose about this choice */
+ if (e->verbose)
+ message(
+ "Looking for proxies up to %d top-level cells away (delta_m=%d "
+ "delta_p=%d)",
+ delta_cells, delta_m, delta_p);
+
+ /* Loop over each cell in the space. */
+ for (int i = 0; i < cdim[0]; i++) {
+ for (int j = 0; j < cdim[1]; j++) {
+ for (int k = 0; k < cdim[2]; k++) {
+
+ /* Get the cell ID. */
+ const int cid = cell_getid(cdim, i, j, k);
+
+ /* Loop over all its neighbours neighbours in range. */
+ for (int ii = -delta_m; ii <= delta_p; ii++) {
+ int iii = i + ii;
+ if (!periodic && (iii < 0 || iii >= cdim[0])) continue;
+ iii = (iii + cdim[0]) % cdim[0];
+ for (int jj = -delta_m; jj <= delta_p; jj++) {
+ int jjj = j + jj;
+ if (!periodic && (jjj < 0 || jjj >= cdim[1])) continue;
+ jjj = (jjj + cdim[1]) % cdim[1];
+ for (int kk = -delta_m; kk <= delta_p; kk++) {
+ int kkk = k + kk;
+ if (!periodic && (kkk < 0 || kkk >= cdim[2])) continue;
+ kkk = (kkk + cdim[2]) % cdim[2];
+
+ /* Get the cell ID. */
+ const int cjd = cell_getid(cdim, iii, jjj, kkk);
+
+ /* Early abort */
+ if (cid >= cjd) continue;
+
+ /* Early abort (both same node) */
+ if (cells[cid].nodeID == nodeID && cells[cjd].nodeID == nodeID)
+ continue;
+
+ /* Early abort (both foreign node) */
+ if (cells[cid].nodeID != nodeID && cells[cjd].nodeID != nodeID)
+ continue;
+
+ int proxy_type = 0;
+
+ /* In the hydro case, only care about direct neighbours */
+ if (with_hydro) {
+
+ // MATTHIEU: to do: Write a better expression for the
+ // non-periodic case.
+
+ /* This is super-ugly but checks for direct neighbours */
+ /* with periodic BC */
+ if (((abs(i - iii) <= 1 || abs(i - iii - cdim[0]) <= 1 ||
+ abs(i - iii + cdim[0]) <= 1) &&
+ (abs(j - jjj) <= 1 || abs(j - jjj - cdim[1]) <= 1 ||
+ abs(j - jjj + cdim[1]) <= 1) &&
+ (abs(k - kkk) <= 1 || abs(k - kkk - cdim[2]) <= 1 ||
+ abs(k - kkk + cdim[2]) <= 1)))
+ proxy_type |= (int)proxy_cell_type_hydro;
+ }
+
+ /* In the gravity case, check distances using the MAC. */
+ if (with_gravity) {
+
+ /* First just add the direct neighbours. Then look for
+ some further out if the opening angle demands it */
+
+ /* This is super-ugly but checks for direct neighbours */
+ /* with periodic BC */
+ if (((abs(i - iii) <= 1 || abs(i - iii - cdim[0]) <= 1 ||
+ abs(i - iii + cdim[0]) <= 1) &&
+ (abs(j - jjj) <= 1 || abs(j - jjj - cdim[1]) <= 1 ||
+ abs(j - jjj + cdim[1]) <= 1) &&
+ (abs(k - kkk) <= 1 || abs(k - kkk - cdim[2]) <= 1 ||
+ abs(k - kkk + cdim[2]) <= 1))) {
+
+ proxy_type |= (int)proxy_cell_type_gravity;
+ } else {
+
+ /* We don't have multipoles yet (or their CoMs) so we will
+ have to cook up something based on cell locations only. We
+ hence need a lower limit on the distance that the CoMs in
+ those cells could have and an upper limit on the distance
+ of the furthest particle in the multipole from its CoM.
+ We then can decide whether we are too close for an M2L
+ interaction and hence require a proxy as this pair of cells
+ cannot rely on just an M2L calculation. */
+
+ /* Minimal distance between any two points in the cells */
+ const double min_dist_CoM2 = cell_min_dist2_same_size(
+ &cells[cid], &cells[cjd], periodic, dim);
+
+ /* Are we beyond the distance where the truncated forces are 0
+ * but not too far such that M2L can be used? */
+ if (periodic) {
+
+ if ((min_dist_CoM2 < max_mesh_dist2) &&
+ !(4. * r_max * r_max <
+ theta_crit * theta_crit * min_dist_CoM2))
+ proxy_type |= (int)proxy_cell_type_gravity;
+
+ } else {
+
+ if (!(4. * r_max * r_max <
+ theta_crit * theta_crit * min_dist_CoM2)) {
+ proxy_type |= (int)proxy_cell_type_gravity;
+ }
+ }
+ }
+ }
+
+ /* Abort if not in range at all */
+ if (proxy_type == proxy_cell_type_none) continue;
+
+ /* Add to proxies? */
+ if (cells[cid].nodeID == nodeID && cells[cjd].nodeID != nodeID) {
+
+ /* Do we already have a relationship with this node? */
+ int proxy_id = e->proxy_ind[cells[cjd].nodeID];
+ if (proxy_id < 0) {
+ if (e->nr_proxies == engine_maxproxies)
+ error("Maximum number of proxies exceeded.");
+
+ /* Ok, start a new proxy for this pair of nodes */
+ proxy_init(&proxies[e->nr_proxies], e->nodeID,
+ cells[cjd].nodeID);
+
+ /* Store the information */
+ e->proxy_ind[cells[cjd].nodeID] = e->nr_proxies;
+ proxy_id = e->nr_proxies;
+ e->nr_proxies += 1;
+
+ /* Check the maximal proxy limit */
+ if ((size_t)proxy_id > 8 * sizeof(long long))
+ error(
+ "Created more than %zd proxies. cell.mpi.sendto will "
+ "overflow.",
+ 8 * sizeof(long long));
+ }
+
+ /* Add the cell to the proxy */
+ proxy_addcell_in(&proxies[proxy_id], &cells[cjd], proxy_type);
+ proxy_addcell_out(&proxies[proxy_id], &cells[cid], proxy_type);
+
+ /* Store info about where to send the cell */
+ cells[cid].mpi.sendto |= (1ULL << proxy_id);
+ }
+
+ /* Same for the symmetric case? */
+ if (cells[cjd].nodeID == nodeID && cells[cid].nodeID != nodeID) {
+
+ /* Do we already have a relationship with this node? */
+ int proxy_id = e->proxy_ind[cells[cid].nodeID];
+ if (proxy_id < 0) {
+ if (e->nr_proxies == engine_maxproxies)
+ error("Maximum number of proxies exceeded.");
+
+ /* Ok, start a new proxy for this pair of nodes */
+ proxy_init(&proxies[e->nr_proxies], e->nodeID,
+ cells[cid].nodeID);
+
+ /* Store the information */
+ e->proxy_ind[cells[cid].nodeID] = e->nr_proxies;
+ proxy_id = e->nr_proxies;
+ e->nr_proxies += 1;
+
+ /* Check the maximal proxy limit */
+ if ((size_t)proxy_id > 8 * sizeof(long long))
+ error(
+ "Created more than %zd proxies. cell.mpi.sendto will "
+ "overflow.",
+ 8 * sizeof(long long));
+ }
+
+ /* Add the cell to the proxy */
+ proxy_addcell_in(&proxies[proxy_id], &cells[cid], proxy_type);
+ proxy_addcell_out(&proxies[proxy_id], &cells[cjd], proxy_type);
+
+ /* Store info about where to send the cell */
+ cells[cjd].mpi.sendto |= (1ULL << proxy_id);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /* Be clear about the time */
+ if (e->verbose)
+ message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+ clocks_getunit());
+#else
+ error("SWIFT was not compiled with MPI support.");
+#endif
+}
diff --git a/src/engine_strays.c b/src/engine_strays.c
new file mode 100644
index 0000000000000000000000000000000000000000..c096b2d671ca01d9c0b73ebb57b751b6e48fa785
--- /dev/null
+++ b/src/engine_strays.c
@@ -0,0 +1,565 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* MPI headers. */
+#ifdef WITH_MPI
+#include <mpi.h>
+#endif
+
+/* This object's header. */
+#include "engine.h"
+
+/* Local headers. */
+#include "proxy.h"
+
+/**
+ * @brief Exchange straying particles with other nodes.
+ *
+ * @param e The #engine.
+ * @param offset_parts The index in the parts array as of which the foreign
+ * parts reside (i.e. the current number of local #part).
+ * @param ind_part The foreign #cell ID of each part.
+ * @param Npart The number of stray parts, contains the number of parts received
+ * on return.
+ * @param offset_gparts The index in the gparts array as of which the foreign
+ * parts reside (i.e. the current number of local #gpart).
+ * @param ind_gpart The foreign #cell ID of each gpart.
+ * @param Ngpart The number of stray gparts, contains the number of gparts
+ * received on return.
+ * @param offset_sparts The index in the sparts array as of which the foreign
+ * parts reside (i.e. the current number of local #spart).
+ * @param ind_spart The foreign #cell ID of each spart.
+ * @param Nspart The number of stray sparts, contains the number of sparts
+ * received on return.
+ * @param offset_bparts The index in the bparts array as of which the foreign
+ * parts reside (i.e. the current number of local #bpart).
+ * @param ind_bpart The foreign #cell ID of each bpart.
+ * @param Nbpart The number of stray bparts, contains the number of bparts
+ * received on return.
+ *
+ * Note that this function does not mess-up the linkage between parts and
+ * gparts, i.e. the received particles have correct linkeage.
+ */
+void engine_exchange_strays(struct engine *e, const size_t offset_parts,
+ const int *restrict ind_part, size_t *Npart,
+ const size_t offset_gparts,
+ const int *restrict ind_gpart, size_t *Ngpart,
+ const size_t offset_sparts,
+ const int *restrict ind_spart, size_t *Nspart,
+ const size_t offset_bparts,
+ const int *restrict ind_bpart, size_t *Nbpart) {
+
+#ifdef WITH_MPI
+ struct space *s = e->s;
+ ticks tic = getticks();
+
+ /* Re-set the proxies. */
+ for (int k = 0; k < e->nr_proxies; k++) {
+ e->proxies[k].nr_parts_out = 0;
+ e->proxies[k].nr_gparts_out = 0;
+ e->proxies[k].nr_sparts_out = 0;
+ e->proxies[k].nr_bparts_out = 0;
+ }
+
+ /* Put the parts into the corresponding proxies. */
+ for (size_t k = 0; k < *Npart; k++) {
+
+ /* Ignore the particles we want to get rid of (inhibited, ...). */
+ if (ind_part[k] == -1) continue;
+
+ /* Get the target node and proxy ID. */
+ const int node_id = e->s->cells_top[ind_part[k]].nodeID;
+ if (node_id < 0 || node_id >= e->nr_nodes)
+ error("Bad node ID %i.", node_id);
+ const int pid = e->proxy_ind[node_id];
+ if (pid < 0) {
+ error(
+ "Do not have a proxy for the requested nodeID %i for part with "
+ "id=%lld, x=[%e,%e,%e].",
+ node_id, s->parts[offset_parts + k].id,
+ s->parts[offset_parts + k].x[0], s->parts[offset_parts + k].x[1],
+ s->parts[offset_parts + k].x[2]);
+ }
+
+ /* Re-link the associated gpart with the buffer offset of the part. */
+ if (s->parts[offset_parts + k].gpart != NULL) {
+ s->parts[offset_parts + k].gpart->id_or_neg_offset =
+ -e->proxies[pid].nr_parts_out;
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (s->parts[offset_parts + k].time_bin == time_bin_inhibited)
+ error("Attempting to exchange an inhibited particle");
+#endif
+
+ /* Load the part and xpart into the proxy. */
+ proxy_parts_load(&e->proxies[pid], &s->parts[offset_parts + k],
+ &s->xparts[offset_parts + k], 1);
+
+#ifdef WITH_LOGGER
+ if (e->policy & engine_policy_logger) {
+ /* Log the particle when leaving a rank. */
+ logger_log_part(
+ e->logger, &s->parts[offset_parts + k], &s->xparts[offset_parts + k],
+ e, /* log_all_fields */ 1,
+ logger_pack_flags_and_data(logger_flag_mpi_exit, node_id));
+ }
+#endif
+ }
+
+ /* Put the sparts into the corresponding proxies. */
+ for (size_t k = 0; k < *Nspart; k++) {
+
+ /* Ignore the particles we want to get rid of (inhibited, ...). */
+ if (ind_spart[k] == -1) continue;
+
+ /* Get the target node and proxy ID. */
+ const int node_id = e->s->cells_top[ind_spart[k]].nodeID;
+ if (node_id < 0 || node_id >= e->nr_nodes)
+ error("Bad node ID %i.", node_id);
+ const int pid = e->proxy_ind[node_id];
+ if (pid < 0) {
+ error(
+ "Do not have a proxy for the requested nodeID %i for part with "
+ "id=%lld, x=[%e,%e,%e].",
+ node_id, s->sparts[offset_sparts + k].id,
+ s->sparts[offset_sparts + k].x[0], s->sparts[offset_sparts + k].x[1],
+ s->sparts[offset_sparts + k].x[2]);
+ }
+
+ /* Re-link the associated gpart with the buffer offset of the spart. */
+ if (s->sparts[offset_sparts + k].gpart != NULL) {
+ s->sparts[offset_sparts + k].gpart->id_or_neg_offset =
+ -e->proxies[pid].nr_sparts_out;
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (s->sparts[offset_sparts + k].time_bin == time_bin_inhibited)
+ error("Attempting to exchange an inhibited particle");
+#endif
+
+ /* Load the spart into the proxy */
+ proxy_sparts_load(&e->proxies[pid], &s->sparts[offset_sparts + k], 1);
+
+#ifdef WITH_LOGGER
+ if (e->policy & engine_policy_logger) {
+ /* Log the particle when leaving a rank. */
+ logger_log_spart(
+ e->logger, &s->sparts[offset_sparts + k], e,
+ /* log_all_fields */ 1,
+ logger_pack_flags_and_data(logger_flag_mpi_exit, node_id));
+ }
+#endif
+ }
+
+ /* Put the bparts into the corresponding proxies. */
+ for (size_t k = 0; k < *Nbpart; k++) {
+
+ /* Ignore the particles we want to get rid of (inhibited, ...). */
+ if (ind_bpart[k] == -1) continue;
+
+ /* Get the target node and proxy ID. */
+ const int node_id = e->s->cells_top[ind_bpart[k]].nodeID;
+ if (node_id < 0 || node_id >= e->nr_nodes)
+ error("Bad node ID %i.", node_id);
+ const int pid = e->proxy_ind[node_id];
+ if (pid < 0) {
+ error(
+ "Do not have a proxy for the requested nodeID %i for part with "
+ "id=%lld, x=[%e,%e,%e].",
+ node_id, s->bparts[offset_bparts + k].id,
+ s->bparts[offset_bparts + k].x[0], s->bparts[offset_bparts + k].x[1],
+ s->bparts[offset_bparts + k].x[2]);
+ }
+
+ /* Re-link the associated gpart with the buffer offset of the bpart. */
+ if (s->bparts[offset_bparts + k].gpart != NULL) {
+ s->bparts[offset_bparts + k].gpart->id_or_neg_offset =
+ -e->proxies[pid].nr_bparts_out;
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (s->bparts[offset_bparts + k].time_bin == time_bin_inhibited)
+ error("Attempting to exchange an inhibited particle");
+#endif
+
+ /* Load the bpart into the proxy */
+ proxy_bparts_load(&e->proxies[pid], &s->bparts[offset_bparts + k], 1);
+
+#ifdef WITH_LOGGER
+ if (e->policy & engine_policy_logger) {
+ error("Not yet implemented.");
+ }
+#endif
+ }
+
+ /* Put the gparts into the corresponding proxies. */
+ for (size_t k = 0; k < *Ngpart; k++) {
+
+ /* Ignore the particles we want to get rid of (inhibited, ...). */
+ if (ind_gpart[k] == -1) continue;
+
+ /* Get the target node and proxy ID. */
+ const int node_id = e->s->cells_top[ind_gpart[k]].nodeID;
+ if (node_id < 0 || node_id >= e->nr_nodes)
+ error("Bad node ID %i.", node_id);
+ const int pid = e->proxy_ind[node_id];
+ if (pid < 0) {
+ error(
+ "Do not have a proxy for the requested nodeID %i for part with "
+ "id=%lli, x=[%e,%e,%e].",
+ node_id, s->gparts[offset_gparts + k].id_or_neg_offset,
+ s->gparts[offset_gparts + k].x[0], s->gparts[offset_gparts + k].x[1],
+ s->gparts[offset_gparts + k].x[2]);
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (s->gparts[offset_gparts + k].time_bin == time_bin_inhibited)
+ error("Attempting to exchange an inhibited particle");
+#endif
+
+ /* Load the gpart into the proxy */
+ proxy_gparts_load(&e->proxies[pid], &s->gparts[offset_gparts + k], 1);
+
+#ifdef WITH_LOGGER
+ /* Write only the dark matter particles */
+ if ((e->policy & engine_policy_logger) &&
+ s->gparts[offset_gparts + k].type == swift_type_dark_matter) {
+
+ /* Log the particle when leaving a rank. */
+ logger_log_gpart(
+ e->logger, &s->gparts[offset_gparts + k], e,
+ /* log_all_fields */ 1,
+ logger_pack_flags_and_data(logger_flag_mpi_exit, node_id));
+ }
+#endif
+ }
+
+ /* Launch the proxies. */
+ MPI_Request reqs_in[5 * engine_maxproxies];
+ MPI_Request reqs_out[5 * engine_maxproxies];
+ for (int k = 0; k < e->nr_proxies; k++) {
+ proxy_parts_exchange_first(&e->proxies[k]);
+ reqs_in[k] = e->proxies[k].req_parts_count_in;
+ reqs_out[k] = e->proxies[k].req_parts_count_out;
+ }
+
+ /* Wait for each count to come in and start the recv. */
+ for (int k = 0; k < e->nr_proxies; k++) {
+ int pid = MPI_UNDEFINED;
+ if (MPI_Waitany(e->nr_proxies, reqs_in, &pid, MPI_STATUS_IGNORE) !=
+ MPI_SUCCESS ||
+ pid == MPI_UNDEFINED)
+ error("MPI_Waitany failed.");
+ // message( "request from proxy %i has arrived." , pid );
+ proxy_parts_exchange_second(&e->proxies[pid]);
+ }
+
+ /* Wait for all the sends to have finished too. */
+ if (MPI_Waitall(e->nr_proxies, reqs_out, MPI_STATUSES_IGNORE) != MPI_SUCCESS)
+ error("MPI_Waitall on sends failed.");
+
+ /* Count the total number of incoming particles and make sure we have
+ enough space to accommodate them. */
+ int count_parts_in = 0;
+ int count_gparts_in = 0;
+ int count_sparts_in = 0;
+ int count_bparts_in = 0;
+ for (int k = 0; k < e->nr_proxies; k++) {
+ count_parts_in += e->proxies[k].nr_parts_in;
+ count_gparts_in += e->proxies[k].nr_gparts_in;
+ count_sparts_in += e->proxies[k].nr_sparts_in;
+ count_bparts_in += e->proxies[k].nr_bparts_in;
+ }
+ if (e->verbose) {
+ message(
+ "sent out %zu/%zu/%zu/%zu parts/gparts/sparts/bparts, got %i/%i/%i/%i "
+ "back.",
+ *Npart, *Ngpart, *Nspart, *Nbpart, count_parts_in, count_gparts_in,
+ count_sparts_in, count_bparts_in);
+ }
+
+ /* Reallocate the particle arrays if necessary */
+ if (offset_parts + count_parts_in > s->size_parts) {
+ s->size_parts = (offset_parts + count_parts_in) * engine_parts_size_grow;
+ struct part *parts_new = NULL;
+ struct xpart *xparts_new = NULL;
+ if (swift_memalign("parts", (void **)&parts_new, part_align,
+ sizeof(struct part) * s->size_parts) != 0 ||
+ swift_memalign("xparts", (void **)&xparts_new, xpart_align,
+ sizeof(struct xpart) * s->size_parts) != 0)
+ error("Failed to allocate new part data.");
+ memcpy(parts_new, s->parts, sizeof(struct part) * offset_parts);
+ memcpy(xparts_new, s->xparts, sizeof(struct xpart) * offset_parts);
+ swift_free("parts", s->parts);
+ swift_free("xparts", s->xparts);
+ s->parts = parts_new;
+ s->xparts = xparts_new;
+
+ /* Reset the links */
+ for (size_t k = 0; k < offset_parts; k++) {
+ if (s->parts[k].gpart != NULL) {
+ s->parts[k].gpart->id_or_neg_offset = -k;
+ }
+ }
+ }
+
+ if (offset_sparts + count_sparts_in > s->size_sparts) {
+ s->size_sparts = (offset_sparts + count_sparts_in) * engine_parts_size_grow;
+ struct spart *sparts_new = NULL;
+ if (swift_memalign("sparts", (void **)&sparts_new, spart_align,
+ sizeof(struct spart) * s->size_sparts) != 0)
+ error("Failed to allocate new spart data.");
+ memcpy(sparts_new, s->sparts, sizeof(struct spart) * offset_sparts);
+ swift_free("sparts", s->sparts);
+ s->sparts = sparts_new;
+
+ /* Reset the links */
+ for (size_t k = 0; k < offset_sparts; k++) {
+ if (s->sparts[k].gpart != NULL) {
+ s->sparts[k].gpart->id_or_neg_offset = -k;
+ }
+ }
+ }
+
+ if (offset_bparts + count_bparts_in > s->size_bparts) {
+ s->size_bparts = (offset_bparts + count_bparts_in) * engine_parts_size_grow;
+ struct bpart *bparts_new = NULL;
+ if (swift_memalign("bparts", (void **)&bparts_new, bpart_align,
+ sizeof(struct bpart) * s->size_bparts) != 0)
+ error("Failed to allocate new bpart data.");
+ memcpy(bparts_new, s->bparts, sizeof(struct bpart) * offset_bparts);
+ swift_free("bparts", s->bparts);
+ s->bparts = bparts_new;
+
+ /* Reset the links */
+ for (size_t k = 0; k < offset_bparts; k++) {
+ if (s->bparts[k].gpart != NULL) {
+ s->bparts[k].gpart->id_or_neg_offset = -k;
+ }
+ }
+ }
+
+ if (offset_gparts + count_gparts_in > s->size_gparts) {
+ s->size_gparts = (offset_gparts + count_gparts_in) * engine_parts_size_grow;
+ struct gpart *gparts_new = NULL;
+ if (swift_memalign("gparts", (void **)&gparts_new, gpart_align,
+ sizeof(struct gpart) * s->size_gparts) != 0)
+ error("Failed to allocate new gpart data.");
+ memcpy(gparts_new, s->gparts, sizeof(struct gpart) * offset_gparts);
+ swift_free("gparts", s->gparts);
+ s->gparts = gparts_new;
+
+ /* Reset the links */
+ for (size_t k = 0; k < offset_gparts; k++) {
+ if (s->gparts[k].type == swift_type_gas) {
+ s->parts[-s->gparts[k].id_or_neg_offset].gpart = &s->gparts[k];
+ } else if (s->gparts[k].type == swift_type_stars) {
+ s->sparts[-s->gparts[k].id_or_neg_offset].gpart = &s->gparts[k];
+ } else if (s->gparts[k].type == swift_type_black_hole) {
+ s->bparts[-s->gparts[k].id_or_neg_offset].gpart = &s->gparts[k];
+ }
+ }
+ }
+
+ /* Collect the requests for the particle data from the proxies. */
+ int nr_in = 0, nr_out = 0;
+ for (int k = 0; k < e->nr_proxies; k++) {
+ if (e->proxies[k].nr_parts_in > 0) {
+ reqs_in[5 * k] = e->proxies[k].req_parts_in;
+ reqs_in[5 * k + 1] = e->proxies[k].req_xparts_in;
+ nr_in += 2;
+ } else {
+ reqs_in[5 * k] = reqs_in[5 * k + 1] = MPI_REQUEST_NULL;
+ }
+ if (e->proxies[k].nr_gparts_in > 0) {
+ reqs_in[5 * k + 2] = e->proxies[k].req_gparts_in;
+ nr_in += 1;
+ } else {
+ reqs_in[5 * k + 2] = MPI_REQUEST_NULL;
+ }
+ if (e->proxies[k].nr_sparts_in > 0) {
+ reqs_in[5 * k + 3] = e->proxies[k].req_sparts_in;
+ nr_in += 1;
+ } else {
+ reqs_in[5 * k + 3] = MPI_REQUEST_NULL;
+ }
+ if (e->proxies[k].nr_bparts_in > 0) {
+ reqs_in[5 * k + 4] = e->proxies[k].req_bparts_in;
+ nr_in += 1;
+ } else {
+ reqs_in[5 * k + 4] = MPI_REQUEST_NULL;
+ }
+
+ if (e->proxies[k].nr_parts_out > 0) {
+ reqs_out[5 * k] = e->proxies[k].req_parts_out;
+ reqs_out[5 * k + 1] = e->proxies[k].req_xparts_out;
+ nr_out += 2;
+ } else {
+ reqs_out[5 * k] = reqs_out[5 * k + 1] = MPI_REQUEST_NULL;
+ }
+ if (e->proxies[k].nr_gparts_out > 0) {
+ reqs_out[5 * k + 2] = e->proxies[k].req_gparts_out;
+ nr_out += 1;
+ } else {
+ reqs_out[5 * k + 2] = MPI_REQUEST_NULL;
+ }
+ if (e->proxies[k].nr_sparts_out > 0) {
+ reqs_out[5 * k + 3] = e->proxies[k].req_sparts_out;
+ nr_out += 1;
+ } else {
+ reqs_out[5 * k + 3] = MPI_REQUEST_NULL;
+ }
+ if (e->proxies[k].nr_bparts_out > 0) {
+ reqs_out[5 * k + 4] = e->proxies[k].req_bparts_out;
+ nr_out += 1;
+ } else {
+ reqs_out[5 * k + 4] = MPI_REQUEST_NULL;
+ }
+ }
+
+ /* Wait for each part array to come in and collect the new
+ parts from the proxies. */
+ int count_parts = 0, count_gparts = 0, count_sparts = 0, count_bparts = 0;
+ for (int k = 0; k < nr_in; k++) {
+ int err, pid;
+ if ((err = MPI_Waitany(5 * e->nr_proxies, reqs_in, &pid,
+ MPI_STATUS_IGNORE)) != MPI_SUCCESS) {
+ char buff[MPI_MAX_ERROR_STRING];
+ int res;
+ MPI_Error_string(err, buff, &res);
+ error("MPI_Waitany failed (%s).", buff);
+ }
+ if (pid == MPI_UNDEFINED) break;
+ // message( "request from proxy %i has arrived." , pid / 5 );
+ pid = 5 * (pid / 5);
+
+ /* If all the requests for a given proxy have arrived... */
+ if (reqs_in[pid + 0] == MPI_REQUEST_NULL &&
+ reqs_in[pid + 1] == MPI_REQUEST_NULL &&
+ reqs_in[pid + 2] == MPI_REQUEST_NULL &&
+ reqs_in[pid + 3] == MPI_REQUEST_NULL &&
+ reqs_in[pid + 4] == MPI_REQUEST_NULL) {
+ /* Copy the particle data to the part/xpart/gpart arrays. */
+ struct proxy *prox = &e->proxies[pid / 5];
+ memcpy(&s->parts[offset_parts + count_parts], prox->parts_in,
+ sizeof(struct part) * prox->nr_parts_in);
+ memcpy(&s->xparts[offset_parts + count_parts], prox->xparts_in,
+ sizeof(struct xpart) * prox->nr_parts_in);
+ memcpy(&s->gparts[offset_gparts + count_gparts], prox->gparts_in,
+ sizeof(struct gpart) * prox->nr_gparts_in);
+ memcpy(&s->sparts[offset_sparts + count_sparts], prox->sparts_in,
+ sizeof(struct spart) * prox->nr_sparts_in);
+ memcpy(&s->bparts[offset_bparts + count_bparts], prox->bparts_in,
+ sizeof(struct bpart) * prox->nr_bparts_in);
+
+#ifdef WITH_LOGGER
+ if (e->policy & engine_policy_logger) {
+ const uint32_t flag =
+ logger_pack_flags_and_data(logger_flag_mpi_enter, prox->nodeID);
+
+ struct part *parts = &s->parts[offset_parts + count_parts];
+ struct xpart *xparts = &s->xparts[offset_parts + count_parts];
+ struct spart *sparts = &s->sparts[offset_sparts + count_sparts];
+ struct gpart *gparts = &s->gparts[offset_gparts + count_gparts];
+
+ /* Log the gas particles */
+ logger_log_parts(e->logger, parts, xparts, prox->nr_parts_in, e,
+ /* log_all_fields */ 1, flag);
+
+ /* Log the stellar particles */
+ logger_log_sparts(e->logger, sparts, prox->nr_sparts_in, e,
+ /* log_all_fields */ 1, flag);
+
+ /* Log the gparts */
+ logger_log_gparts(e->logger, gparts, prox->nr_gparts_in, e,
+ /* log_all_fields */ 1, flag);
+
+ /* Log the bparts */
+ if (prox->nr_bparts_in > 0) {
+ error("TODO");
+ }
+ }
+#endif
+ /* for (int k = offset; k < offset + count; k++)
+ message(
+ "received particle %lli, x=[%.3e %.3e %.3e], h=%.3e, from node %i.",
+ s->parts[k].id, s->parts[k].x[0], s->parts[k].x[1],
+ s->parts[k].x[2], s->parts[k].h, p->nodeID); */
+
+ /* Re-link the gparts. */
+ for (int kk = 0; kk < prox->nr_gparts_in; kk++) {
+ struct gpart *gp = &s->gparts[offset_gparts + count_gparts + kk];
+
+ if (gp->type == swift_type_gas) {
+ struct part *p =
+ &s->parts[offset_parts + count_parts - gp->id_or_neg_offset];
+ gp->id_or_neg_offset = s->parts - p;
+ p->gpart = gp;
+ } else if (gp->type == swift_type_stars) {
+ struct spart *sp =
+ &s->sparts[offset_sparts + count_sparts - gp->id_or_neg_offset];
+ gp->id_or_neg_offset = s->sparts - sp;
+ sp->gpart = gp;
+ } else if (gp->type == swift_type_black_hole) {
+ struct bpart *bp =
+ &s->bparts[offset_bparts + count_bparts - gp->id_or_neg_offset];
+ gp->id_or_neg_offset = s->bparts - bp;
+ bp->gpart = gp;
+ }
+ }
+
+ /* Advance the counters. */
+ count_parts += prox->nr_parts_in;
+ count_gparts += prox->nr_gparts_in;
+ count_sparts += prox->nr_sparts_in;
+ count_bparts += prox->nr_bparts_in;
+ }
+ }
+
+ /* Wait for all the sends to have finished too. */
+ if (nr_out > 0)
+ if (MPI_Waitall(5 * e->nr_proxies, reqs_out, MPI_STATUSES_IGNORE) !=
+ MPI_SUCCESS)
+ error("MPI_Waitall on sends failed.");
+
+ /* Free the proxy memory */
+ for (int k = 0; k < e->nr_proxies; k++) {
+ proxy_free_particle_buffers(&e->proxies[k]);
+ }
+
+ if (e->verbose)
+ message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+ clocks_getunit());
+
+ /* Return the number of harvested parts. */
+ *Npart = count_parts;
+ *Ngpart = count_gparts;
+ *Nspart = count_sparts;
+ *Nbpart = count_bparts;
+
+#else
+ error("SWIFT was not compiled with MPI support.");
+#endif
+}
diff --git a/src/parallel_io.c b/src/parallel_io.c
index 4e98c24397d51da2e1a43899aa40ab22f43db96d..55b345912da6b57060bf587179089afcf3c88f9a 100644
--- a/src/parallel_io.c
+++ b/src/parallel_io.c
@@ -1046,7 +1046,7 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units,
}
/* If we are remapping ParticleIDs later, start by setting them to 1. */
- if (remap_ids) set_ids_to_one(*gparts, *Ngparts);
+ if (remap_ids) io_set_ids_to_one(*gparts, *Ngparts);
if (!dry_run && with_gravity) {
diff --git a/src/proxy.c b/src/proxy.c
index ed002132df8745128b8aa2700bf37d79ec43775d..2207b34ac8b7bca1dbff881c147d3506ec28f5e0 100644
--- a/src/proxy.c
+++ b/src/proxy.c
@@ -289,7 +289,8 @@ void proxy_cells_count_mapper(void *map_data, int num_elements,
struct cell *cells = (struct cell *)map_data;
for (int k = 0; k < num_elements; k++) {
- if (cells[k].mpi.sendto) cells[k].mpi.pcell_size = cell_getsize(&cells[k]);
+ if (cells[k].mpi.sendto)
+ cells[k].mpi.pcell_size = cell_get_tree_size(&cells[k]);
}
}
diff --git a/src/serial_io.c b/src/serial_io.c
index b734d5fa7082b059537887f3260ceafa7c1a22a6..eb801c8f10a21f1336e6dc13897f57925a0905bb 100644
--- a/src/serial_io.c
+++ b/src/serial_io.c
@@ -851,7 +851,7 @@ void read_ic_serial(char* fileName, const struct unit_system* internal_units,
}
/* If we are remapping ParticleIDs later, start by setting them to 1. */
- if (remap_ids) set_ids_to_one(*gparts, *Ngparts);
+ if (remap_ids) io_set_ids_to_one(*gparts, *Ngparts);
/* Duplicate the parts for gravity */
if (!dry_run && with_gravity) {
diff --git a/src/single_io.c b/src/single_io.c
index 3765257f3914a0610763ad611ac2d8aef99c88cb..e0912ba45e53945a3ee98d9d945343df77292b53 100644
--- a/src/single_io.c
+++ b/src/single_io.c
@@ -708,7 +708,7 @@ void read_ic_single(const char* fileName,
}
/* If we are remapping ParticleIDs later, start by setting them to 1. */
- if (remap_ids) set_ids_to_one(*gparts, *Ngparts);
+ if (remap_ids) io_set_ids_to_one(*gparts, *Ngparts);
/* Duplicate the parts for gravity */
if (!dry_run && with_gravity) {
diff --git a/src/space.c b/src/space.c
index 026b4ce48ee1e51c9bc07d9133ba5f0586f1a93d..0d1c6ed164669e83ee3dc344932f356cf6af2089 100644
--- a/src/space.c
+++ b/src/space.c
@@ -41,34 +41,20 @@
/* Local headers. */
#include "atomic.h"
-#include "black_holes.h"
-#include "chemistry.h"
#include "const.h"
#include "cooling.h"
-#include "debug.h"
#include "engine.h"
#include "error.h"
-#include "gravity.h"
-#include "hydro.h"
#include "kernel_hydro.h"
#include "lock.h"
-#include "memswap.h"
-#include "memuse.h"
#include "minmax.h"
-#include "multipole.h"
-#include "pressure_floor.h"
#include "proxy.h"
#include "restart.h"
-#include "rt.h"
-#include "sink.h"
#include "sort_part.h"
#include "space_unique_id.h"
#include "star_formation.h"
-#include "star_formation_logger.h"
-#include "stars.h"
#include "threadpool.h"
#include "tools.h"
-#include "tracers.h"
/* Split size. */
int space_splitsize = space_splitsize_default;
@@ -106,6 +92,8 @@ int engine_star_resort_task_depth = engine_star_resort_task_depth_default;
/*! Expected maximal number of strays received at a rebuild */
int space_expected_max_nr_strays = space_expected_max_nr_strays_default;
+
+/*! Counter for cell IDs (when debugging) */
#if defined(SWIFT_DEBUG_CHECKS) || defined(SWIFT_CELL_GRAPH)
int last_cell_id;
#endif
@@ -119,223 +107,6 @@ struct qstack {
volatile int ready;
};
-/**
- * @brief Information required to compute the particle cell indices.
- */
-struct index_data {
- struct space *s;
- int *ind;
- int *cell_counts;
- size_t count_inhibited_part;
- size_t count_inhibited_gpart;
- size_t count_inhibited_spart;
- size_t count_inhibited_bpart;
- size_t count_inhibited_sink;
- size_t count_extra_part;
- size_t count_extra_gpart;
- size_t count_extra_spart;
- size_t count_extra_bpart;
- size_t count_extra_sink;
-};
-
-/**
- * @brief Recursively dismantle a cell tree.
- *
- * @param s The #space.
- * @param c The #cell to recycle.
- * @param cell_rec_begin Pointer to the start of the list of cells to recycle.
- * @param cell_rec_end Pointer to the end of the list of cells to recycle.
- * @param multipole_rec_begin Pointer to the start of the list of multipoles to
- * recycle.
- * @param multipole_rec_end Pointer to the end of the list of multipoles to
- * recycle.
- */
-void space_rebuild_recycle_rec(struct space *s, struct cell *c,
- struct cell **cell_rec_begin,
- struct cell **cell_rec_end,
- struct gravity_tensors **multipole_rec_begin,
- struct gravity_tensors **multipole_rec_end) {
- if (c->split)
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL) {
- space_rebuild_recycle_rec(s, c->progeny[k], cell_rec_begin,
- cell_rec_end, multipole_rec_begin,
- multipole_rec_end);
-
- c->progeny[k]->next = *cell_rec_begin;
- *cell_rec_begin = c->progeny[k];
-
- if (s->with_self_gravity) {
- c->progeny[k]->grav.multipole->next = *multipole_rec_begin;
- *multipole_rec_begin = c->progeny[k]->grav.multipole;
- }
-
- if (*cell_rec_end == NULL) *cell_rec_end = *cell_rec_begin;
- if (s->with_self_gravity && *multipole_rec_end == NULL)
- *multipole_rec_end = *multipole_rec_begin;
-
- c->progeny[k]->grav.multipole = NULL;
- c->progeny[k] = NULL;
- }
-}
-
-void space_rebuild_recycle_mapper(void *map_data, int num_elements,
- void *extra_data) {
-
- struct space *s = (struct space *)extra_data;
- struct cell *cells = (struct cell *)map_data;
-
- for (int k = 0; k < num_elements; k++) {
- struct cell *c = &cells[k];
- struct cell *cell_rec_begin = NULL, *cell_rec_end = NULL;
- struct gravity_tensors *multipole_rec_begin = NULL,
- *multipole_rec_end = NULL;
- space_rebuild_recycle_rec(s, c, &cell_rec_begin, &cell_rec_end,
- &multipole_rec_begin, &multipole_rec_end);
- if (cell_rec_begin != NULL)
- space_recycle_list(s, cell_rec_begin, cell_rec_end, multipole_rec_begin,
- multipole_rec_end);
- c->hydro.sorts = NULL;
- c->stars.sorts = NULL;
- c->nr_tasks = 0;
- c->grav.nr_mm_tasks = 0;
- c->hydro.density = NULL;
- c->hydro.gradient = NULL;
- c->hydro.force = NULL;
- c->hydro.limiter = NULL;
- c->grav.grav = NULL;
- c->grav.mm = NULL;
- c->hydro.dx_max_part = 0.0f;
- c->hydro.dx_max_sort = 0.0f;
- c->sinks.dx_max_part = 0.f;
- c->stars.dx_max_part = 0.f;
- c->stars.dx_max_sort = 0.f;
- c->black_holes.dx_max_part = 0.f;
- c->hydro.sorted = 0;
- c->hydro.sort_allocated = 0;
- c->stars.sorted = 0;
- c->hydro.count = 0;
- c->hydro.count_total = 0;
- c->hydro.updated = 0;
- c->grav.count = 0;
- c->grav.count_total = 0;
- c->grav.updated = 0;
- c->sinks.count = 0;
- c->stars.count = 0;
- c->stars.count_total = 0;
- c->stars.updated = 0;
- c->black_holes.count = 0;
- c->black_holes.count_total = 0;
- c->black_holes.updated = 0;
- c->grav.init = NULL;
- c->grav.init_out = NULL;
- c->hydro.extra_ghost = NULL;
- c->hydro.ghost_in = NULL;
- c->hydro.ghost_out = NULL;
- c->hydro.ghost = NULL;
- c->hydro.sink_formation = NULL;
- c->hydro.star_formation = NULL;
- c->hydro.stars_resort = NULL;
- c->stars.ghost = NULL;
- c->stars.density = NULL;
- c->stars.feedback = NULL;
- c->black_holes.density_ghost = NULL;
- c->black_holes.swallow_ghost[0] = NULL;
- c->black_holes.swallow_ghost[1] = NULL;
- c->black_holes.swallow_ghost[2] = NULL;
- c->black_holes.density = NULL;
- c->black_holes.swallow = NULL;
- c->black_holes.do_gas_swallow = NULL;
- c->black_holes.do_bh_swallow = NULL;
- c->black_holes.feedback = NULL;
- c->kick1 = NULL;
- c->kick2 = NULL;
- c->timestep = NULL;
- c->timestep_limiter = NULL;
- c->timestep_sync = NULL;
- c->hydro.end_force = NULL;
- c->hydro.drift = NULL;
- c->sinks.drift = NULL;
- c->stars.drift = NULL;
- c->stars.stars_in = NULL;
- c->stars.stars_out = NULL;
- c->black_holes.drift = NULL;
- c->black_holes.black_holes_in = NULL;
- c->black_holes.black_holes_out = NULL;
- c->sinks.sink_in = NULL;
- c->sinks.sink_out = NULL;
- c->grav.drift = NULL;
- c->grav.drift_out = NULL;
- c->hydro.cooling_in = NULL;
- c->hydro.cooling_out = NULL;
- c->hydro.cooling = NULL;
- c->grav.long_range = NULL;
- c->grav.down_in = NULL;
- c->grav.down = NULL;
- c->grav.end_force = NULL;
- c->top = c;
- c->super = c;
- c->hydro.super = c;
- c->grav.super = c;
- c->hydro.parts = NULL;
- c->hydro.xparts = NULL;
- c->grav.parts = NULL;
- c->grav.parts_rebuild = NULL;
- c->sinks.parts = NULL;
- c->stars.parts = NULL;
- c->stars.parts_rebuild = NULL;
- c->black_holes.parts = NULL;
- c->flags = 0;
- c->hydro.ti_end_min = -1;
- c->hydro.ti_end_max = -1;
- c->grav.ti_end_min = -1;
- c->grav.ti_end_max = -1;
- c->sinks.ti_end_min = -1;
- c->sinks.ti_end_max = -1;
- c->stars.ti_end_min = -1;
- c->stars.ti_end_max = -1;
- c->black_holes.ti_end_min = -1;
- c->black_holes.ti_end_max = -1;
- c->hydro.rt_inject = NULL;
- c->hydro.rt_in = NULL;
- c->hydro.rt_out = NULL;
- c->hydro.rt_ghost1 = NULL;
- star_formation_logger_init(&c->stars.sfh);
-#if defined(SWIFT_DEBUG_CHECKS) || defined(SWIFT_CELL_GRAPH)
- c->cellID = 0;
-#endif
- if (s->with_self_gravity)
- bzero(c->grav.multipole, sizeof(struct gravity_tensors));
-
- cell_free_hydro_sorts(c);
- cell_free_stars_sorts(c);
-#if WITH_MPI
- c->mpi.tag = -1;
- c->mpi.recv = NULL;
- c->mpi.send = NULL;
-#endif
- }
-}
-
-/**
- * @brief Free up any allocated cells.
- *
- * @param s The #space.
- */
-void space_free_cells(struct space *s) {
-
- ticks tic = getticks();
-
- threadpool_map(&s->e->threadpool, space_rebuild_recycle_mapper, s->cells_top,
- s->nr_cells, sizeof(struct cell), threadpool_auto_chunk_size,
- s);
- s->maxdepth = 0;
-
- if (s->e->verbose)
- message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
- clocks_getunit());
-}
-
/**
* @brief Free any memory in use for foreign particles.
*
@@ -376,5034 +147,598 @@ void space_free_foreign_parts(struct space *s, const int clear_cell_pointers) {
#endif
}
-/**
- * @brief Re-build the top-level cell grid.
- *
- * @param s The #space.
- * @param verbose Print messages to stdout or not.
- */
-void space_regrid(struct space *s, int verbose) {
+void space_reorder_extra_parts_mapper(void *map_data, int num_cells,
+ void *extra_data) {
+ int *local_cells = (int *)map_data;
+ struct space *s = (struct space *)extra_data;
+ struct cell *cells_top = s->cells_top;
- const size_t nr_parts = s->nr_parts;
- const size_t nr_sparts = s->nr_sparts;
- const size_t nr_bparts = s->nr_bparts;
- const size_t nr_sinks = s->nr_sinks;
- const ticks tic = getticks();
- const integertime_t ti_current = (s->e != NULL) ? s->e->ti_current : 0;
-
- /* Run through the cells and get the current h_max. */
- // tic = getticks();
- float h_max = s->cell_min / kernel_gamma / space_stretch;
- if (nr_parts > 0) {
-
- /* Can we use the list of local non-empty top-level cells? */
- if (s->local_cells_with_particles_top != NULL) {
- for (int k = 0; k < s->nr_local_cells_with_particles; ++k) {
- const struct cell *c =
- &s->cells_top[s->local_cells_with_particles_top[k]];
- if (c->hydro.h_max > h_max) {
- h_max = c->hydro.h_max;
- }
- if (c->stars.h_max > h_max) {
- h_max = c->stars.h_max;
- }
- if (c->black_holes.h_max > h_max) {
- h_max = c->black_holes.h_max;
- }
- if (c->sinks.r_cut_max > h_max) {
- h_max = c->sinks.r_cut_max / kernel_gamma;
- }
- }
+ for (int ind = 0; ind < num_cells; ind++) {
+ struct cell *c = &cells_top[local_cells[ind]];
+ cell_reorder_extra_parts(c, c->hydro.parts - s->parts);
+ }
+}
- /* Can we instead use all the top-level cells? */
- } else if (s->cells_top != NULL) {
- for (int k = 0; k < s->nr_cells; k++) {
- const struct cell *c = &s->cells_top[k];
- if (c->nodeID == engine_rank && c->hydro.h_max > h_max) {
- h_max = c->hydro.h_max;
- }
- if (c->nodeID == engine_rank && c->stars.h_max > h_max) {
- h_max = c->stars.h_max;
- }
- if (c->nodeID == engine_rank && c->black_holes.h_max > h_max) {
- h_max = c->black_holes.h_max;
- }
- if (c->nodeID == engine_rank && c->sinks.r_cut_max > h_max) {
- h_max = c->sinks.r_cut_max / kernel_gamma;
- }
- }
+void space_reorder_extra_gparts_mapper(void *map_data, int num_cells,
+ void *extra_data) {
- /* Last option: run through the particles */
- } else {
- for (size_t k = 0; k < nr_parts; k++) {
- if (s->parts[k].h > h_max) h_max = s->parts[k].h;
- }
- for (size_t k = 0; k < nr_sparts; k++) {
- if (s->sparts[k].h > h_max) h_max = s->sparts[k].h;
- }
- for (size_t k = 0; k < nr_bparts; k++) {
- if (s->bparts[k].h > h_max) h_max = s->bparts[k].h;
- }
- for (size_t k = 0; k < nr_sinks; k++) {
- if (s->sinks[k].r_cut > h_max) h_max = s->sinks[k].r_cut / kernel_gamma;
- }
- }
- }
+ int *local_cells = (int *)map_data;
+ struct space *s = (struct space *)extra_data;
+ struct cell *cells_top = s->cells_top;
-/* If we are running in parallel, make sure everybody agrees on
- how large the largest cell should be. */
-#ifdef WITH_MPI
- {
- float buff;
- if (MPI_Allreduce(&h_max, &buff, 1, MPI_FLOAT, MPI_MAX, MPI_COMM_WORLD) !=
- MPI_SUCCESS)
- error("Failed to aggregate the rebuild flag across nodes.");
- h_max = buff;
- }
-#endif
- if (verbose) message("h_max is %.3e (cell_min=%.3e).", h_max, s->cell_min);
-
- /* Get the new putative cell dimensions. */
- const int cdim[3] = {
- (int)floor(s->dim[0] /
- fmax(h_max * kernel_gamma * space_stretch, s->cell_min)),
- (int)floor(s->dim[1] /
- fmax(h_max * kernel_gamma * space_stretch, s->cell_min)),
- (int)floor(s->dim[2] /
- fmax(h_max * kernel_gamma * space_stretch, s->cell_min))};
-
- /* Check if we have enough cells for periodicity. */
- if (s->periodic && (cdim[0] < 3 || cdim[1] < 3 || cdim[2] < 3))
- error(
- "Must have at least 3 cells in each spatial dimension when periodicity "
- "is switched on.\nThis error is often caused by any of the "
- "followings:\n"
- " - too few particles to generate a sensible grid,\n"
- " - the initial value of 'Scheduler:max_top_level_cells' is too "
- "small,\n"
- " - the (minimal) time-step is too large leading to particles with "
- "predicted smoothing lengths too large for the box size,\n"
- " - particles with velocities so large that they move by more than two "
- "box sizes per time-step.\n");
-
-/* In MPI-Land, changing the top-level cell size requires that the
- * global partition is recomputed and the particles redistributed.
- * Be prepared to do that. */
-#ifdef WITH_MPI
- double oldwidth[3];
- double oldcdim[3];
- int *oldnodeIDs = NULL;
- if (cdim[0] < s->cdim[0] || cdim[1] < s->cdim[1] || cdim[2] < s->cdim[2]) {
-
- /* Capture state of current space. */
- oldcdim[0] = s->cdim[0];
- oldcdim[1] = s->cdim[1];
- oldcdim[2] = s->cdim[2];
- oldwidth[0] = s->width[0];
- oldwidth[1] = s->width[1];
- oldwidth[2] = s->width[2];
-
- if ((oldnodeIDs =
- (int *)swift_malloc("nodeIDs", sizeof(int) * s->nr_cells)) == NULL)
- error("Failed to allocate temporary nodeIDs.");
-
- int cid = 0;
- for (int i = 0; i < s->cdim[0]; i++) {
- for (int j = 0; j < s->cdim[1]; j++) {
- for (int k = 0; k < s->cdim[2]; k++) {
- cid = cell_getid(oldcdim, i, j, k);
- oldnodeIDs[cid] = s->cells_top[cid].nodeID;
- }
- }
- }
+ for (int ind = 0; ind < num_cells; ind++) {
+ struct cell *c = &cells_top[local_cells[ind]];
+ cell_reorder_extra_gparts(c, s->parts, s->sparts);
}
+}
- /* Are we about to allocate new top level cells without a regrid?
- * Can happen when restarting the application. */
- const int no_regrid = (s->cells_top == NULL && oldnodeIDs == NULL);
-#endif
+void space_reorder_extra_sparts_mapper(void *map_data, int num_cells,
+ void *extra_data) {
- /* Do we need to re-build the upper-level cells? */
- // tic = getticks();
- if (s->cells_top == NULL || cdim[0] < s->cdim[0] || cdim[1] < s->cdim[1] ||
- cdim[2] < s->cdim[2]) {
+ int *local_cells = (int *)map_data;
+ struct space *s = (struct space *)extra_data;
+ struct cell *cells_top = s->cells_top;
-/* Be verbose about this. */
-#ifdef SWIFT_DEBUG_CHECKS
- message("(re)griding space cdim=(%d %d %d)", cdim[0], cdim[1], cdim[2]);
- fflush(stdout);
-#endif
+ for (int ind = 0; ind < num_cells; ind++) {
+ struct cell *c = &cells_top[local_cells[ind]];
+ cell_reorder_extra_sparts(c, c->stars.parts - s->sparts);
+ }
+}
- /* Free the old cells, if they were allocated. */
- if (s->cells_top != NULL) {
- space_free_cells(s);
- swift_free("local_cells_with_tasks_top", s->local_cells_with_tasks_top);
- swift_free("local_cells_top", s->local_cells_top);
- swift_free("cells_with_particles_top", s->cells_with_particles_top);
- swift_free("local_cells_with_particles_top",
- s->local_cells_with_particles_top);
- swift_free("cells_top", s->cells_top);
- swift_free("multipoles_top", s->multipoles_top);
- }
+void space_reorder_extra_sinks_mapper(void *map_data, int num_cells,
+ void *extra_data) {
- /* Also free the task arrays, these will be regenerated and we can use the
- * memory while copying the particle arrays. */
- if (s->e != NULL) scheduler_free_tasks(&s->e->sched);
+ int *local_cells = (int *)map_data;
+ struct space *s = (struct space *)extra_data;
+ struct cell *cells_top = s->cells_top;
- /* Set the new cell dimensions only if smaller. */
- for (int k = 0; k < 3; k++) {
- s->cdim[k] = cdim[k];
- s->width[k] = s->dim[k] / cdim[k];
- s->iwidth[k] = 1.0 / s->width[k];
- }
- const float dmin = min3(s->width[0], s->width[1], s->width[2]);
+ for (int ind = 0; ind < num_cells; ind++) {
+ struct cell *c = &cells_top[local_cells[ind]];
+ cell_reorder_extra_sinks(c, c->sinks.parts - s->sinks);
+ }
+}
- /* Allocate the highest level of cells. */
- s->tot_cells = s->nr_cells = cdim[0] * cdim[1] * cdim[2];
+/**
+ * @brief Re-orders the particles in each cell such that the extra particles
+ * for on-the-fly creation are located at the end of their respective cells.
+ *
+ * This assumes that all the particles (real and extra) have already been sorted
+ * in their correct top-level cell.
+ *
+ * @param s The #space to act upon.
+ * @param verbose Are we talkative?
+ */
+void space_reorder_extras(struct space *s, int verbose) {
- if (swift_memalign("cells_top", (void **)&s->cells_top, cell_align,
- s->nr_cells * sizeof(struct cell)) != 0)
- error("Failed to allocate top-level cells.");
- bzero(s->cells_top, s->nr_cells * sizeof(struct cell));
+ /* Re-order the gas particles */
+ if (space_extra_parts)
+ threadpool_map(&s->e->threadpool, space_reorder_extra_parts_mapper,
+ s->local_cells_top, s->nr_local_cells, sizeof(int),
+ threadpool_auto_chunk_size, s);
- /* Allocate the multipoles for the top-level cells. */
- if (s->with_self_gravity) {
- if (swift_memalign("multipoles_top", (void **)&s->multipoles_top,
- multipole_align,
- s->nr_cells * sizeof(struct gravity_tensors)) != 0)
- error("Failed to allocate top-level multipoles.");
- bzero(s->multipoles_top, s->nr_cells * sizeof(struct gravity_tensors));
- }
+ /* Re-order the gravity particles */
+ if (space_extra_gparts)
+ threadpool_map(&s->e->threadpool, space_reorder_extra_gparts_mapper,
+ s->local_cells_top, s->nr_local_cells, sizeof(int),
+ threadpool_auto_chunk_size, s);
- /* Allocate the indices of local cells */
- if (swift_memalign("local_cells_top", (void **)&s->local_cells_top,
- SWIFT_STRUCT_ALIGNMENT, s->nr_cells * sizeof(int)) != 0)
- error("Failed to allocate indices of local top-level cells.");
- bzero(s->local_cells_top, s->nr_cells * sizeof(int));
-
- /* Allocate the indices of local cells with tasks */
- if (swift_memalign("local_cells_with_tasks_top",
- (void **)&s->local_cells_with_tasks_top,
- SWIFT_STRUCT_ALIGNMENT, s->nr_cells * sizeof(int)) != 0)
- error("Failed to allocate indices of local top-level cells with tasks.");
- bzero(s->local_cells_with_tasks_top, s->nr_cells * sizeof(int));
-
- /* Allocate the indices of cells with particles */
- if (swift_memalign("cells_with_particles_top",
- (void **)&s->cells_with_particles_top,
- SWIFT_STRUCT_ALIGNMENT, s->nr_cells * sizeof(int)) != 0)
- error("Failed to allocate indices of top-level cells with particles.");
- bzero(s->cells_with_particles_top, s->nr_cells * sizeof(int));
-
- /* Allocate the indices of local cells with particles */
- if (swift_memalign("local_cells_with_particles_top",
- (void **)&s->local_cells_with_particles_top,
- SWIFT_STRUCT_ALIGNMENT, s->nr_cells * sizeof(int)) != 0)
- error(
- "Failed to allocate indices of local top-level cells with "
- "particles.");
- bzero(s->local_cells_with_particles_top, s->nr_cells * sizeof(int));
-
- /* Set the cells' locks */
- for (int k = 0; k < s->nr_cells; k++) {
- if (lock_init(&s->cells_top[k].hydro.lock) != 0)
- error("Failed to init spinlock for hydro.");
- if (lock_init(&s->cells_top[k].grav.plock) != 0)
- error("Failed to init spinlock for gravity.");
- if (lock_init(&s->cells_top[k].grav.mlock) != 0)
- error("Failed to init spinlock for multipoles.");
- if (lock_init(&s->cells_top[k].grav.star_formation_lock) != 0)
- error("Failed to init spinlock for star formation (gpart).");
- if (lock_init(&s->cells_top[k].stars.lock) != 0)
- error("Failed to init spinlock for stars.");
- if (lock_init(&s->cells_top[k].sinks.lock) != 0)
- error("Failed to init spinlock for sinks.");
- if (lock_init(&s->cells_top[k].sinks.sink_formation_lock) != 0)
- error("Failed to init spinlock for sink formation.");
- if (lock_init(&s->cells_top[k].black_holes.lock) != 0)
- error("Failed to init spinlock for black holes.");
- if (lock_init(&s->cells_top[k].stars.star_formation_lock) != 0)
- error("Failed to init spinlock for star formation (spart).");
- }
+ /* Re-order the star particles */
+ if (space_extra_sparts)
+ threadpool_map(&s->e->threadpool, space_reorder_extra_sparts_mapper,
+ s->local_cells_top, s->nr_local_cells, sizeof(int),
+ threadpool_auto_chunk_size, s);
- /* Set the cell location and sizes. */
- for (int i = 0; i < cdim[0]; i++)
- for (int j = 0; j < cdim[1]; j++)
- for (int k = 0; k < cdim[2]; k++) {
- const size_t cid = cell_getid(cdim, i, j, k);
- struct cell *restrict c = &s->cells_top[cid];
- c->loc[0] = i * s->width[0];
- c->loc[1] = j * s->width[1];
- c->loc[2] = k * s->width[2];
- c->width[0] = s->width[0];
- c->width[1] = s->width[1];
- c->width[2] = s->width[2];
- c->dmin = dmin;
- c->depth = 0;
- c->split = 0;
- c->hydro.count = 0;
- c->grav.count = 0;
- c->stars.count = 0;
- c->sinks.count = 0;
- c->top = c;
- c->super = c;
- c->hydro.super = c;
- c->grav.super = c;
- c->hydro.ti_old_part = ti_current;
- c->grav.ti_old_part = ti_current;
- c->stars.ti_old_part = ti_current;
- c->sinks.ti_old_part = ti_current;
- c->black_holes.ti_old_part = ti_current;
- c->grav.ti_old_multipole = ti_current;
-#ifdef WITH_MPI
- c->mpi.tag = -1;
- c->mpi.recv = NULL;
- c->mpi.send = NULL;
-#endif // WITH_MPI
- if (s->with_self_gravity) c->grav.multipole = &s->multipoles_top[cid];
-#if defined(SWIFT_DEBUG_CHECKS) || defined(SWIFT_CELL_GRAPH)
- c->cellID = -last_cell_id;
- last_cell_id++;
-#endif
- }
+ /* Re-order the black hole particles */
+ if (space_extra_bparts)
+ error("Missing implementation of BH extra reordering");
- /* Be verbose about the change. */
- if (verbose)
- message("set cell dimensions to [ %i %i %i ].", cdim[0], cdim[1],
- cdim[2]);
+ /* Re-order the sink particles */
+ if (space_extra_sinks)
+ threadpool_map(&s->e->threadpool, space_reorder_extra_sinks_mapper,
+ s->local_cells_top, s->nr_local_cells, sizeof(int),
+ threadpool_auto_chunk_size, s);
+}
-#ifdef WITH_MPI
- if (oldnodeIDs != NULL) {
- /* We have changed the top-level cell dimension, so need to redistribute
- * cells around the nodes. We repartition using the old space node
- * positions as a grid to resample. */
- if (s->e->nodeID == 0)
- message(
- "basic cell dimensions have increased - recalculating the "
- "global partition.");
-
- if (!partition_space_to_space(oldwidth, oldcdim, oldnodeIDs, s)) {
-
- /* Failed, try another technique that requires no settings. */
- message("Failed to get a new partition, trying less optimal method");
- struct partition initial_partition;
-#if defined(HAVE_PARMETIS) || defined(HAVE_METIS)
- initial_partition.type = INITPART_METIS_NOWEIGHT;
-#else
- initial_partition.type = INITPART_VECTORIZE;
-#endif
- partition_initial_partition(&initial_partition, s->e->nodeID,
- s->e->nr_nodes, s);
- }
+/**
+ * @brief #threadpool mapper function to sanitize the cells
+ *
+ * @param map_data Pointers towards the top-level cells.
+ * @param num_cells The number of top-level cells.
+ * @param extra_data Unused parameters.
+ */
+void space_sanitize_mapper(void *map_data, int num_cells, void *extra_data) {
+ /* Unpack the inputs. */
+ struct cell *cells_top = (struct cell *)map_data;
- /* Re-distribute the particles to their new nodes. */
- engine_redistribute(s->e);
+ for (int ind = 0; ind < num_cells; ind++) {
+ struct cell *c = &cells_top[ind];
+ cell_sanitize(c, 0);
+ }
+}
- /* Make the proxies. */
- engine_makeproxies(s->e);
+/**
+ * @brief Runs through the top-level cells and sanitize their h values
+ *
+ * @param s The #space to act upon.
+ */
+void space_sanitize(struct space *s) {
- /* Finished with these. */
- swift_free("nodeIDs", oldnodeIDs);
+ if (s->e->nodeID == 0) message("Cleaning up unreasonable values of h");
- } else if (no_regrid && s->e != NULL) {
- /* If we have created the top-levels cells and not done an initial
- * partition (can happen when restarting), then the top-level cells
- * are not assigned to a node, we must do that and then associate the
- * particles with the cells. Note requires that
- * partition_store_celllist() was called once before, or just before
- * dumping the restart files.*/
- partition_restore_celllist(s, s->e->reparttype);
+ threadpool_map(&s->e->threadpool, space_sanitize_mapper, s->cells_top,
+ s->nr_cells, sizeof(struct cell), threadpool_auto_chunk_size,
+ /*extra_data=*/NULL);
+}
- /* Now re-distribute the particles, should just add to cells? */
- engine_redistribute(s->e);
+/**
+ * @brief Mapping function to free the sorted indices buffers.
+ */
+void space_map_clearsort(struct cell *c, void *data) {
- /* Make the proxies. */
- engine_makeproxies(s->e);
- }
-#endif /* WITH_MPI */
+ cell_free_hydro_sorts(c);
+ cell_free_stars_sorts(c);
+}
- // message( "rebuilding upper-level cells took %.3f %s." ,
- // clocks_from_ticks(double)(getticks() - tic), clocks_getunit());
+/**
+ * @brief Map a function to all particles in a cell recursively.
+ *
+ * @param c The #cell we are working in.
+ * @param fun Function pointer to apply on the cells.
+ * @param data Data passed to the function fun.
+ */
+static void rec_map_parts(struct cell *c,
+ void (*fun)(struct part *p, struct cell *c,
+ void *data),
+ void *data) {
+ /* No progeny? */
+ if (!c->split)
+ for (int k = 0; k < c->hydro.count; k++) fun(&c->hydro.parts[k], c, data);
- } /* re-build upper-level cells? */
- else { /* Otherwise, just clean up the cells. */
+ /* Otherwise, recurse. */
+ else
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL) rec_map_parts(c->progeny[k], fun, data);
+}
- /* Free the old cells, if they were allocated. */
- space_free_cells(s);
- }
+/**
+ * @brief Map a function to all particles in a space.
+ *
+ * @param s The #space we are working in.
+ * @param fun Function pointer to apply on the cells.
+ * @param data Data passed to the function fun.
+ */
+void space_map_parts(struct space *s,
+ void (*fun)(struct part *p, struct cell *c, void *data),
+ void *data) {
- if (verbose)
- message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
- clocks_getunit());
+ /* Call the recursive function on all higher-level cells. */
+ for (int cid = 0; cid < s->nr_cells; cid++)
+ rec_map_parts(&s->cells_top[cid], fun, data);
}
/**
- * @brief Allocate memory for the extra particles used for on-the-fly creation.
- *
- * This rarely actually allocates memory. Most of the time, we convert
- * pre-allocated memory inot extra particles.
- *
- * This function also sets the extra particles' location to their top-level
- * cells. They can then be sorted into their correct memory position later on.
+ * @brief Map a function to all particles in a cell recursively.
*
- * @param s The current #space.
- * @param verbose Are we talkative?
+ * @param c The #cell we are working in.
+ * @param fun Function pointer to apply on the cells.
*/
-void space_allocate_extras(struct space *s, int verbose) {
-
- const int local_nodeID = s->e->nodeID;
-
- /* Anything to do here? (Abort if we don't want extras)*/
- if (space_extra_parts == 0 && space_extra_gparts == 0 &&
- space_extra_sparts == 0 && space_extra_bparts == 0 &&
- space_extra_sinks == 0)
- return;
-
- /* The top-level cells */
- const struct cell *cells = s->cells_top;
- const double half_cell_width[3] = {0.5 * cells[0].width[0],
- 0.5 * cells[0].width[1],
- 0.5 * cells[0].width[2]};
-
- /* The current number of particles (including spare ones) */
- size_t nr_parts = s->nr_parts;
- size_t nr_gparts = s->nr_gparts;
- size_t nr_sparts = s->nr_sparts;
- size_t nr_bparts = s->nr_bparts;
- size_t nr_sinks = s->nr_sinks;
-
- /* The current number of actual particles */
- size_t nr_actual_parts = nr_parts - s->nr_extra_parts;
- size_t nr_actual_gparts = nr_gparts - s->nr_extra_gparts;
- size_t nr_actual_sparts = nr_sparts - s->nr_extra_sparts;
- size_t nr_actual_bparts = nr_bparts - s->nr_extra_bparts;
- size_t nr_actual_sinks = nr_sinks - s->nr_extra_sinks;
-
- /* The number of particles we allocated memory for (MPI overhead) */
- size_t size_parts = s->size_parts;
- size_t size_gparts = s->size_gparts;
- size_t size_sparts = s->size_sparts;
- size_t size_bparts = s->size_bparts;
- size_t size_sinks = s->size_sinks;
-
- int *local_cells = (int *)malloc(sizeof(int) * s->nr_cells);
- if (local_cells == NULL)
- error("Failed to allocate list of local top-level cells");
-
- /* List the local cells */
- size_t nr_local_cells = 0;
- for (int i = 0; i < s->nr_cells; ++i) {
- if (s->cells_top[i].nodeID == local_nodeID) {
- local_cells[nr_local_cells] = i;
- ++nr_local_cells;
- }
- }
-
- /* Number of extra particles we want for each type */
- const size_t expected_num_extra_parts = nr_local_cells * space_extra_parts;
- const size_t expected_num_extra_gparts = nr_local_cells * space_extra_gparts;
- const size_t expected_num_extra_sparts = nr_local_cells * space_extra_sparts;
- const size_t expected_num_extra_bparts = nr_local_cells * space_extra_bparts;
- const size_t expected_num_extra_sinks = nr_local_cells * space_extra_sinks;
+static void rec_map_parts_xparts(struct cell *c,
+ void (*fun)(struct part *p, struct xpart *xp,
+ struct cell *c)) {
- if (verbose) {
- message("Currently have %zd/%zd/%zd/%zd/%zd real particles.",
- nr_actual_parts, nr_actual_gparts, nr_actual_sinks,
- nr_actual_sparts, nr_actual_bparts);
- message("Currently have %zd/%zd/%zd/%zd/%zd spaces for extra particles.",
- s->nr_extra_parts, s->nr_extra_gparts, s->nr_extra_sinks,
- s->nr_extra_sparts, s->nr_extra_bparts);
- message(
- "Requesting space for future %zd/%zd/%zd/%zd/%zd "
- "part/gpart/sinks/sparts/bparts.",
- expected_num_extra_parts, expected_num_extra_gparts,
- expected_num_extra_sinks, expected_num_extra_sparts,
- expected_num_extra_bparts);
- }
+ /* No progeny? */
+ if (!c->split)
+ for (int k = 0; k < c->hydro.count; k++)
+ fun(&c->hydro.parts[k], &c->hydro.xparts[k], c);
- if (expected_num_extra_parts < s->nr_extra_parts)
- error("Reduction in top-level cells number not handled.");
- if (expected_num_extra_gparts < s->nr_extra_gparts)
- error("Reduction in top-level cells number not handled.");
- if (expected_num_extra_sparts < s->nr_extra_sparts)
- error("Reduction in top-level cells number not handled.");
- if (expected_num_extra_bparts < s->nr_extra_bparts)
- error("Reduction in top-level cells number not handled.");
- if (expected_num_extra_sinks < s->nr_extra_sinks)
- error("Reduction in top-level cells number not handled.");
-
- /* Do we have enough space for the extra gparts (i.e. we haven't used up any)
- * ? */
- if (nr_actual_gparts + expected_num_extra_gparts > nr_gparts) {
-
- /* Ok... need to put some more in the game */
-
- /* Do we need to reallocate? */
- if (nr_actual_gparts + expected_num_extra_gparts > size_gparts) {
-
- size_gparts = (nr_actual_gparts + expected_num_extra_gparts) *
- engine_redistribute_alloc_margin;
-
- if (verbose)
- message("Re-allocating gparts array from %zd to %zd", s->size_gparts,
- size_gparts);
-
- /* Create more space for parts */
- struct gpart *gparts_new = NULL;
- if (swift_memalign("gparts", (void **)&gparts_new, gpart_align,
- sizeof(struct gpart) * size_gparts) != 0)
- error("Failed to allocate new gpart data");
- const ptrdiff_t delta = gparts_new - s->gparts;
- memcpy(gparts_new, s->gparts, sizeof(struct gpart) * s->size_gparts);
- swift_free("gparts", s->gparts);
- s->gparts = gparts_new;
-
- /* Update the counter */
- s->size_gparts = size_gparts;
-
- /* We now need to reset all the part and spart pointers */
- for (size_t i = 0; i < nr_parts; ++i) {
- if (s->parts[i].time_bin != time_bin_not_created)
- s->parts[i].gpart += delta;
- }
- for (size_t i = 0; i < nr_sparts; ++i) {
- if (s->sparts[i].time_bin != time_bin_not_created)
- s->sparts[i].gpart += delta;
- }
- for (size_t i = 0; i < nr_bparts; ++i) {
- if (s->bparts[i].time_bin != time_bin_not_created)
- s->bparts[i].gpart += delta;
- }
- }
-
- /* Turn some of the allocated spares into particles we can use */
- for (size_t i = nr_gparts; i < nr_actual_gparts + expected_num_extra_gparts;
- ++i) {
- bzero(&s->gparts[i], sizeof(struct gpart));
- s->gparts[i].time_bin = time_bin_not_created;
- s->gparts[i].type = swift_type_dark_matter;
- s->gparts[i].id_or_neg_offset = -1;
- }
-
- /* Put the spare particles in their correct cell */
- size_t local_cell_id = 0;
- int current_cell = local_cells[local_cell_id];
- int count_in_cell = 0;
- size_t count_extra_gparts = 0;
- for (size_t i = 0; i < nr_actual_gparts + expected_num_extra_gparts; ++i) {
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (current_cell == s->nr_cells)
- error("Cell counter beyond the maximal nr. cells.");
-#endif
-
- if (s->gparts[i].time_bin == time_bin_not_created) {
-
- /* We want the extra particles to be at the centre of their cell */
- s->gparts[i].x[0] = cells[current_cell].loc[0] + half_cell_width[0];
- s->gparts[i].x[1] = cells[current_cell].loc[1] + half_cell_width[1];
- s->gparts[i].x[2] = cells[current_cell].loc[2] + half_cell_width[2];
- ++count_in_cell;
- count_extra_gparts++;
- }
-
- /* Once we have reached the number of extra gpart per cell, we move to the
- * next */
- if (count_in_cell == space_extra_gparts) {
- ++local_cell_id;
-
- if (local_cell_id == nr_local_cells) break;
-
- current_cell = local_cells[local_cell_id];
- count_in_cell = 0;
- }
- }
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (count_extra_gparts != expected_num_extra_gparts)
- error("Constructed the wrong number of extra gparts (%zd vs. %zd)",
- count_extra_gparts, expected_num_extra_gparts);
-#endif
-
- /* Update the counters */
- s->nr_gparts = nr_actual_gparts + expected_num_extra_gparts;
- s->nr_extra_gparts = expected_num_extra_gparts;
- }
-
- /* Do we have enough space for the extra parts (i.e. we haven't used up any) ?
- */
- if (nr_actual_parts + expected_num_extra_parts > nr_parts) {
-
- /* Ok... need to put some more in the game */
-
- /* Do we need to reallocate? */
- if (nr_actual_parts + expected_num_extra_parts > size_parts) {
-
- size_parts = (nr_actual_parts + expected_num_extra_parts) *
- engine_redistribute_alloc_margin;
-
- if (verbose)
- message("Re-allocating parts array from %zd to %zd", s->size_parts,
- size_parts);
-
- /* Create more space for parts */
- struct part *parts_new = NULL;
- if (swift_memalign("parts", (void **)&parts_new, part_align,
- sizeof(struct part) * size_parts) != 0)
- error("Failed to allocate new part data");
- memcpy(parts_new, s->parts, sizeof(struct part) * s->size_parts);
- swift_free("parts", s->parts);
- s->parts = parts_new;
-
- /* Same for xparts */
- struct xpart *xparts_new = NULL;
- if (swift_memalign("xparts", (void **)&xparts_new, xpart_align,
- sizeof(struct xpart) * size_parts) != 0)
- error("Failed to allocate new xpart data");
- memcpy(xparts_new, s->xparts, sizeof(struct xpart) * s->size_parts);
- swift_free("xparts", s->xparts);
- s->xparts = xparts_new;
-
- /* Update the counter */
- s->size_parts = size_parts;
- }
-
- /* Turn some of the allocated spares into particles we can use */
- for (size_t i = nr_parts; i < nr_actual_parts + expected_num_extra_parts;
- ++i) {
- bzero(&s->parts[i], sizeof(struct part));
- bzero(&s->xparts[i], sizeof(struct xpart));
- s->parts[i].time_bin = time_bin_not_created;
- s->parts[i].id = -42;
- }
-
- /* Put the spare particles in their correct cell */
- size_t local_cell_id = 0;
- int current_cell = local_cells[local_cell_id];
- int count_in_cell = 0;
- size_t count_extra_parts = 0;
- for (size_t i = 0; i < nr_actual_parts + expected_num_extra_parts; ++i) {
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (current_cell == s->nr_cells)
- error("Cell counter beyond the maximal nr. cells.");
-#endif
-
- if (s->parts[i].time_bin == time_bin_not_created) {
-
- /* We want the extra particles to be at the centre of their cell */
- s->parts[i].x[0] = cells[current_cell].loc[0] + half_cell_width[0];
- s->parts[i].x[1] = cells[current_cell].loc[1] + half_cell_width[1];
- s->parts[i].x[2] = cells[current_cell].loc[2] + half_cell_width[2];
- ++count_in_cell;
- count_extra_parts++;
- }
-
- /* Once we have reached the number of extra part per cell, we move to the
- * next */
- if (count_in_cell == space_extra_parts) {
- ++local_cell_id;
-
- if (local_cell_id == nr_local_cells) break;
-
- current_cell = local_cells[local_cell_id];
- count_in_cell = 0;
- }
- }
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (count_extra_parts != expected_num_extra_parts)
- error("Constructed the wrong number of extra parts (%zd vs. %zd)",
- count_extra_parts, expected_num_extra_parts);
-#endif
-
- /* Update the counters */
- s->nr_parts = nr_actual_parts + expected_num_extra_parts;
- s->nr_extra_parts = expected_num_extra_parts;
- }
-
- /* Do we have enough space for the extra sinks (i.e. we haven't used up any)
- * ? */
- if (nr_actual_sinks + expected_num_extra_sinks > nr_sinks) {
- /* Ok... need to put some more in the game */
-
- /* Do we need to reallocate? */
- if (nr_actual_sinks + expected_num_extra_sinks > size_sinks) {
-
- size_sinks = (nr_actual_sinks + expected_num_extra_sinks) *
- engine_redistribute_alloc_margin;
-
- if (verbose)
- message("Re-allocating sinks array from %zd to %zd", s->size_sinks,
- size_sinks);
-
- /* Create more space for parts */
- struct sink *sinks_new = NULL;
- if (swift_memalign("sinks", (void **)&sinks_new, sink_align,
- sizeof(struct sink) * size_sinks) != 0)
- error("Failed to allocate new sink data");
- memcpy(sinks_new, s->sinks, sizeof(struct sink) * s->size_sinks);
- swift_free("sinks", s->sinks);
- s->sinks = sinks_new;
-
- /* Update the counter */
- s->size_sinks = size_sinks;
- }
-
- /* Turn some of the allocated spares into particles we can use */
- for (size_t i = nr_sinks; i < nr_actual_sinks + expected_num_extra_sinks;
- ++i) {
- bzero(&s->sinks[i], sizeof(struct sink));
- s->sinks[i].time_bin = time_bin_not_created;
- s->sinks[i].id = -42;
- }
-
- /* Put the spare particles in their correct cell */
- size_t local_cell_id = 0;
- int current_cell = local_cells[local_cell_id];
- int count_in_cell = 0;
- size_t count_extra_sinks = 0;
- for (size_t i = 0; i < nr_actual_sinks + expected_num_extra_sinks; ++i) {
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (current_cell == s->nr_cells)
- error("Cell counter beyond the maximal nr. cells.");
-#endif
-
- if (s->sinks[i].time_bin == time_bin_not_created) {
-
- /* We want the extra particles to be at the centre of their cell */
- s->sinks[i].x[0] = cells[current_cell].loc[0] + half_cell_width[0];
- s->sinks[i].x[1] = cells[current_cell].loc[1] + half_cell_width[1];
- s->sinks[i].x[2] = cells[current_cell].loc[2] + half_cell_width[2];
- ++count_in_cell;
- count_extra_sinks++;
- }
-
- /* Once we have reached the number of extra sink per cell, we move to the
- * next */
- if (count_in_cell == space_extra_sinks) {
- ++local_cell_id;
-
- if (local_cell_id == nr_local_cells) break;
-
- current_cell = local_cells[local_cell_id];
- count_in_cell = 0;
- }
- }
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (count_extra_sinks != expected_num_extra_sinks)
- error("Constructed the wrong number of extra sinks (%zd vs. %zd)",
- count_extra_sinks, expected_num_extra_sinks);
-#endif
-
- /* Update the counters */
- s->nr_sinks = nr_actual_sinks + expected_num_extra_sinks;
- s->nr_extra_sinks = expected_num_extra_sinks;
- }
-
- /* Do we have enough space for the extra sparts (i.e. we haven't used up any)
- * ? */
- if (nr_actual_sparts + expected_num_extra_sparts > nr_sparts) {
-
- /* Ok... need to put some more in the game */
-
- /* Do we need to reallocate? */
- if (nr_actual_sparts + expected_num_extra_sparts > size_sparts) {
-
- size_sparts = (nr_actual_sparts + expected_num_extra_sparts) *
- engine_redistribute_alloc_margin;
-
- if (verbose)
- message("Re-allocating sparts array from %zd to %zd", s->size_sparts,
- size_sparts);
-
- /* Create more space for parts */
- struct spart *sparts_new = NULL;
- if (swift_memalign("sparts", (void **)&sparts_new, spart_align,
- sizeof(struct spart) * size_sparts) != 0)
- error("Failed to allocate new spart data");
- memcpy(sparts_new, s->sparts, sizeof(struct spart) * s->size_sparts);
- swift_free("sparts", s->sparts);
- s->sparts = sparts_new;
-
- /* Update the counter */
- s->size_sparts = size_sparts;
- }
-
- /* Turn some of the allocated spares into particles we can use */
- for (size_t i = nr_sparts; i < nr_actual_sparts + expected_num_extra_sparts;
- ++i) {
- bzero(&s->sparts[i], sizeof(struct spart));
- s->sparts[i].time_bin = time_bin_not_created;
- s->sparts[i].id = -42;
- }
-
- /* Put the spare particles in their correct cell */
- size_t local_cell_id = 0;
- int current_cell = local_cells[local_cell_id];
- int count_in_cell = 0;
- size_t count_extra_sparts = 0;
- for (size_t i = 0; i < nr_actual_sparts + expected_num_extra_sparts; ++i) {
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (current_cell == s->nr_cells)
- error("Cell counter beyond the maximal nr. cells.");
-#endif
-
- if (s->sparts[i].time_bin == time_bin_not_created) {
-
- /* We want the extra particles to be at the centre of their cell */
- s->sparts[i].x[0] = cells[current_cell].loc[0] + half_cell_width[0];
- s->sparts[i].x[1] = cells[current_cell].loc[1] + half_cell_width[1];
- s->sparts[i].x[2] = cells[current_cell].loc[2] + half_cell_width[2];
- ++count_in_cell;
- count_extra_sparts++;
- }
-
- /* Once we have reached the number of extra spart per cell, we move to the
- * next */
- if (count_in_cell == space_extra_sparts) {
- ++local_cell_id;
-
- if (local_cell_id == nr_local_cells) break;
-
- current_cell = local_cells[local_cell_id];
- count_in_cell = 0;
- }
- }
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (count_extra_sparts != expected_num_extra_sparts)
- error("Constructed the wrong number of extra sparts (%zd vs. %zd)",
- count_extra_sparts, expected_num_extra_sparts);
-#endif
-
- /* Update the counters */
- s->nr_sparts = nr_actual_sparts + expected_num_extra_sparts;
- s->nr_extra_sparts = expected_num_extra_sparts;
- }
-
- /* Do we have enough space for the extra bparts (i.e. we haven't used up any)
- * ? */
- if (nr_actual_bparts + expected_num_extra_bparts > nr_bparts) {
-
- /* Ok... need to put some more in the game */
-
- /* Do we need to reallocate? */
- if (nr_actual_bparts + expected_num_extra_bparts > size_bparts) {
-
- size_bparts = (nr_actual_bparts + expected_num_extra_bparts) *
- engine_redistribute_alloc_margin;
-
- if (verbose)
- message("Re-allocating bparts array from %zd to %zd", s->size_bparts,
- size_bparts);
-
- /* Create more space for parts */
- struct bpart *bparts_new = NULL;
- if (swift_memalign("bparts", (void **)&bparts_new, bpart_align,
- sizeof(struct bpart) * size_bparts) != 0)
- error("Failed to allocate new bpart data");
- memcpy(bparts_new, s->bparts, sizeof(struct bpart) * s->size_bparts);
- swift_free("bparts", s->bparts);
- s->bparts = bparts_new;
-
- /* Update the counter */
- s->size_bparts = size_bparts;
- }
-
- /* Turn some of the allocated spares into particles we can use */
- for (size_t i = nr_bparts; i < nr_actual_bparts + expected_num_extra_bparts;
- ++i) {
- bzero(&s->bparts[i], sizeof(struct bpart));
- s->bparts[i].time_bin = time_bin_not_created;
- s->bparts[i].id = -42;
- }
-
- /* Put the spare particles in their correct cell */
- size_t local_cell_id = 0;
- int current_cell = local_cells[local_cell_id];
- int count_in_cell = 0;
- size_t count_extra_bparts = 0;
- for (size_t i = 0; i < nr_actual_bparts + expected_num_extra_bparts; ++i) {
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (current_cell == s->nr_cells)
- error("Cell counter beyond the maximal nr. cells.");
-#endif
-
- if (s->bparts[i].time_bin == time_bin_not_created) {
-
- /* We want the extra particles to be at the centre of their cell */
- s->bparts[i].x[0] = cells[current_cell].loc[0] + half_cell_width[0];
- s->bparts[i].x[1] = cells[current_cell].loc[1] + half_cell_width[1];
- s->bparts[i].x[2] = cells[current_cell].loc[2] + half_cell_width[2];
- ++count_in_cell;
- count_extra_bparts++;
- }
-
- /* Once we have reached the number of extra bpart per cell, we move to the
- * next */
- if (count_in_cell == space_extra_bparts) {
- ++local_cell_id;
-
- if (local_cell_id == nr_local_cells) break;
-
- current_cell = local_cells[local_cell_id];
- count_in_cell = 0;
- }
- }
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (count_extra_bparts != expected_num_extra_bparts)
- error("Constructed the wrong number of extra bparts (%zd vs. %zd)",
- count_extra_bparts, expected_num_extra_bparts);
-#endif
-
- /* Update the counters */
- s->nr_bparts = nr_actual_bparts + expected_num_extra_bparts;
- s->nr_extra_bparts = expected_num_extra_bparts;
- }
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Verify that the links are correct */
- if ((nr_gparts > 0 && nr_parts > 0) || (nr_gparts > 0 && nr_sparts > 0) ||
- (nr_gparts > 0 && nr_bparts > 0) || (nr_gparts > 0 && nr_sinks > 0))
- part_verify_links(s->parts, s->gparts, s->sinks, s->sparts, s->bparts,
- nr_parts, nr_gparts, nr_sinks, nr_sparts, nr_bparts,
- verbose);
-#endif
-
- /* Free the list of local cells */
- free(local_cells);
-}
-
-/**
- * @brief Re-build the cells as well as the tasks.
- *
- * @param s The #space in which to update the cells.
- * @param repartitioned Did we just repartition?
- * @param verbose Print messages to stdout or not
- */
-void space_rebuild(struct space *s, int repartitioned, int verbose) {
-
- const ticks tic = getticks();
-
-/* Be verbose about this. */
-#ifdef SWIFT_DEBUG_CHECKS
- if (s->e->nodeID == 0 || verbose) message("(re)building space");
- fflush(stdout);
-#endif
-#if defined(SWIFT_DEBUG_CHECKS) || defined(SWIFT_CELL_GRAPH)
- /* Reset the cell counter */
- last_cell_id = 1;
-#endif
-
- /* Re-grid if necessary, or just re-set the cell data. */
- space_regrid(s, verbose);
-
- /* Allocate extra space for particles that will be created */
- if (s->with_star_formation || s->e->policy & engine_policy_sinks)
- space_allocate_extras(s, verbose);
-
- struct cell *cells_top = s->cells_top;
- const integertime_t ti_current = (s->e != NULL) ? s->e->ti_current : 0;
- const int local_nodeID = s->e->nodeID;
-
- /* The current number of particles */
- size_t nr_parts = s->nr_parts;
- size_t nr_gparts = s->nr_gparts;
- size_t nr_sparts = s->nr_sparts;
- size_t nr_bparts = s->nr_bparts;
- size_t nr_sinks = s->nr_sinks;
-
- /* The number of particles we allocated memory for */
- size_t size_parts = s->size_parts;
- size_t size_gparts = s->size_gparts;
- size_t size_sparts = s->size_sparts;
- size_t size_bparts = s->size_bparts;
- size_t size_sinks = s->size_sinks;
-
- /* Counter for the number of inhibited particles found on the node */
- size_t count_inhibited_parts = 0;
- size_t count_inhibited_gparts = 0;
- size_t count_inhibited_sparts = 0;
- size_t count_inhibited_bparts = 0;
- size_t count_inhibited_sinks = 0;
-
- /* Counter for the number of extra particles found on the node */
- size_t count_extra_parts = 0;
- size_t count_extra_gparts = 0;
- size_t count_extra_sparts = 0;
- size_t count_extra_bparts = 0;
- size_t count_extra_sinks = 0;
-
- /* Number of particles we expect to have after strays exchange */
- const size_t h_index_size = size_parts + space_expected_max_nr_strays;
- const size_t g_index_size = size_gparts + space_expected_max_nr_strays;
- const size_t s_index_size = size_sparts + space_expected_max_nr_strays;
- const size_t b_index_size = size_bparts + space_expected_max_nr_strays;
- const size_t sink_index_size = size_sinks + space_expected_max_nr_strays;
-
- /* Allocate arrays to store the indices of the cells where particles
- belong. We allocate extra space to allow for particles we may
- receive from other nodes */
- int *h_index = (int *)swift_malloc("h_index", sizeof(int) * h_index_size);
- int *g_index = (int *)swift_malloc("g_index", sizeof(int) * g_index_size);
- int *s_index = (int *)swift_malloc("s_index", sizeof(int) * s_index_size);
- int *b_index = (int *)swift_malloc("b_index", sizeof(int) * b_index_size);
- int *sink_index =
- (int *)swift_malloc("sink_index", sizeof(int) * sink_index_size);
- if (h_index == NULL || g_index == NULL || s_index == NULL ||
- b_index == NULL || sink_index == NULL)
- error("Failed to allocate temporary particle indices.");
-
- /* Allocate counters of particles that will land in each cell */
- int *cell_part_counts =
- (int *)swift_malloc("cell_part_counts", sizeof(int) * s->nr_cells);
- int *cell_gpart_counts =
- (int *)swift_malloc("cell_gpart_counts", sizeof(int) * s->nr_cells);
- int *cell_spart_counts =
- (int *)swift_malloc("cell_spart_counts", sizeof(int) * s->nr_cells);
- int *cell_bpart_counts =
- (int *)swift_malloc("cell_bpart_counts", sizeof(int) * s->nr_cells);
- int *cell_sink_counts =
- (int *)swift_malloc("cell_sink_counts", sizeof(int) * s->nr_cells);
-
- if (cell_part_counts == NULL || cell_gpart_counts == NULL ||
- cell_spart_counts == NULL || cell_bpart_counts == NULL ||
- cell_sink_counts == NULL)
- error("Failed to allocate cell particle count buffer.");
-
- /* Initialise the counters, including buffer space for future particles */
- for (int i = 0; i < s->nr_cells; ++i) {
- cell_part_counts[i] = 0;
- cell_gpart_counts[i] = 0;
- cell_spart_counts[i] = 0;
- cell_bpart_counts[i] = 0;
- cell_sink_counts[i] = 0;
- }
-
- /* Run through the particles and get their cell index. */
- if (nr_parts > 0)
- space_parts_get_cell_index(s, h_index, cell_part_counts,
- &count_inhibited_parts, &count_extra_parts,
- verbose);
- if (nr_gparts > 0)
- space_gparts_get_cell_index(s, g_index, cell_gpart_counts,
- &count_inhibited_gparts, &count_extra_gparts,
- verbose);
- if (nr_sparts > 0)
- space_sparts_get_cell_index(s, s_index, cell_spart_counts,
- &count_inhibited_sparts, &count_extra_sparts,
- verbose);
- if (nr_bparts > 0)
- space_bparts_get_cell_index(s, b_index, cell_bpart_counts,
- &count_inhibited_bparts, &count_extra_bparts,
- verbose);
- if (nr_sinks > 0)
- space_sinks_get_cell_index(s, sink_index, cell_sink_counts,
- &count_inhibited_sinks, &count_extra_sinks,
- verbose);
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Some safety checks */
- if (repartitioned && count_inhibited_parts)
- error("We just repartitioned but still found inhibited parts.");
- if (repartitioned && count_inhibited_sparts)
- error("We just repartitioned but still found inhibited sparts.");
- if (repartitioned && count_inhibited_gparts)
- error("We just repartitioned but still found inhibited gparts.");
- if (repartitioned && count_inhibited_bparts)
- error("We just repartitioned but still found inhibited bparts.");
- if (repartitioned && count_inhibited_sinks)
- error("We just repartitioned but still found inhibited sinks.");
-
- if (count_extra_parts != s->nr_extra_parts)
- error(
- "Number of extra parts in the part array not matching the space "
- "counter.");
- if (count_extra_gparts != s->nr_extra_gparts)
- error(
- "Number of extra gparts in the gpart array not matching the space "
- "counter.");
- if (count_extra_sparts != s->nr_extra_sparts)
- error(
- "Number of extra sparts in the spart array not matching the space "
- "counter.");
- if (count_extra_bparts != s->nr_extra_bparts)
- error(
- "Number of extra bparts in the bpart array not matching the space "
- "counter.");
- if (count_extra_sinks != s->nr_extra_sinks)
- error(
- "Number of extra sinks in the sink array not matching the space "
- "counter.");
-#endif
-
- const ticks tic2 = getticks();
-
- /* Move non-local parts and inhibited parts to the end of the list. */
- if (!repartitioned && (s->e->nr_nodes > 1 || count_inhibited_parts > 0)) {
-
- for (size_t k = 0; k < nr_parts; /* void */) {
-
- /* Inhibited particle or foreign particle */
- if (h_index[k] == -1 || cells_top[h_index[k]].nodeID != local_nodeID) {
-
- /* One fewer particle */
- nr_parts -= 1;
-
- /* Swap the particle */
- memswap(&s->parts[k], &s->parts[nr_parts], sizeof(struct part));
-
- /* Swap the link with the gpart */
- if (s->parts[k].gpart != NULL) {
- s->parts[k].gpart->id_or_neg_offset = -k;
- }
- if (s->parts[nr_parts].gpart != NULL) {
- s->parts[nr_parts].gpart->id_or_neg_offset = -nr_parts;
- }
-
- /* Swap the xpart */
- memswap(&s->xparts[k], &s->xparts[nr_parts], sizeof(struct xpart));
- /* Swap the index */
- memswap(&h_index[k], &h_index[nr_parts], sizeof(int));
-
- } else {
- /* Increment when not exchanging otherwise we need to retest "k".*/
- k++;
- }
- }
- }
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Check that all parts are in the correct places. */
- size_t check_count_inhibited_part = 0;
- for (size_t k = 0; k < nr_parts; k++) {
- if (h_index[k] == -1 || cells_top[h_index[k]].nodeID != local_nodeID) {
- error("Failed to move all non-local parts to send list");
- }
- }
- for (size_t k = nr_parts; k < s->nr_parts; k++) {
- if (h_index[k] != -1 && cells_top[h_index[k]].nodeID == local_nodeID) {
- error("Failed to remove local parts from send list");
- }
- if (h_index[k] == -1) ++check_count_inhibited_part;
- }
- if (check_count_inhibited_part != count_inhibited_parts)
- error("Counts of inhibited particles do not match!");
-#endif /* SWIFT_DEBUG_CHECKS */
-
- /* Move non-local sparts and inhibited sparts to the end of the list. */
- if (!repartitioned && (s->e->nr_nodes > 1 || count_inhibited_sparts > 0)) {
-
- for (size_t k = 0; k < nr_sparts; /* void */) {
-
- /* Inhibited particle or foreign particle */
- if (s_index[k] == -1 || cells_top[s_index[k]].nodeID != local_nodeID) {
-
- /* One fewer particle */
- nr_sparts -= 1;
-
- /* Swap the particle */
- memswap(&s->sparts[k], &s->sparts[nr_sparts], sizeof(struct spart));
-
- /* Swap the link with the gpart */
- if (s->sparts[k].gpart != NULL) {
- s->sparts[k].gpart->id_or_neg_offset = -k;
- }
- if (s->sparts[nr_sparts].gpart != NULL) {
- s->sparts[nr_sparts].gpart->id_or_neg_offset = -nr_sparts;
- }
-
- /* Swap the index */
- memswap(&s_index[k], &s_index[nr_sparts], sizeof(int));
-
- } else {
- /* Increment when not exchanging otherwise we need to retest "k".*/
- k++;
- }
- }
- }
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Check that all sparts are in the correct place. */
- size_t check_count_inhibited_spart = 0;
- for (size_t k = 0; k < nr_sparts; k++) {
- if (s_index[k] == -1 || cells_top[s_index[k]].nodeID != local_nodeID) {
- error("Failed to move all non-local sparts to send list");
- }
- }
- for (size_t k = nr_sparts; k < s->nr_sparts; k++) {
- if (s_index[k] != -1 && cells_top[s_index[k]].nodeID == local_nodeID) {
- error("Failed to remove local sparts from send list");
- }
- if (s_index[k] == -1) ++check_count_inhibited_spart;
- }
- if (check_count_inhibited_spart != count_inhibited_sparts)
- error("Counts of inhibited s-particles do not match!");
-#endif /* SWIFT_DEBUG_CHECKS */
-
- /* Move non-local bparts and inhibited bparts to the end of the list. */
- if (!repartitioned && (s->e->nr_nodes > 1 || count_inhibited_bparts > 0)) {
-
- for (size_t k = 0; k < nr_bparts; /* void */) {
-
- /* Inhibited particle or foreign particle */
- if (b_index[k] == -1 || cells_top[b_index[k]].nodeID != local_nodeID) {
-
- /* One fewer particle */
- nr_bparts -= 1;
-
- /* Swap the particle */
- memswap(&s->bparts[k], &s->bparts[nr_bparts], sizeof(struct bpart));
-
- /* Swap the link with the gpart */
- if (s->bparts[k].gpart != NULL) {
- s->bparts[k].gpart->id_or_neg_offset = -k;
- }
- if (s->bparts[nr_bparts].gpart != NULL) {
- s->bparts[nr_bparts].gpart->id_or_neg_offset = -nr_bparts;
- }
-
- /* Swap the index */
- memswap(&b_index[k], &b_index[nr_bparts], sizeof(int));
-
- } else {
- /* Increment when not exchanging otherwise we need to retest "k".*/
- k++;
- }
- }
- }
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Check that all bparts are in the correct place. */
- size_t check_count_inhibited_bpart = 0;
- for (size_t k = 0; k < nr_bparts; k++) {
- if (b_index[k] == -1 || cells_top[b_index[k]].nodeID != local_nodeID) {
- error("Failed to move all non-local bparts to send list");
- }
- }
- for (size_t k = nr_bparts; k < s->nr_bparts; k++) {
- if (b_index[k] != -1 && cells_top[b_index[k]].nodeID == local_nodeID) {
- error("Failed to remove local bparts from send list");
- }
- if (b_index[k] == -1) ++check_count_inhibited_bpart;
- }
- if (check_count_inhibited_bpart != count_inhibited_bparts)
- error("Counts of inhibited b-particles do not match!");
-#endif /* SWIFT_DEBUG_CHECKS */
-
- /* Move non-local sinks and inhibited sinks to the end of the list. */
- if (!repartitioned && (s->e->nr_nodes > 1 || count_inhibited_sinks > 0)) {
-
- for (size_t k = 0; k < nr_sinks; /* void */) {
-
- /* Inhibited particle or foreign particle */
- if (sink_index[k] == -1 ||
- cells_top[sink_index[k]].nodeID != local_nodeID) {
-
- /* One fewer particle */
- nr_sinks -= 1;
-
- /* Swap the particle */
- memswap(&s->sinks[k], &s->sinks[nr_sinks], sizeof(struct sink));
-
- /* Swap the link with the gpart */
- if (s->sinks[k].gpart != NULL) {
- s->sinks[k].gpart->id_or_neg_offset = -k;
- }
- if (s->sinks[nr_sinks].gpart != NULL) {
- s->sinks[nr_sinks].gpart->id_or_neg_offset = -nr_sinks;
- }
-
- /* Swap the index */
- memswap(&sink_index[k], &sink_index[nr_sinks], sizeof(int));
-
- } else {
- /* Increment when not exchanging otherwise we need to retest "k".*/
- k++;
- }
- }
- }
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Check that all sinks are in the correct place. */
- size_t check_count_inhibited_sinks = 0;
- for (size_t k = 0; k < nr_sinks; k++) {
- if (sink_index[k] == -1 ||
- cells_top[sink_index[k]].nodeID != local_nodeID) {
- error("Failed to move all non-local sinks to send list");
- }
- }
- for (size_t k = nr_sinks; k < s->nr_sinks; k++) {
- if (sink_index[k] != -1 &&
- cells_top[sink_index[k]].nodeID == local_nodeID) {
- error("Failed to remove local sinks from send list");
- }
- if (sink_index[k] == -1) ++check_count_inhibited_sinks;
- }
- if (check_count_inhibited_sinks != count_inhibited_sinks)
- error("Counts of inhibited sink-particles do not match!");
-#endif /* SWIFT_DEBUG_CHECKS */
-
- /* Move non-local gparts and inhibited parts to the end of the list. */
- if (!repartitioned && (s->e->nr_nodes > 1 || count_inhibited_gparts > 0)) {
-
- for (size_t k = 0; k < nr_gparts; /* void */) {
-
- /* Inhibited particle or foreign particle */
- if (g_index[k] == -1 || cells_top[g_index[k]].nodeID != local_nodeID) {
-
- /* One fewer particle */
- nr_gparts -= 1;
-
- /* Swap the particle */
- memswap_unaligned(&s->gparts[k], &s->gparts[nr_gparts],
- sizeof(struct gpart));
-
- /* Swap the link with part/spart */
- if (s->gparts[k].type == swift_type_gas) {
- s->parts[-s->gparts[k].id_or_neg_offset].gpart = &s->gparts[k];
- } else if (s->gparts[k].type == swift_type_stars) {
- s->sparts[-s->gparts[k].id_or_neg_offset].gpart = &s->gparts[k];
- } else if (s->gparts[k].type == swift_type_sink) {
- s->sparts[-s->gparts[k].id_or_neg_offset].gpart = &s->gparts[k];
- } else if (s->gparts[k].type == swift_type_black_hole) {
- s->bparts[-s->gparts[k].id_or_neg_offset].gpart = &s->gparts[k];
- }
-
- if (s->gparts[nr_gparts].type == swift_type_gas) {
- s->parts[-s->gparts[nr_gparts].id_or_neg_offset].gpart =
- &s->gparts[nr_gparts];
- } else if (s->gparts[nr_gparts].type == swift_type_stars) {
- s->sparts[-s->gparts[nr_gparts].id_or_neg_offset].gpart =
- &s->gparts[nr_gparts];
- } else if (s->gparts[nr_gparts].type == swift_type_sink) {
- s->sparts[-s->gparts[nr_gparts].id_or_neg_offset].gpart =
- &s->gparts[nr_gparts];
- } else if (s->gparts[nr_gparts].type == swift_type_black_hole) {
- s->bparts[-s->gparts[nr_gparts].id_or_neg_offset].gpart =
- &s->gparts[nr_gparts];
- }
-
- /* Swap the index */
- memswap(&g_index[k], &g_index[nr_gparts], sizeof(int));
- } else {
- /* Increment when not exchanging otherwise we need to retest "k".*/
- k++;
- }
- }
- }
-
- if (verbose)
- message("Moving non-local particles took %.3f %s.",
- clocks_from_ticks(getticks() - tic2), clocks_getunit());
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Check that all gparts are in the correct place. */
- size_t check_count_inhibited_gpart = 0;
- for (size_t k = 0; k < nr_gparts; k++) {
- if (g_index[k] == -1 || cells_top[g_index[k]].nodeID != local_nodeID) {
- error("Failed to move all non-local gparts to send list");
- }
- }
- for (size_t k = nr_gparts; k < s->nr_gparts; k++) {
- if (g_index[k] != -1 && cells_top[g_index[k]].nodeID == local_nodeID) {
- error("Failed to remove local gparts from send list");
- }
- if (g_index[k] == -1) ++check_count_inhibited_gpart;
- }
- if (check_count_inhibited_gpart != count_inhibited_gparts)
- error("Counts of inhibited g-particles do not match!");
-#endif /* SWIFT_DEBUG_CHECKS */
-
-#ifdef WITH_MPI
-
- /* Exchange the strays, note that this potentially re-allocates
- the parts arrays. This can be skipped if we just repartitioned space
- as there should be no strays in that case */
- if (!repartitioned) {
-
- size_t nr_parts_exchanged = s->nr_parts - nr_parts;
- size_t nr_gparts_exchanged = s->nr_gparts - nr_gparts;
- size_t nr_sparts_exchanged = s->nr_sparts - nr_sparts;
- size_t nr_bparts_exchanged = s->nr_bparts - nr_bparts;
- engine_exchange_strays(s->e, nr_parts, &h_index[nr_parts],
- &nr_parts_exchanged, nr_gparts, &g_index[nr_gparts],
- &nr_gparts_exchanged, nr_sparts, &s_index[nr_sparts],
- &nr_sparts_exchanged, nr_bparts, &b_index[nr_bparts],
- &nr_bparts_exchanged);
-
- /* Set the new particle counts. */
- s->nr_parts = nr_parts + nr_parts_exchanged;
- s->nr_gparts = nr_gparts + nr_gparts_exchanged;
- s->nr_sparts = nr_sparts + nr_sparts_exchanged;
- s->nr_bparts = nr_bparts + nr_bparts_exchanged;
-
- } else {
-#ifdef SWIFT_DEBUG_CHECKS
- if (s->nr_parts != nr_parts)
- error("Number of parts changing after repartition");
- if (s->nr_sparts != nr_sparts)
- error("Number of sparts changing after repartition");
- if (s->nr_gparts != nr_gparts)
- error("Number of gparts changing after repartition");
-#endif
- }
-
- /* Clear non-local cell counts. */
- for (int k = 0; k < s->nr_cells; k++) {
- if (s->cells_top[k].nodeID != local_nodeID) {
- cell_part_counts[k] = 0;
- cell_spart_counts[k] = 0;
- cell_gpart_counts[k] = 0;
- cell_bpart_counts[k] = 0;
- }
- }
-
- /* Re-allocate the index array for the parts if needed.. */
- if (s->nr_parts + 1 > h_index_size) {
- int *ind_new;
- if ((ind_new = (int *)swift_malloc(
- "h_index", sizeof(int) * (s->nr_parts + 1))) == NULL)
- error("Failed to allocate temporary particle indices.");
- memcpy(ind_new, h_index, sizeof(int) * nr_parts);
- swift_free("h_index", h_index);
- h_index = ind_new;
- }
-
- /* Re-allocate the index array for the sparts if needed.. */
- if (s->nr_sparts + 1 > s_index_size) {
- int *sind_new;
- if ((sind_new = (int *)swift_malloc(
- "s_index", sizeof(int) * (s->nr_sparts + 1))) == NULL)
- error("Failed to allocate temporary s-particle indices.");
- memcpy(sind_new, s_index, sizeof(int) * nr_sparts);
- swift_free("s_index", s_index);
- s_index = sind_new;
- }
-
- /* Re-allocate the index array for the bparts if needed.. */
- if (s->nr_bparts + 1 > s_index_size) {
- int *bind_new;
- if ((bind_new = (int *)swift_malloc(
- "b_index", sizeof(int) * (s->nr_bparts + 1))) == NULL)
- error("Failed to allocate temporary s-particle indices.");
- memcpy(bind_new, b_index, sizeof(int) * nr_bparts);
- swift_free("b_index", b_index);
- b_index = bind_new;
- }
-
- const int cdim[3] = {s->cdim[0], s->cdim[1], s->cdim[2]};
- const double ih[3] = {s->iwidth[0], s->iwidth[1], s->iwidth[2]};
-
- /* Assign each received part to its cell. */
- for (size_t k = nr_parts; k < s->nr_parts; k++) {
- const struct part *const p = &s->parts[k];
- h_index[k] =
- cell_getid(cdim, p->x[0] * ih[0], p->x[1] * ih[1], p->x[2] * ih[2]);
- cell_part_counts[h_index[k]]++;
-#ifdef SWIFT_DEBUG_CHECKS
- if (cells_top[h_index[k]].nodeID != local_nodeID)
- error("Received part that does not belong to me (nodeID=%i).",
- cells_top[h_index[k]].nodeID);
-#endif
- }
- nr_parts = s->nr_parts;
-
- /* Assign each received spart to its cell. */
- for (size_t k = nr_sparts; k < s->nr_sparts; k++) {
- const struct spart *const sp = &s->sparts[k];
- s_index[k] =
- cell_getid(cdim, sp->x[0] * ih[0], sp->x[1] * ih[1], sp->x[2] * ih[2]);
- cell_spart_counts[s_index[k]]++;
-#ifdef SWIFT_DEBUG_CHECKS
- if (cells_top[s_index[k]].nodeID != local_nodeID)
- error("Received s-part that does not belong to me (nodeID=%i).",
- cells_top[s_index[k]].nodeID);
-#endif
- }
- nr_sparts = s->nr_sparts;
-
- /* Assign each received bpart to its cell. */
- for (size_t k = nr_bparts; k < s->nr_bparts; k++) {
- const struct bpart *const bp = &s->bparts[k];
- b_index[k] =
- cell_getid(cdim, bp->x[0] * ih[0], bp->x[1] * ih[1], bp->x[2] * ih[2]);
- cell_bpart_counts[b_index[k]]++;
-#ifdef SWIFT_DEBUG_CHECKS
- if (cells_top[b_index[k]].nodeID != local_nodeID)
- error("Received s-part that does not belong to me (nodeID=%i).",
- cells_top[b_index[k]].nodeID);
-#endif
- }
- nr_bparts = s->nr_bparts;
-
-#else /* WITH_MPI */
-
- /* Update the part, spart and bpart counters */
- s->nr_parts = nr_parts;
- s->nr_sparts = nr_sparts;
- s->nr_bparts = nr_bparts;
- s->nr_sinks = nr_sinks;
-
-#endif /* WITH_MPI */
-
- /* Sort the parts according to their cells. */
- if (nr_parts > 0)
- space_parts_sort(s->parts, s->xparts, h_index, cell_part_counts,
- s->nr_cells, 0);
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Verify that the part have been sorted correctly. */
- for (size_t k = 0; k < nr_parts; k++) {
- const struct part *p = &s->parts[k];
-
- if (p->time_bin == time_bin_inhibited)
- error("Inhibited particle sorted into a cell!");
-
- /* New cell index */
- const int new_ind =
- cell_getid(s->cdim, p->x[0] * s->iwidth[0], p->x[1] * s->iwidth[1],
- p->x[2] * s->iwidth[2]);
-
- /* New cell of this part */
- const struct cell *c = &s->cells_top[new_ind];
-
- if (h_index[k] != new_ind)
- error("part's new cell index not matching sorted index.");
-
- if (p->x[0] < c->loc[0] || p->x[0] > c->loc[0] + c->width[0] ||
- p->x[1] < c->loc[1] || p->x[1] > c->loc[1] + c->width[1] ||
- p->x[2] < c->loc[2] || p->x[2] > c->loc[2] + c->width[2])
- error("part not sorted into the right top-level cell!");
- }
-#endif /* SWIFT_DEBUG_CHECKS */
-
- /* Sort the sparts according to their cells. */
- if (nr_sparts > 0)
- space_sparts_sort(s->sparts, s_index, cell_spart_counts, s->nr_cells, 0);
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Verify that the spart have been sorted correctly. */
- for (size_t k = 0; k < nr_sparts; k++) {
- const struct spart *sp = &s->sparts[k];
-
- if (sp->time_bin == time_bin_inhibited)
- error("Inhibited particle sorted into a cell!");
-
- /* New cell index */
- const int new_sind =
- cell_getid(s->cdim, sp->x[0] * s->iwidth[0], sp->x[1] * s->iwidth[1],
- sp->x[2] * s->iwidth[2]);
-
- /* New cell of this spart */
- const struct cell *c = &s->cells_top[new_sind];
-
- if (s_index[k] != new_sind)
- error("spart's new cell index not matching sorted index.");
-
- if (sp->x[0] < c->loc[0] || sp->x[0] > c->loc[0] + c->width[0] ||
- sp->x[1] < c->loc[1] || sp->x[1] > c->loc[1] + c->width[1] ||
- sp->x[2] < c->loc[2] || sp->x[2] > c->loc[2] + c->width[2])
- error("spart not sorted into the right top-level cell!");
- }
-#endif /* SWIFT_DEBUG_CHECKS */
-
- /* Sort the bparts according to their cells. */
- if (nr_bparts > 0)
- space_bparts_sort(s->bparts, b_index, cell_bpart_counts, s->nr_cells, 0);
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Verify that the bpart have been sorted correctly. */
- for (size_t k = 0; k < nr_bparts; k++) {
- const struct bpart *bp = &s->bparts[k];
-
- if (bp->time_bin == time_bin_inhibited)
- error("Inhibited particle sorted into a cell!");
-
- /* New cell index */
- const int new_bind =
- cell_getid(s->cdim, bp->x[0] * s->iwidth[0], bp->x[1] * s->iwidth[1],
- bp->x[2] * s->iwidth[2]);
-
- /* New cell of this bpart */
- const struct cell *c = &s->cells_top[new_bind];
-
- if (b_index[k] != new_bind)
- error("bpart's new cell index not matching sorted index.");
-
- if (bp->x[0] < c->loc[0] || bp->x[0] > c->loc[0] + c->width[0] ||
- bp->x[1] < c->loc[1] || bp->x[1] > c->loc[1] + c->width[1] ||
- bp->x[2] < c->loc[2] || bp->x[2] > c->loc[2] + c->width[2])
- error("bpart not sorted into the right top-level cell!");
- }
-#endif /* SWIFT_DEBUG_CHECKS */
-
- /* Sort the sink according to their cells. */
- if (nr_sinks > 0)
- space_sinks_sort(s->sinks, sink_index, cell_sink_counts, s->nr_cells, 0);
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Verify that the sink have been sorted correctly. */
- for (size_t k = 0; k < nr_sinks; k++) {
- const struct sink *sink = &s->sinks[k];
-
- if (sink->time_bin == time_bin_inhibited)
- error("Inhibited particle sorted into a cell!");
-
- /* New cell index */
- const int new_bind =
- cell_getid(s->cdim, sink->x[0] * s->iwidth[0],
- sink->x[1] * s->iwidth[1], sink->x[2] * s->iwidth[2]);
-
- /* New cell of this sink */
- const struct cell *c = &s->cells_top[new_bind];
-
- if (sink_index[k] != new_bind)
- error("sink's new cell index not matching sorted index.");
-
- if (sink->x[0] < c->loc[0] || sink->x[0] > c->loc[0] + c->width[0] ||
- sink->x[1] < c->loc[1] || sink->x[1] > c->loc[1] + c->width[1] ||
- sink->x[2] < c->loc[2] || sink->x[2] > c->loc[2] + c->width[2])
- error("sink not sorted into the right top-level cell!");
- }
-#endif /* SWIFT_DEBUG_CHECKS */
-
- /* Extract the cell counts from the sorted indices. Deduct the extra
- * particles. */
- size_t last_index = 0;
- h_index[nr_parts] = s->nr_cells; // sentinel.
- for (size_t k = 0; k < nr_parts; k++) {
- if (h_index[k] < h_index[k + 1]) {
- cells_top[h_index[k]].hydro.count =
- k - last_index + 1 - space_extra_parts;
- last_index = k + 1;
- }
- }
-
- /* Extract the cell counts from the sorted indices. Deduct the extra
- * particles. */
- size_t last_sindex = 0;
- s_index[nr_sparts] = s->nr_cells; // sentinel.
- for (size_t k = 0; k < nr_sparts; k++) {
- if (s_index[k] < s_index[k + 1]) {
- cells_top[s_index[k]].stars.count =
- k - last_sindex + 1 - space_extra_sparts;
- last_sindex = k + 1;
- }
- }
-
- /* Extract the cell counts from the sorted indices. Deduct the extra
- * particles. */
- size_t last_bindex = 0;
- b_index[nr_bparts] = s->nr_cells; // sentinel.
- for (size_t k = 0; k < nr_bparts; k++) {
- if (b_index[k] < b_index[k + 1]) {
- cells_top[b_index[k]].black_holes.count =
- k - last_bindex + 1 - space_extra_bparts;
- last_bindex = k + 1;
- }
- }
-
- /* Extract the cell counts from the sorted indices. Deduct the extra
- * particles. */
- size_t last_sink_index = 0;
- sink_index[nr_sinks] = s->nr_cells; // sentinel.
- for (size_t k = 0; k < nr_sinks; k++) {
- if (sink_index[k] < sink_index[k + 1]) {
- cells_top[sink_index[k]].sinks.count =
- k - last_sink_index + 1 - space_extra_sinks;
- last_sink_index = k + 1;
- }
- }
-
- /* We no longer need the indices as of here. */
- swift_free("h_index", h_index);
- swift_free("cell_part_counts", cell_part_counts);
- swift_free("s_index", s_index);
- swift_free("cell_spart_counts", cell_spart_counts);
- swift_free("b_index", b_index);
- swift_free("cell_bpart_counts", cell_bpart_counts);
- swift_free("sink_index", sink_index);
- swift_free("cell_sink_counts", cell_sink_counts);
-
- /* Update the slice of unique IDs. */
- space_update_unique_id(s);
-
-#ifdef WITH_MPI
-
- /* Re-allocate the index array for the gparts if needed.. */
- if (s->nr_gparts + 1 > g_index_size) {
- int *gind_new;
- if ((gind_new = (int *)swift_malloc(
- "g_index", sizeof(int) * (s->nr_gparts + 1))) == NULL)
- error("Failed to allocate temporary g-particle indices.");
- memcpy(gind_new, g_index, sizeof(int) * nr_gparts);
- swift_free("g_index", g_index);
- g_index = gind_new;
- }
-
- /* Assign each received gpart to its cell. */
- for (size_t k = nr_gparts; k < s->nr_gparts; k++) {
- const struct gpart *const p = &s->gparts[k];
- g_index[k] =
- cell_getid(cdim, p->x[0] * ih[0], p->x[1] * ih[1], p->x[2] * ih[2]);
- cell_gpart_counts[g_index[k]]++;
-#ifdef SWIFT_DEBUG_CHECKS
- if (cells_top[g_index[k]].nodeID != s->e->nodeID)
- error("Received g-part that does not belong to me (nodeID=%i).",
- cells_top[g_index[k]].nodeID);
-#endif
- }
- nr_gparts = s->nr_gparts;
-
-#else /* WITH_MPI */
-
- /* Update the gpart counter */
- s->nr_gparts = nr_gparts;
-
-#endif /* WITH_MPI */
-
- /* Mark that there are no inhibited particles left */
- s->nr_inhibited_parts = 0;
- s->nr_inhibited_gparts = 0;
- s->nr_inhibited_sparts = 0;
- s->nr_inhibited_bparts = 0;
- s->nr_inhibited_sinks = 0;
-
- /* Sort the gparts according to their cells. */
- if (nr_gparts > 0)
- space_gparts_sort(s->gparts, s->parts, s->sinks, s->sparts, s->bparts,
- g_index, cell_gpart_counts, s->nr_cells);
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Verify that the gpart have been sorted correctly. */
- for (size_t k = 0; k < nr_gparts; k++) {
- const struct gpart *gp = &s->gparts[k];
-
- if (gp->time_bin == time_bin_inhibited)
- error("Inhibited particle sorted into a cell!");
-
- /* New cell index */
- const int new_gind =
- cell_getid(s->cdim, gp->x[0] * s->iwidth[0], gp->x[1] * s->iwidth[1],
- gp->x[2] * s->iwidth[2]);
-
- /* New cell of this gpart */
- const struct cell *c = &s->cells_top[new_gind];
-
- if (g_index[k] != new_gind)
- error("gpart's new cell index not matching sorted index.");
-
- if (gp->x[0] < c->loc[0] || gp->x[0] > c->loc[0] + c->width[0] ||
- gp->x[1] < c->loc[1] || gp->x[1] > c->loc[1] + c->width[1] ||
- gp->x[2] < c->loc[2] || gp->x[2] > c->loc[2] + c->width[2])
- error("gpart not sorted into the right top-level cell!");
- }
-#endif /* SWIFT_DEBUG_CHECKS */
-
- /* Extract the cell counts from the sorted indices. Deduct the extra
- * particles. */
- size_t last_gindex = 0;
- g_index[nr_gparts] = s->nr_cells;
- for (size_t k = 0; k < nr_gparts; k++) {
- if (g_index[k] < g_index[k + 1]) {
- cells_top[g_index[k]].grav.count =
- k - last_gindex + 1 - space_extra_gparts;
- last_gindex = k + 1;
- }
- }
-
- /* We no longer need the indices as of here. */
- swift_free("g_index", g_index);
- swift_free("cell_gpart_counts", cell_gpart_counts);
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Verify that the links are correct */
- if ((nr_gparts > 0 && nr_parts > 0) || (nr_gparts > 0 && nr_sparts > 0) ||
- (nr_gparts > 0 && nr_bparts > 0) || (nr_gparts > 0 && nr_sinks > 0))
- part_verify_links(s->parts, s->gparts, s->sinks, s->sparts, s->bparts,
- nr_parts, nr_gparts, nr_sinks, nr_sparts, nr_bparts,
- verbose);
-#endif
-
- /* Hook the cells up to the parts. Make list of local and non-empty cells */
- const ticks tic3 = getticks();
- struct part *finger = s->parts;
- struct xpart *xfinger = s->xparts;
- struct gpart *gfinger = s->gparts;
- struct spart *sfinger = s->sparts;
- struct bpart *bfinger = s->bparts;
- struct sink *sink_finger = s->sinks;
- s->nr_cells_with_particles = 0;
- s->nr_local_cells_with_particles = 0;
- s->nr_local_cells = 0;
- for (int k = 0; k < s->nr_cells; k++) {
- struct cell *restrict c = &cells_top[k];
- c->hydro.ti_old_part = ti_current;
- c->grav.ti_old_part = ti_current;
- c->grav.ti_old_multipole = ti_current;
- c->stars.ti_old_part = ti_current;
- c->sinks.ti_old_part = ti_current;
- c->black_holes.ti_old_part = ti_current;
-
-#if defined(SWIFT_DEBUG_CHECKS) || defined(SWIFT_CELL_GRAPH)
- c->cellID = -last_cell_id;
- last_cell_id++;
-#endif
-
- const int is_local = (c->nodeID == engine_rank);
- const int has_particles =
- (c->hydro.count > 0) || (c->grav.count > 0) || (c->stars.count > 0) ||
- (c->black_holes.count > 0) || (c->sinks.count > 0);
-
- if (is_local) {
- c->hydro.parts = finger;
- c->hydro.xparts = xfinger;
- c->grav.parts = gfinger;
- c->stars.parts = sfinger;
- c->black_holes.parts = bfinger;
- c->sinks.parts = sink_finger;
-
- /* Store the state at rebuild time */
- c->stars.parts_rebuild = c->stars.parts;
- c->grav.parts_rebuild = c->grav.parts;
-
- c->hydro.count_total = c->hydro.count + space_extra_parts;
- c->grav.count_total = c->grav.count + space_extra_gparts;
- c->stars.count_total = c->stars.count + space_extra_sparts;
- c->sinks.count_total = c->sinks.count + space_extra_sinks;
- c->black_holes.count_total = c->black_holes.count + space_extra_bparts;
-
- finger = &finger[c->hydro.count_total];
- xfinger = &xfinger[c->hydro.count_total];
- gfinger = &gfinger[c->grav.count_total];
- sfinger = &sfinger[c->stars.count_total];
- bfinger = &bfinger[c->black_holes.count_total];
- sink_finger = &sink_finger[c->sinks.count_total];
-
- /* Add this cell to the list of local cells */
- s->local_cells_top[s->nr_local_cells] = k;
- s->nr_local_cells++;
- }
-
- if (is_local && has_particles) {
-
- /* Add this cell to the list of non-empty cells */
- s->local_cells_with_particles_top[s->nr_local_cells_with_particles] = k;
- s->nr_local_cells_with_particles++;
- }
- }
- if (verbose) {
- message("Have %d local top-level cells with particles (total=%d)",
- s->nr_local_cells_with_particles, s->nr_cells);
- message("Have %d local top-level cells (total=%d)", s->nr_local_cells,
- s->nr_cells);
- message("hooking up cells took %.3f %s.",
- clocks_from_ticks(getticks() - tic3), clocks_getunit());
- }
-
- /* Re-order the extra particles such that they are at the end of their cell's
- memory pool. */
- if (s->with_star_formation || s->e->policy & engine_policy_sinks)
- space_reorder_extras(s, verbose);
-
- /* At this point, we have the upper-level cells. Now recursively split each
- cell to get the full AMR grid. */
- space_split(s, verbose);
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Check that the multipole construction went OK */
- if (s->with_self_gravity)
- for (int k = 0; k < s->nr_cells; k++)
- cell_check_multipole(&s->cells_top[k], s->e->gravity_properties);
-#endif
-
- /* Clean up any stray sort indices in the cell buffer. */
- space_free_buff_sort_indices(s);
-
- if (verbose)
- message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
- clocks_getunit());
-}
-
-/**
- * @brief Split particles between cells of a hierarchy.
- *
- * This is done in parallel using threads in the #threadpool.
- * Only do this for the local non-empty top-level cells.
- *
- * @param s The #space.
- * @param verbose Are we talkative ?
- */
-void space_split(struct space *s, int verbose) {
-
- const ticks tic = getticks();
-
- threadpool_map(&s->e->threadpool, space_split_mapper,
- s->local_cells_with_particles_top,
- s->nr_local_cells_with_particles, sizeof(int),
- threadpool_auto_chunk_size, s);
-
- if (verbose)
- message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
- clocks_getunit());
-}
-
-void space_reorder_extra_parts_mapper(void *map_data, int num_cells,
- void *extra_data) {
- int *local_cells = (int *)map_data;
- struct space *s = (struct space *)extra_data;
- struct cell *cells_top = s->cells_top;
-
- for (int ind = 0; ind < num_cells; ind++) {
- struct cell *c = &cells_top[local_cells[ind]];
- cell_reorder_extra_parts(c, c->hydro.parts - s->parts);
- }
-}
-
-void space_reorder_extra_gparts_mapper(void *map_data, int num_cells,
- void *extra_data) {
-
- int *local_cells = (int *)map_data;
- struct space *s = (struct space *)extra_data;
- struct cell *cells_top = s->cells_top;
-
- for (int ind = 0; ind < num_cells; ind++) {
- struct cell *c = &cells_top[local_cells[ind]];
- cell_reorder_extra_gparts(c, s->parts, s->sparts);
- }
-}
-
-void space_reorder_extra_sparts_mapper(void *map_data, int num_cells,
- void *extra_data) {
-
- int *local_cells = (int *)map_data;
- struct space *s = (struct space *)extra_data;
- struct cell *cells_top = s->cells_top;
-
- for (int ind = 0; ind < num_cells; ind++) {
- struct cell *c = &cells_top[local_cells[ind]];
- cell_reorder_extra_sparts(c, c->stars.parts - s->sparts);
- }
-}
-
-void space_reorder_extra_sinks_mapper(void *map_data, int num_cells,
- void *extra_data) {
-
- int *local_cells = (int *)map_data;
- struct space *s = (struct space *)extra_data;
- struct cell *cells_top = s->cells_top;
-
- for (int ind = 0; ind < num_cells; ind++) {
- struct cell *c = &cells_top[local_cells[ind]];
- cell_reorder_extra_sinks(c, c->sinks.parts - s->sinks);
- }
-}
-
-/**
- * @brief Re-orders the particles in each cell such that the extra particles
- * for on-the-fly creation are located at the end of their respective cells.
- *
- * This assumes that all the particles (real and extra) have already been sorted
- * in their correct top-level cell.
- *
- * @param s The #space to act upon.
- * @param verbose Are we talkative?
- */
-void space_reorder_extras(struct space *s, int verbose) {
-
- /* Re-order the gas particles */
- if (space_extra_parts)
- threadpool_map(&s->e->threadpool, space_reorder_extra_parts_mapper,
- s->local_cells_top, s->nr_local_cells, sizeof(int),
- threadpool_auto_chunk_size, s);
-
- /* Re-order the gravity particles */
- if (space_extra_gparts)
- threadpool_map(&s->e->threadpool, space_reorder_extra_gparts_mapper,
- s->local_cells_top, s->nr_local_cells, sizeof(int),
- threadpool_auto_chunk_size, s);
-
- /* Re-order the star particles */
- if (space_extra_sparts)
- threadpool_map(&s->e->threadpool, space_reorder_extra_sparts_mapper,
- s->local_cells_top, s->nr_local_cells, sizeof(int),
- threadpool_auto_chunk_size, s);
-
- /* Re-order the black hole particles */
- if (space_extra_bparts)
- error("Missing implementation of BH extra reordering");
-
- /* Re-order the sink particles */
- if (space_extra_sinks)
- threadpool_map(&s->e->threadpool, space_reorder_extra_sinks_mapper,
- s->local_cells_top, s->nr_local_cells, sizeof(int),
- threadpool_auto_chunk_size, s);
-}
-
-/**
- * @brief #threadpool mapper function to sanitize the cells
- *
- * @param map_data Pointers towards the top-level cells.
- * @param num_cells The number of top-level cells.
- * @param extra_data Unused parameters.
- */
-void space_sanitize_mapper(void *map_data, int num_cells, void *extra_data) {
- /* Unpack the inputs. */
- struct cell *cells_top = (struct cell *)map_data;
-
- for (int ind = 0; ind < num_cells; ind++) {
- struct cell *c = &cells_top[ind];
- cell_sanitize(c, 0);
- }
-}
-
-/**
- * @brief Runs through the top-level cells and sanitize their h values
- *
- * @param s The #space to act upon.
- */
-void space_sanitize(struct space *s) {
-
- if (s->e->nodeID == 0) message("Cleaning up unreasonable values of h");
-
- threadpool_map(&s->e->threadpool, space_sanitize_mapper, s->cells_top,
- s->nr_cells, sizeof(struct cell), threadpool_auto_chunk_size,
- /*extra_data=*/NULL);
-}
-
-/**
- * @brief #threadpool mapper function to compute the particle cell indices.
- *
- * @param map_data Pointer towards the particles.
- * @param nr_parts The number of particles to treat.
- * @param extra_data Pointers to the space and index list
- */
-void space_parts_get_cell_index_mapper(void *map_data, int nr_parts,
- void *extra_data) {
-
- /* Unpack the data */
- struct part *restrict parts = (struct part *)map_data;
- struct index_data *data = (struct index_data *)extra_data;
- struct space *s = data->s;
- int *const ind = data->ind + (ptrdiff_t)(parts - s->parts);
-
- /* Get some constants */
- const double dim_x = s->dim[0];
- const double dim_y = s->dim[1];
- const double dim_z = s->dim[2];
- const int cdim[3] = {s->cdim[0], s->cdim[1], s->cdim[2]};
- const double ih_x = s->iwidth[0];
- const double ih_y = s->iwidth[1];
- const double ih_z = s->iwidth[2];
-
- /* Init the local count buffer. */
- int *cell_counts = (int *)calloc(sizeof(int), s->nr_cells);
- if (cell_counts == NULL)
- error("Failed to allocate temporary cell count buffer.");
-
- /* Init the local collectors */
- float min_mass = FLT_MAX;
- float sum_vel_norm = 0.f;
- size_t count_inhibited_part = 0;
- size_t count_extra_part = 0;
-
- /* Loop over the parts. */
- for (int k = 0; k < nr_parts; k++) {
-
- /* Get the particle */
- struct part *restrict p = &parts[k];
-
- double old_pos_x = p->x[0];
- double old_pos_y = p->x[1];
- double old_pos_z = p->x[2];
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (!s->periodic && p->time_bin != time_bin_inhibited) {
- if (old_pos_x < 0. || old_pos_x > dim_x)
- error("Particle outside of volume along X.");
- if (old_pos_y < 0. || old_pos_y > dim_y)
- error("Particle outside of volume along Y.");
- if (old_pos_z < 0. || old_pos_z > dim_z)
- error("Particle outside of volume along Z.");
- }
-#endif
-
- /* Put it back into the simulation volume */
- double pos_x = box_wrap(old_pos_x, 0.0, dim_x);
- double pos_y = box_wrap(old_pos_y, 0.0, dim_y);
- double pos_z = box_wrap(old_pos_z, 0.0, dim_z);
-
- /* Treat the case where a particle was wrapped back exactly onto
- * the edge because of rounding issues (more accuracy around 0
- * than around dim) */
- if (pos_x == dim_x) pos_x = 0.0;
- if (pos_y == dim_y) pos_y = 0.0;
- if (pos_z == dim_z) pos_z = 0.0;
-
- /* Get its cell index */
- const int index =
- cell_getid(cdim, pos_x * ih_x, pos_y * ih_y, pos_z * ih_z);
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (index < 0 || index >= cdim[0] * cdim[1] * cdim[2])
- error("Invalid index=%d cdim=[%d %d %d] p->x=[%e %e %e]", index, cdim[0],
- cdim[1], cdim[2], pos_x, pos_y, pos_z);
-
- if (pos_x >= dim_x || pos_y >= dim_y || pos_z >= dim_z || pos_x < 0. ||
- pos_y < 0. || pos_z < 0.)
- error("Particle outside of simulation box. p->x=[%e %e %e]", pos_x, pos_y,
- pos_z);
-#endif
-
- if (p->time_bin == time_bin_inhibited) {
- /* Is this particle to be removed? */
- ind[k] = -1;
- ++count_inhibited_part;
- } else if (p->time_bin == time_bin_not_created) {
- /* Is this a place-holder for on-the-fly creation? */
- ind[k] = index;
- cell_counts[index]++;
- ++count_extra_part;
-
- } else {
- /* Normal case: list its top-level cell index */
- ind[k] = index;
- cell_counts[index]++;
-
- /* Compute minimal mass */
- min_mass = min(min_mass, hydro_get_mass(p));
-
- /* Compute sum of velocity norm */
- sum_vel_norm += p->v[0] * p->v[0] + p->v[1] * p->v[1] + p->v[2] * p->v[2];
-
- /* Update the position */
- p->x[0] = pos_x;
- p->x[1] = pos_y;
- p->x[2] = pos_z;
- }
- }
-
- /* Write the counts back to the global array. */
- for (int k = 0; k < s->nr_cells; k++)
- if (cell_counts[k]) atomic_add(&data->cell_counts[k], cell_counts[k]);
- free(cell_counts);
-
- /* Write the count of inhibited and extra parts */
- if (count_inhibited_part)
- atomic_add(&data->count_inhibited_part, count_inhibited_part);
- if (count_extra_part) atomic_add(&data->count_extra_part, count_extra_part);
-
- /* Write back the minimal part mass and velocity sum */
- atomic_min_f(&s->min_part_mass, min_mass);
- atomic_add_f(&s->sum_part_vel_norm, sum_vel_norm);
-}
-
-/**
- * @brief #threadpool mapper function to compute the g-particle cell indices.
- *
- * @param map_data Pointer towards the g-particles.
- * @param nr_gparts The number of g-particles to treat.
- * @param extra_data Pointers to the space and index list
- */
-void space_gparts_get_cell_index_mapper(void *map_data, int nr_gparts,
- void *extra_data) {
-
- /* Unpack the data */
- struct gpart *restrict gparts = (struct gpart *)map_data;
- struct index_data *data = (struct index_data *)extra_data;
- struct space *s = data->s;
- int *const ind = data->ind + (ptrdiff_t)(gparts - s->gparts);
-
- /* Get some constants */
- const double dim_x = s->dim[0];
- const double dim_y = s->dim[1];
- const double dim_z = s->dim[2];
- const int cdim[3] = {s->cdim[0], s->cdim[1], s->cdim[2]};
- const double ih_x = s->iwidth[0];
- const double ih_y = s->iwidth[1];
- const double ih_z = s->iwidth[2];
-
- /* Init the local count buffer. */
- int *cell_counts = (int *)calloc(sizeof(int), s->nr_cells);
- if (cell_counts == NULL)
- error("Failed to allocate temporary cell count buffer.");
-
- /* Init the local collectors */
- float min_mass = FLT_MAX;
- float sum_vel_norm = 0.f;
- size_t count_inhibited_gpart = 0;
- size_t count_extra_gpart = 0;
-
- for (int k = 0; k < nr_gparts; k++) {
-
- /* Get the particle */
- struct gpart *restrict gp = &gparts[k];
-
- double old_pos_x = gp->x[0];
- double old_pos_y = gp->x[1];
- double old_pos_z = gp->x[2];
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (!s->periodic && gp->time_bin != time_bin_inhibited) {
- if (old_pos_x < 0. || old_pos_x > dim_x)
- error("Particle outside of volume along X.");
- if (old_pos_y < 0. || old_pos_y > dim_y)
- error("Particle outside of volume along Y.");
- if (old_pos_z < 0. || old_pos_z > dim_z)
- error("Particle outside of volume along Z.");
- }
-#endif
-
- /* Put it back into the simulation volume */
- double pos_x = box_wrap(old_pos_x, 0.0, dim_x);
- double pos_y = box_wrap(old_pos_y, 0.0, dim_y);
- double pos_z = box_wrap(old_pos_z, 0.0, dim_z);
-
- /* Treat the case where a particle was wrapped back exactly onto
- * the edge because of rounding issues (more accuracy around 0
- * than around dim) */
- if (pos_x == dim_x) pos_x = 0.0;
- if (pos_y == dim_y) pos_y = 0.0;
- if (pos_z == dim_z) pos_z = 0.0;
-
- /* Get its cell index */
- const int index =
- cell_getid(cdim, pos_x * ih_x, pos_y * ih_y, pos_z * ih_z);
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (index < 0 || index >= cdim[0] * cdim[1] * cdim[2])
- error("Invalid index=%d cdim=[%d %d %d] p->x=[%e %e %e]", index, cdim[0],
- cdim[1], cdim[2], pos_x, pos_y, pos_z);
-
- if (pos_x >= dim_x || pos_y >= dim_y || pos_z >= dim_z || pos_x < 0. ||
- pos_y < 0. || pos_z < 0.)
- error("Particle outside of simulation box. p->x=[%e %e %e]", pos_x, pos_y,
- pos_z);
-#endif
-
- if (gp->time_bin == time_bin_inhibited) {
- /* Is this particle to be removed? */
- ind[k] = -1;
- ++count_inhibited_gpart;
- } else if (gp->time_bin == time_bin_not_created) {
- /* Is this a place-holder for on-the-fly creation? */
- ind[k] = index;
- cell_counts[index]++;
- ++count_extra_gpart;
-
- } else {
- /* List its top-level cell index */
- ind[k] = index;
- cell_counts[index]++;
-
- if (gp->type == swift_type_dark_matter) {
-
- /* Compute minimal mass */
- min_mass = min(min_mass, gp->mass);
-
- /* Compute sum of velocity norm */
- sum_vel_norm += gp->v_full[0] * gp->v_full[0] +
- gp->v_full[1] * gp->v_full[1] +
- gp->v_full[2] * gp->v_full[2];
- }
-
- /* Update the position */
- gp->x[0] = pos_x;
- gp->x[1] = pos_y;
- gp->x[2] = pos_z;
- }
- }
-
- /* Write the counts back to the global array. */
- for (int k = 0; k < s->nr_cells; k++)
- if (cell_counts[k]) atomic_add(&data->cell_counts[k], cell_counts[k]);
- free(cell_counts);
-
- /* Write the count of inhibited and extra gparts */
- if (count_inhibited_gpart)
- atomic_add(&data->count_inhibited_gpart, count_inhibited_gpart);
- if (count_extra_gpart)
- atomic_add(&data->count_extra_gpart, count_extra_gpart);
-
- /* Write back the minimal part mass and velocity sum */
- atomic_min_f(&s->min_gpart_mass, min_mass);
- atomic_add_f(&s->sum_gpart_vel_norm, sum_vel_norm);
-}
-
-/**
- * @brief #threadpool mapper function to compute the s-particle cell indices.
- *
- * @param map_data Pointer towards the s-particles.
- * @param nr_sparts The number of s-particles to treat.
- * @param extra_data Pointers to the space and index list
- */
-void space_sparts_get_cell_index_mapper(void *map_data, int nr_sparts,
- void *extra_data) {
-
- /* Unpack the data */
- struct spart *restrict sparts = (struct spart *)map_data;
- struct index_data *data = (struct index_data *)extra_data;
- struct space *s = data->s;
- int *const ind = data->ind + (ptrdiff_t)(sparts - s->sparts);
-
- /* Get some constants */
- const double dim_x = s->dim[0];
- const double dim_y = s->dim[1];
- const double dim_z = s->dim[2];
- const int cdim[3] = {s->cdim[0], s->cdim[1], s->cdim[2]};
- const double ih_x = s->iwidth[0];
- const double ih_y = s->iwidth[1];
- const double ih_z = s->iwidth[2];
-
- /* Init the local count buffer. */
- int *cell_counts = (int *)calloc(sizeof(int), s->nr_cells);
- if (cell_counts == NULL)
- error("Failed to allocate temporary cell count buffer.");
-
- /* Init the local collectors */
- float min_mass = FLT_MAX;
- float sum_vel_norm = 0.f;
- size_t count_inhibited_spart = 0;
- size_t count_extra_spart = 0;
-
- for (int k = 0; k < nr_sparts; k++) {
-
- /* Get the particle */
- struct spart *restrict sp = &sparts[k];
-
- double old_pos_x = sp->x[0];
- double old_pos_y = sp->x[1];
- double old_pos_z = sp->x[2];
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (!s->periodic && sp->time_bin != time_bin_inhibited) {
- if (old_pos_x < 0. || old_pos_x > dim_x)
- error("Particle outside of volume along X.");
- if (old_pos_y < 0. || old_pos_y > dim_y)
- error("Particle outside of volume along Y.");
- if (old_pos_z < 0. || old_pos_z > dim_z)
- error("Particle outside of volume along Z.");
- }
-#endif
-
- /* Put it back into the simulation volume */
- double pos_x = box_wrap(old_pos_x, 0.0, dim_x);
- double pos_y = box_wrap(old_pos_y, 0.0, dim_y);
- double pos_z = box_wrap(old_pos_z, 0.0, dim_z);
-
- /* Treat the case where a particle was wrapped back exactly onto
- * the edge because of rounding issues (more accuracy around 0
- * than around dim) */
- if (pos_x == dim_x) pos_x = 0.0;
- if (pos_y == dim_y) pos_y = 0.0;
- if (pos_z == dim_z) pos_z = 0.0;
-
- /* Get its cell index */
- const int index =
- cell_getid(cdim, pos_x * ih_x, pos_y * ih_y, pos_z * ih_z);
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (index < 0 || index >= cdim[0] * cdim[1] * cdim[2])
- error("Invalid index=%d cdim=[%d %d %d] p->x=[%e %e %e]", index, cdim[0],
- cdim[1], cdim[2], pos_x, pos_y, pos_z);
-
- if (pos_x >= dim_x || pos_y >= dim_y || pos_z >= dim_z || pos_x < 0. ||
- pos_y < 0. || pos_z < 0.)
- error("Particle outside of simulation box. p->x=[%e %e %e]", pos_x, pos_y,
- pos_z);
-#endif
-
- /* Is this particle to be removed? */
- if (sp->time_bin == time_bin_inhibited) {
- ind[k] = -1;
- ++count_inhibited_spart;
- } else if (sp->time_bin == time_bin_not_created) {
- /* Is this a place-holder for on-the-fly creation? */
- ind[k] = index;
- cell_counts[index]++;
- ++count_extra_spart;
-
- } else {
- /* List its top-level cell index */
- ind[k] = index;
- cell_counts[index]++;
-
- /* Compute minimal mass */
- min_mass = min(min_mass, sp->mass);
-
- /* Compute sum of velocity norm */
- sum_vel_norm +=
- sp->v[0] * sp->v[0] + sp->v[1] * sp->v[1] + sp->v[2] * sp->v[2];
-
- /* Update the position */
- sp->x[0] = pos_x;
- sp->x[1] = pos_y;
- sp->x[2] = pos_z;
- }
- }
-
- /* Write the counts back to the global array. */
- for (int k = 0; k < s->nr_cells; k++)
- if (cell_counts[k]) atomic_add(&data->cell_counts[k], cell_counts[k]);
- free(cell_counts);
-
- /* Write the count of inhibited and extra sparts */
- if (count_inhibited_spart)
- atomic_add(&data->count_inhibited_spart, count_inhibited_spart);
- if (count_extra_spart)
- atomic_add(&data->count_extra_spart, count_extra_spart);
-
- /* Write back the minimal part mass and velocity sum */
- atomic_min_f(&s->min_spart_mass, min_mass);
- atomic_add_f(&s->sum_spart_vel_norm, sum_vel_norm);
-}
-
-/**
- * @brief #threadpool mapper function to compute the b-particle cell indices.
- *
- * @param map_data Pointer towards the b-particles.
- * @param nr_bparts The number of b-particles to treat.
- * @param extra_data Pointers to the space and index list
- */
-void space_bparts_get_cell_index_mapper(void *map_data, int nr_bparts,
- void *extra_data) {
-
- /* Unpack the data */
- struct bpart *restrict bparts = (struct bpart *)map_data;
- struct index_data *data = (struct index_data *)extra_data;
- struct space *s = data->s;
- int *const ind = data->ind + (ptrdiff_t)(bparts - s->bparts);
-
- /* Get some constants */
- const double dim_x = s->dim[0];
- const double dim_y = s->dim[1];
- const double dim_z = s->dim[2];
- const int cdim[3] = {s->cdim[0], s->cdim[1], s->cdim[2]};
- const double ih_x = s->iwidth[0];
- const double ih_y = s->iwidth[1];
- const double ih_z = s->iwidth[2];
-
- /* Init the local count buffer. */
- int *cell_counts = (int *)calloc(sizeof(int), s->nr_cells);
- if (cell_counts == NULL)
- error("Failed to allocate temporary cell count buffer.");
-
- /* Init the local collectors */
- float min_mass = FLT_MAX;
- float sum_vel_norm = 0.f;
- size_t count_inhibited_bpart = 0;
- size_t count_extra_bpart = 0;
-
- for (int k = 0; k < nr_bparts; k++) {
-
- /* Get the particle */
- struct bpart *restrict bp = &bparts[k];
-
- double old_pos_x = bp->x[0];
- double old_pos_y = bp->x[1];
- double old_pos_z = bp->x[2];
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (!s->periodic && bp->time_bin != time_bin_inhibited) {
- if (old_pos_x < 0. || old_pos_x > dim_x)
- error("Particle outside of volume along X.");
- if (old_pos_y < 0. || old_pos_y > dim_y)
- error("Particle outside of volume along Y.");
- if (old_pos_z < 0. || old_pos_z > dim_z)
- error("Particle outside of volume along Z.");
- }
-#endif
-
- /* Put it back into the simulation volume */
- double pos_x = box_wrap(old_pos_x, 0.0, dim_x);
- double pos_y = box_wrap(old_pos_y, 0.0, dim_y);
- double pos_z = box_wrap(old_pos_z, 0.0, dim_z);
-
- /* Treat the case where a particle was wrapped back exactly onto
- * the edge because of rounding issues (more accuracy around 0
- * than around dim) */
- if (pos_x == dim_x) pos_x = 0.0;
- if (pos_y == dim_y) pos_y = 0.0;
- if (pos_z == dim_z) pos_z = 0.0;
-
- /* Get its cell index */
- const int index =
- cell_getid(cdim, pos_x * ih_x, pos_y * ih_y, pos_z * ih_z);
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (index < 0 || index >= cdim[0] * cdim[1] * cdim[2])
- error("Invalid index=%d cdim=[%d %d %d] p->x=[%e %e %e]", index, cdim[0],
- cdim[1], cdim[2], pos_x, pos_y, pos_z);
-
- if (pos_x >= dim_x || pos_y >= dim_y || pos_z >= dim_z || pos_x < 0. ||
- pos_y < 0. || pos_z < 0.)
- error("Particle outside of simulation box. p->x=[%e %e %e]", pos_x, pos_y,
- pos_z);
-#endif
-
- /* Is this particle to be removed? */
- if (bp->time_bin == time_bin_inhibited) {
- ind[k] = -1;
- ++count_inhibited_bpart;
- } else if (bp->time_bin == time_bin_not_created) {
- /* Is this a place-holder for on-the-fly creation? */
- ind[k] = index;
- cell_counts[index]++;
- ++count_extra_bpart;
-
- } else {
- /* List its top-level cell index */
- ind[k] = index;
- cell_counts[index]++;
-
- /* Compute minimal mass */
- min_mass = min(min_mass, bp->mass);
-
- /* Compute sum of velocity norm */
- sum_vel_norm +=
- bp->v[0] * bp->v[0] + bp->v[1] * bp->v[1] + bp->v[2] * bp->v[2];
-
- /* Update the position */
- bp->x[0] = pos_x;
- bp->x[1] = pos_y;
- bp->x[2] = pos_z;
- }
- }
-
- /* Write the counts back to the global array. */
- for (int k = 0; k < s->nr_cells; k++)
- if (cell_counts[k]) atomic_add(&data->cell_counts[k], cell_counts[k]);
- free(cell_counts);
-
- /* Write the count of inhibited and extra bparts */
- if (count_inhibited_bpart)
- atomic_add(&data->count_inhibited_bpart, count_inhibited_bpart);
- if (count_extra_bpart)
- atomic_add(&data->count_extra_bpart, count_extra_bpart);
-
- /* Write back the minimal part mass and velocity sum */
- atomic_min_f(&s->min_bpart_mass, min_mass);
- atomic_add_f(&s->sum_bpart_vel_norm, sum_vel_norm);
-}
-
-/**
- * @brief #threadpool mapper function to compute the sink-particle cell indices.
- *
- * @param map_data Pointer towards the sink-particles.
- * @param nr_sinks The number of sink-particles to treat.
- * @param extra_data Pointers to the space and index list
- */
-void space_sinks_get_cell_index_mapper(void *map_data, int nr_sinks,
- void *extra_data) {
-
- /* Unpack the data */
- struct sink *restrict sinks = (struct sink *)map_data;
- struct index_data *data = (struct index_data *)extra_data;
- struct space *s = data->s;
- int *const ind = data->ind + (ptrdiff_t)(sinks - s->sinks);
-
- /* Get some constants */
- const double dim_x = s->dim[0];
- const double dim_y = s->dim[1];
- const double dim_z = s->dim[2];
- const int cdim[3] = {s->cdim[0], s->cdim[1], s->cdim[2]};
- const double ih_x = s->iwidth[0];
- const double ih_y = s->iwidth[1];
- const double ih_z = s->iwidth[2];
-
- /* Init the local count buffer. */
- int *cell_counts = (int *)calloc(sizeof(int), s->nr_cells);
- if (cell_counts == NULL)
- error("Failed to allocate temporary cell count buffer.");
-
- /* Init the local collectors */
- size_t count_inhibited_sink = 0;
- size_t count_extra_sink = 0;
-
- for (int k = 0; k < nr_sinks; k++) {
-
- /* Get the particle */
- struct sink *restrict sink = &sinks[k];
-
- double old_pos_x = sink->x[0];
- double old_pos_y = sink->x[1];
- double old_pos_z = sink->x[2];
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (!s->periodic && sink->time_bin != time_bin_inhibited) {
- if (old_pos_x < 0. || old_pos_x > dim_x)
- error("Particle outside of volume along X.");
- if (old_pos_y < 0. || old_pos_y > dim_y)
- error("Particle outside of volume along Y.");
- if (old_pos_z < 0. || old_pos_z > dim_z)
- error("Particle outside of volume along Z.");
- }
-#endif
-
- /* Put it back into the simulation volume */
- double pos_x = box_wrap(old_pos_x, 0.0, dim_x);
- double pos_y = box_wrap(old_pos_y, 0.0, dim_y);
- double pos_z = box_wrap(old_pos_z, 0.0, dim_z);
-
- /* Treat the case where a particle was wrapped back exactly onto
- * the edge because of rounding issues (more accuracy around 0
- * than around dim) */
- if (pos_x == dim_x) pos_x = 0.0;
- if (pos_y == dim_y) pos_y = 0.0;
- if (pos_z == dim_z) pos_z = 0.0;
-
- /* Get its cell index */
- const int index =
- cell_getid(cdim, pos_x * ih_x, pos_y * ih_y, pos_z * ih_z);
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (index < 0 || index >= cdim[0] * cdim[1] * cdim[2])
- error("Invalid index=%d cdim=[%d %d %d] p->x=[%e %e %e]", index, cdim[0],
- cdim[1], cdim[2], pos_x, pos_y, pos_z);
-
- if (pos_x >= dim_x || pos_y >= dim_y || pos_z >= dim_z || pos_x < 0. ||
- pos_y < 0. || pos_z < 0.)
- error("Particle outside of simulation box. p->x=[%e %e %e]", pos_x, pos_y,
- pos_z);
-#endif
-
- /* Is this particle to be removed? */
- if (sink->time_bin == time_bin_inhibited) {
- ind[k] = -1;
- ++count_inhibited_sink;
- } else if (sink->time_bin == time_bin_not_created) {
- /* Is this a place-holder for on-the-fly creation? */
- ind[k] = index;
- cell_counts[index]++;
- ++count_extra_sink;
-
- } else {
- /* List its top-level cell index */
- ind[k] = index;
- cell_counts[index]++;
-
- /* Update the position */
- sink->x[0] = pos_x;
- sink->x[1] = pos_y;
- sink->x[2] = pos_z;
- }
- }
-
- /* Write the counts back to the global array. */
- for (int k = 0; k < s->nr_cells; k++)
- if (cell_counts[k]) atomic_add(&data->cell_counts[k], cell_counts[k]);
- free(cell_counts);
-
- /* Write the count of inhibited and extra sinks */
- if (count_inhibited_sink)
- atomic_add(&data->count_inhibited_sink, count_inhibited_sink);
- if (count_extra_sink) atomic_add(&data->count_extra_sink, count_extra_sink);
-}
-
-/**
- * @brief Computes the cell index of all the particles.
- *
- * Also computes the minimal mass of all #part.
- *
- * @param s The #space.
- * @param ind The array of indices to fill.
- * @param cell_counts The cell counters to update.
- * @param count_inhibited_parts (return) The number of #part to remove.
- * @param count_extra_parts (return) The number of #part for on-the-fly
- * creation.
- * @param verbose Are we talkative ?
- */
-void space_parts_get_cell_index(struct space *s, int *ind, int *cell_counts,
- size_t *count_inhibited_parts,
- size_t *count_extra_parts, int verbose) {
-
- const ticks tic = getticks();
-
- /* Re-set the counters */
- s->min_part_mass = FLT_MAX;
- s->sum_part_vel_norm = 0.f;
-
- /* Pack the extra information */
- struct index_data data;
- data.s = s;
- data.ind = ind;
- data.cell_counts = cell_counts;
- data.count_inhibited_part = 0;
- data.count_inhibited_gpart = 0;
- data.count_inhibited_spart = 0;
- data.count_inhibited_bpart = 0;
- data.count_inhibited_sink = 0;
- data.count_extra_part = 0;
- data.count_extra_gpart = 0;
- data.count_extra_spart = 0;
- data.count_extra_bpart = 0;
- data.count_extra_sink = 0;
-
- threadpool_map(&s->e->threadpool, space_parts_get_cell_index_mapper, s->parts,
- s->nr_parts, sizeof(struct part), threadpool_auto_chunk_size,
- &data);
-
- *count_inhibited_parts = data.count_inhibited_part;
- *count_extra_parts = data.count_extra_part;
-
- if (verbose)
- message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
- clocks_getunit());
-}
-
-/**
- * @brief Computes the cell index of all the g-particles.
- *
- * Also computes the minimal mass of all dark-matter #gpart.
- *
- * @param s The #space.
- * @param gind The array of indices to fill.
- * @param cell_counts The cell counters to update.
- * @param count_inhibited_gparts (return) The number of #gpart to remove.
- * @param count_extra_gparts (return) The number of #gpart for on-the-fly
- * creation.
- * @param verbose Are we talkative ?
- */
-void space_gparts_get_cell_index(struct space *s, int *gind, int *cell_counts,
- size_t *count_inhibited_gparts,
- size_t *count_extra_gparts, int verbose) {
-
- const ticks tic = getticks();
-
- /* Re-set the counters */
- s->min_gpart_mass = FLT_MAX;
- s->sum_gpart_vel_norm = 0.f;
-
- /* Pack the extra information */
- struct index_data data;
- data.s = s;
- data.ind = gind;
- data.cell_counts = cell_counts;
- data.count_inhibited_part = 0;
- data.count_inhibited_gpart = 0;
- data.count_inhibited_spart = 0;
- data.count_inhibited_bpart = 0;
- data.count_inhibited_sink = 0;
- data.count_extra_part = 0;
- data.count_extra_gpart = 0;
- data.count_extra_spart = 0;
- data.count_extra_bpart = 0;
- data.count_extra_sink = 0;
-
- threadpool_map(&s->e->threadpool, space_gparts_get_cell_index_mapper,
- s->gparts, s->nr_gparts, sizeof(struct gpart),
- threadpool_auto_chunk_size, &data);
-
- *count_inhibited_gparts = data.count_inhibited_gpart;
- *count_extra_gparts = data.count_extra_gpart;
-
- if (verbose)
- message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
- clocks_getunit());
-}
-
-/**
- * @brief Computes the cell index of all the s-particles.
- *
- * Also computes the minimal mass of all #spart.
- *
- * @param s The #space.
- * @param sind The array of indices to fill.
- * @param cell_counts The cell counters to update.
- * @param count_inhibited_sparts (return) The number of #spart to remove.
- * @param count_extra_sparts (return) The number of #spart for on-the-fly
- * creation.
- * @param verbose Are we talkative ?
- */
-void space_sparts_get_cell_index(struct space *s, int *sind, int *cell_counts,
- size_t *count_inhibited_sparts,
- size_t *count_extra_sparts, int verbose) {
-
- const ticks tic = getticks();
-
- /* Re-set the counters */
- s->min_spart_mass = FLT_MAX;
- s->sum_spart_vel_norm = 0.f;
-
- /* Pack the extra information */
- struct index_data data;
- data.s = s;
- data.ind = sind;
- data.cell_counts = cell_counts;
- data.count_inhibited_part = 0;
- data.count_inhibited_gpart = 0;
- data.count_inhibited_spart = 0;
- data.count_inhibited_sink = 0;
- data.count_inhibited_bpart = 0;
- data.count_extra_part = 0;
- data.count_extra_gpart = 0;
- data.count_extra_spart = 0;
- data.count_extra_bpart = 0;
- data.count_extra_sink = 0;
-
- threadpool_map(&s->e->threadpool, space_sparts_get_cell_index_mapper,
- s->sparts, s->nr_sparts, sizeof(struct spart),
- threadpool_auto_chunk_size, &data);
-
- *count_inhibited_sparts = data.count_inhibited_spart;
- *count_extra_sparts = data.count_extra_spart;
-
- if (verbose)
- message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
- clocks_getunit());
-}
-
-/**
- * @brief Computes the cell index of all the sink-particles.
- *
- * @param s The #space.
- * @param sink_ind The array of indices to fill.
- * @param cell_counts The cell counters to update.
- * @param count_inhibited_sinks (return) The number of #sink to remove.
- * @param count_extra_sinks (return) The number of #sink for on-the-fly
- * creation.
- * @param verbose Are we talkative ?
- */
-void space_sinks_get_cell_index(struct space *s, int *sink_ind,
- int *cell_counts, size_t *count_inhibited_sinks,
- size_t *count_extra_sinks, int verbose) {
-
- const ticks tic = getticks();
-
- /* Re-set the counters */
- s->min_sink_mass = FLT_MAX;
- s->sum_sink_vel_norm = 0.f;
-
- /* Pack the extra information */
- struct index_data data;
- data.s = s;
- data.ind = sink_ind;
- data.cell_counts = cell_counts;
- data.count_inhibited_part = 0;
- data.count_inhibited_gpart = 0;
- data.count_inhibited_spart = 0;
- data.count_inhibited_bpart = 0;
- data.count_inhibited_sink = 0;
- data.count_extra_part = 0;
- data.count_extra_gpart = 0;
- data.count_extra_spart = 0;
- data.count_extra_bpart = 0;
- data.count_extra_sink = 0;
-
- threadpool_map(&s->e->threadpool, space_sinks_get_cell_index_mapper, s->sinks,
- s->nr_sinks, sizeof(struct sink), threadpool_auto_chunk_size,
- &data);
-
- *count_inhibited_sinks = data.count_inhibited_sink;
- *count_extra_sinks = data.count_extra_sink;
-
- if (verbose)
- message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
- clocks_getunit());
-}
-
-/**
- * @brief Computes the cell index of all the b-particles.
- *
- * Also computes the minimal mass of all #bpart.
- *
- * @param s The #space.
- * @param bind The array of indices to fill.
- * @param cell_counts The cell counters to update.
- * @param count_inhibited_bparts (return) The number of #bpart to remove.
- * @param count_extra_bparts (return) The number of #bpart for on-the-fly
- * creation.
- * @param verbose Are we talkative ?
- */
-void space_bparts_get_cell_index(struct space *s, int *bind, int *cell_counts,
- size_t *count_inhibited_bparts,
- size_t *count_extra_bparts, int verbose) {
-
- const ticks tic = getticks();
-
- /* Re-set the counters */
- s->min_bpart_mass = FLT_MAX;
- s->sum_bpart_vel_norm = 0.f;
-
- /* Pack the extra information */
- struct index_data data;
- data.s = s;
- data.ind = bind;
- data.cell_counts = cell_counts;
- data.count_inhibited_part = 0;
- data.count_inhibited_gpart = 0;
- data.count_inhibited_spart = 0;
- data.count_inhibited_bpart = 0;
- data.count_inhibited_sink = 0;
- data.count_extra_part = 0;
- data.count_extra_gpart = 0;
- data.count_extra_spart = 0;
- data.count_extra_bpart = 0;
- data.count_extra_sink = 0;
-
- threadpool_map(&s->e->threadpool, space_bparts_get_cell_index_mapper,
- s->bparts, s->nr_bparts, sizeof(struct bpart),
- threadpool_auto_chunk_size, &data);
-
- *count_inhibited_bparts = data.count_inhibited_bpart;
- *count_extra_bparts = data.count_extra_bpart;
-
- if (verbose)
- message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
- clocks_getunit());
-}
-
-/**
- * @brief Sort the particles and condensed particles according to the given
- * indices.
- *
- * @param parts The array of #part to sort.
- * @param xparts The corresponding #xpart array to sort as well.
- * @param ind The indices with respect to which the parts are sorted.
- * @param counts Number of particles per index.
- * @param num_bins Total number of bins (length of count).
- * @param parts_offset Offset of the #part array from the global #part array.
- */
-void space_parts_sort(struct part *parts, struct xpart *xparts,
- int *restrict ind, int *restrict counts, int num_bins,
- ptrdiff_t parts_offset) {
- /* Create the offsets array. */
- size_t *offsets = NULL;
- if (swift_memalign("parts_offsets", (void **)&offsets, SWIFT_STRUCT_ALIGNMENT,
- sizeof(size_t) * (num_bins + 1)) != 0)
- error("Failed to allocate temporary cell offsets array.");
-
- offsets[0] = 0;
- for (int k = 1; k <= num_bins; k++) {
- offsets[k] = offsets[k - 1] + counts[k - 1];
- counts[k - 1] = 0;
- }
-
- /* Loop over local cells. */
- for (int cid = 0; cid < num_bins; cid++) {
- for (size_t k = offsets[cid] + counts[cid]; k < offsets[cid + 1]; k++) {
- counts[cid]++;
- int target_cid = ind[k];
- if (target_cid == cid) {
- continue;
- }
- struct part temp_part = parts[k];
- struct xpart temp_xpart = xparts[k];
- while (target_cid != cid) {
- size_t j = offsets[target_cid] + counts[target_cid]++;
- while (ind[j] == target_cid) {
- j = offsets[target_cid] + counts[target_cid]++;
- }
- memswap(&parts[j], &temp_part, sizeof(struct part));
- memswap(&xparts[j], &temp_xpart, sizeof(struct xpart));
- memswap(&ind[j], &target_cid, sizeof(int));
- if (parts[j].gpart)
- parts[j].gpart->id_or_neg_offset = -(j + parts_offset);
- }
- parts[k] = temp_part;
- xparts[k] = temp_xpart;
- ind[k] = target_cid;
- if (parts[k].gpart)
- parts[k].gpart->id_or_neg_offset = -(k + parts_offset);
- }
- }
-
-#ifdef SWIFT_DEBUG_CHECKS
- for (int k = 0; k < num_bins; k++)
- if (offsets[k + 1] != offsets[k] + counts[k])
- error("Bad offsets after shuffle.");
-#endif /* SWIFT_DEBUG_CHECKS */
-
- swift_free("parts_offsets", offsets);
-}
-
-/**
- * @brief Sort the s-particles according to the given indices.
- *
- * @param sparts The array of #spart to sort.
- * @param ind The indices with respect to which the #spart are sorted.
- * @param counts Number of particles per index.
- * @param num_bins Total number of bins (length of counts).
- * @param sparts_offset Offset of the #spart array from the global #spart.
- * array.
- */
-void space_sparts_sort(struct spart *sparts, int *restrict ind,
- int *restrict counts, int num_bins,
- ptrdiff_t sparts_offset) {
- /* Create the offsets array. */
- size_t *offsets = NULL;
- if (swift_memalign("sparts_offsets", (void **)&offsets,
- SWIFT_STRUCT_ALIGNMENT,
- sizeof(size_t) * (num_bins + 1)) != 0)
- error("Failed to allocate temporary cell offsets array.");
-
- offsets[0] = 0;
- for (int k = 1; k <= num_bins; k++) {
- offsets[k] = offsets[k - 1] + counts[k - 1];
- counts[k - 1] = 0;
- }
-
- /* Loop over local cells. */
- for (int cid = 0; cid < num_bins; cid++) {
- for (size_t k = offsets[cid] + counts[cid]; k < offsets[cid + 1]; k++) {
- counts[cid]++;
- int target_cid = ind[k];
- if (target_cid == cid) {
- continue;
- }
- struct spart temp_spart = sparts[k];
- while (target_cid != cid) {
- size_t j = offsets[target_cid] + counts[target_cid]++;
- while (ind[j] == target_cid) {
- j = offsets[target_cid] + counts[target_cid]++;
- }
- memswap(&sparts[j], &temp_spart, sizeof(struct spart));
- memswap(&ind[j], &target_cid, sizeof(int));
- if (sparts[j].gpart)
- sparts[j].gpart->id_or_neg_offset = -(j + sparts_offset);
- }
- sparts[k] = temp_spart;
- ind[k] = target_cid;
- if (sparts[k].gpart)
- sparts[k].gpart->id_or_neg_offset = -(k + sparts_offset);
- }
- }
-
-#ifdef SWIFT_DEBUG_CHECKS
- for (int k = 0; k < num_bins; k++)
- if (offsets[k + 1] != offsets[k] + counts[k])
- error("Bad offsets after shuffle.");
-#endif /* SWIFT_DEBUG_CHECKS */
-
- swift_free("sparts_offsets", offsets);
-}
-
-/**
- * @brief Sort the b-particles according to the given indices.
- *
- * @param bparts The array of #bpart to sort.
- * @param ind The indices with respect to which the #bpart are sorted.
- * @param counts Number of particles per index.
- * @param num_bins Total number of bins (length of counts).
- * @param bparts_offset Offset of the #bpart array from the global #bpart.
- * array.
- */
-void space_bparts_sort(struct bpart *bparts, int *restrict ind,
- int *restrict counts, int num_bins,
- ptrdiff_t bparts_offset) {
- /* Create the offsets array. */
- size_t *offsets = NULL;
- if (swift_memalign("bparts_offsets", (void **)&offsets,
- SWIFT_STRUCT_ALIGNMENT,
- sizeof(size_t) * (num_bins + 1)) != 0)
- error("Failed to allocate temporary cell offsets array.");
-
- offsets[0] = 0;
- for (int k = 1; k <= num_bins; k++) {
- offsets[k] = offsets[k - 1] + counts[k - 1];
- counts[k - 1] = 0;
- }
-
- /* Loop over local cells. */
- for (int cid = 0; cid < num_bins; cid++) {
- for (size_t k = offsets[cid] + counts[cid]; k < offsets[cid + 1]; k++) {
- counts[cid]++;
- int target_cid = ind[k];
- if (target_cid == cid) {
- continue;
- }
- struct bpart temp_bpart = bparts[k];
- while (target_cid != cid) {
- size_t j = offsets[target_cid] + counts[target_cid]++;
- while (ind[j] == target_cid) {
- j = offsets[target_cid] + counts[target_cid]++;
- }
- memswap(&bparts[j], &temp_bpart, sizeof(struct bpart));
- memswap(&ind[j], &target_cid, sizeof(int));
- if (bparts[j].gpart)
- bparts[j].gpart->id_or_neg_offset = -(j + bparts_offset);
- }
- bparts[k] = temp_bpart;
- ind[k] = target_cid;
- if (bparts[k].gpart)
- bparts[k].gpart->id_or_neg_offset = -(k + bparts_offset);
- }
- }
-
-#ifdef SWIFT_DEBUG_CHECKS
- for (int k = 0; k < num_bins; k++)
- if (offsets[k + 1] != offsets[k] + counts[k])
- error("Bad offsets after shuffle.");
-#endif /* SWIFT_DEBUG_CHECKS */
-
- swift_free("bparts_offsets", offsets);
-}
-
-/**
- * @brief Sort the sink-particles according to the given indices.
- *
- * @param sinks The array of #sink to sort.
- * @param ind The indices with respect to which the #sink are sorted.
- * @param counts Number of particles per index.
- * @param num_bins Total number of bins (length of counts).
- * @param sinks_offset Offset of the #sink array from the global #sink.
- * array.
- */
-void space_sinks_sort(struct sink *sinks, int *restrict ind,
- int *restrict counts, int num_bins,
- ptrdiff_t sinks_offset) {
- /* Create the offsets array. */
- size_t *offsets = NULL;
- if (swift_memalign("sinks_offsets", (void **)&offsets, SWIFT_STRUCT_ALIGNMENT,
- sizeof(size_t) * (num_bins + 1)) != 0)
- error("Failed to allocate temporary cell offsets array.");
-
- offsets[0] = 0;
- for (int k = 1; k <= num_bins; k++) {
- offsets[k] = offsets[k - 1] + counts[k - 1];
- counts[k - 1] = 0;
- }
-
- /* Loop over local cells. */
- for (int cid = 0; cid < num_bins; cid++) {
- for (size_t k = offsets[cid] + counts[cid]; k < offsets[cid + 1]; k++) {
- counts[cid]++;
- int target_cid = ind[k];
- if (target_cid == cid) {
- continue;
- }
- struct sink temp_sink = sinks[k];
- while (target_cid != cid) {
- size_t j = offsets[target_cid] + counts[target_cid]++;
- while (ind[j] == target_cid) {
- j = offsets[target_cid] + counts[target_cid]++;
- }
- memswap(&sinks[j], &temp_sink, sizeof(struct sink));
- memswap(&ind[j], &target_cid, sizeof(int));
- if (sinks[j].gpart)
- sinks[j].gpart->id_or_neg_offset = -(j + sinks_offset);
- }
- sinks[k] = temp_sink;
- ind[k] = target_cid;
- if (sinks[k].gpart)
- sinks[k].gpart->id_or_neg_offset = -(k + sinks_offset);
- }
- }
-
-#ifdef SWIFT_DEBUG_CHECKS
- for (int k = 0; k < num_bins; k++)
- if (offsets[k + 1] != offsets[k] + counts[k])
- error("Bad offsets after shuffle.");
-#endif /* SWIFT_DEBUG_CHECKS */
-
- swift_free("sinks_offsets", offsets);
-}
-
-/**
- * @brief Sort the g-particles according to the given indices.
- *
- * @param gparts The array of #gpart to sort.
- * @param parts Global #part array for re-linking.
- * @param sinks Global #sink array for re-linking.
- * @param sparts Global #spart array for re-linking.
- * @param bparts Global #bpart array for re-linking.
- * @param ind The indices with respect to which the gparts are sorted.
- * @param counts Number of particles per index.
- * @param num_bins Total number of bins (length of counts).
- */
-void space_gparts_sort(struct gpart *gparts, struct part *parts,
- struct sink *sinks, struct spart *sparts,
- struct bpart *bparts, int *restrict ind,
- int *restrict counts, int num_bins) {
- /* Create the offsets array. */
- size_t *offsets = NULL;
- if (swift_memalign("gparts_offsets", (void **)&offsets,
- SWIFT_STRUCT_ALIGNMENT,
- sizeof(size_t) * (num_bins + 1)) != 0)
- error("Failed to allocate temporary cell offsets array.");
-
- offsets[0] = 0;
- for (int k = 1; k <= num_bins; k++) {
- offsets[k] = offsets[k - 1] + counts[k - 1];
- counts[k - 1] = 0;
- }
-
- /* Loop over local cells. */
- for (int cid = 0; cid < num_bins; cid++) {
- for (size_t k = offsets[cid] + counts[cid]; k < offsets[cid + 1]; k++) {
- counts[cid]++;
- int target_cid = ind[k];
- if (target_cid == cid) {
- continue;
- }
- struct gpart temp_gpart = gparts[k];
- while (target_cid != cid) {
- size_t j = offsets[target_cid] + counts[target_cid]++;
- while (ind[j] == target_cid) {
- j = offsets[target_cid] + counts[target_cid]++;
- }
- memswap_unaligned(&gparts[j], &temp_gpart, sizeof(struct gpart));
- memswap(&ind[j], &target_cid, sizeof(int));
- if (gparts[j].type == swift_type_gas) {
- parts[-gparts[j].id_or_neg_offset].gpart = &gparts[j];
- } else if (gparts[j].type == swift_type_stars) {
- sparts[-gparts[j].id_or_neg_offset].gpart = &gparts[j];
- } else if (gparts[j].type == swift_type_black_hole) {
- bparts[-gparts[j].id_or_neg_offset].gpart = &gparts[j];
- } else if (gparts[j].type == swift_type_sink) {
- sinks[-gparts[j].id_or_neg_offset].gpart = &gparts[j];
- }
- }
- gparts[k] = temp_gpart;
- ind[k] = target_cid;
- if (gparts[k].type == swift_type_gas) {
- parts[-gparts[k].id_or_neg_offset].gpart = &gparts[k];
- } else if (gparts[k].type == swift_type_stars) {
- sparts[-gparts[k].id_or_neg_offset].gpart = &gparts[k];
- } else if (gparts[k].type == swift_type_black_hole) {
- bparts[-gparts[k].id_or_neg_offset].gpart = &gparts[k];
- } else if (gparts[k].type == swift_type_sink) {
- sinks[-gparts[k].id_or_neg_offset].gpart = &gparts[k];
- }
- }
- }
-
-#ifdef SWIFT_DEBUG_CHECKS
- for (int k = 0; k < num_bins; k++)
- if (offsets[k + 1] != offsets[k] + counts[k])
- error("Bad offsets after shuffle.");
-#endif /* SWIFT_DEBUG_CHECKS */
-
- swift_free("gparts_offsets", offsets);
-}
-
-/**
- * @brief Mapping function to free the sorted indices buffers.
- */
-void space_map_clearsort(struct cell *c, void *data) {
-
- cell_free_hydro_sorts(c);
- cell_free_stars_sorts(c);
-}
-
-/**
- * @brief Map a function to all particles in a cell recursively.
- *
- * @param c The #cell we are working in.
- * @param fun Function pointer to apply on the cells.
- * @param data Data passed to the function fun.
- */
-static void rec_map_parts(struct cell *c,
- void (*fun)(struct part *p, struct cell *c,
- void *data),
- void *data) {
- /* No progeny? */
- if (!c->split)
- for (int k = 0; k < c->hydro.count; k++) fun(&c->hydro.parts[k], c, data);
-
- /* Otherwise, recurse. */
- else
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL) rec_map_parts(c->progeny[k], fun, data);
-}
-
-/**
- * @brief Map a function to all particles in a space.
- *
- * @param s The #space we are working in.
- * @param fun Function pointer to apply on the cells.
- * @param data Data passed to the function fun.
- */
-void space_map_parts(struct space *s,
- void (*fun)(struct part *p, struct cell *c, void *data),
- void *data) {
-
- /* Call the recursive function on all higher-level cells. */
- for (int cid = 0; cid < s->nr_cells; cid++)
- rec_map_parts(&s->cells_top[cid], fun, data);
-}
-
-/**
- * @brief Map a function to all particles in a cell recursively.
- *
- * @param c The #cell we are working in.
- * @param fun Function pointer to apply on the cells.
- */
-static void rec_map_parts_xparts(struct cell *c,
- void (*fun)(struct part *p, struct xpart *xp,
- struct cell *c)) {
-
- /* No progeny? */
- if (!c->split)
- for (int k = 0; k < c->hydro.count; k++)
- fun(&c->hydro.parts[k], &c->hydro.xparts[k], c);
-
- /* Otherwise, recurse. */
- else
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL) rec_map_parts_xparts(c->progeny[k], fun);
-}
-
-/**
- * @brief Map a function to all particles (#part and #xpart) in a space.
- *
- * @param s The #space we are working in.
- * @param fun Function pointer to apply on the particles in the cells.
- */
-void space_map_parts_xparts(struct space *s,
- void (*fun)(struct part *p, struct xpart *xp,
- struct cell *c)) {
-
- /* Call the recursive function on all higher-level cells. */
- for (int cid = 0; cid < s->nr_cells; cid++)
- rec_map_parts_xparts(&s->cells_top[cid], fun);
-}
-
-/**
- * @brief Map a function to all particles in a cell recursively.
- *
- * @param c The #cell we are working in.
- * @param full Map to all cells, including cells with sub-cells.
- * @param fun Function pointer to apply on the cells.
- * @param data Data passed to the function fun.
- */
-static void rec_map_cells_post(struct cell *c, int full,
- void (*fun)(struct cell *c, void *data),
- void *data) {
- /* Recurse. */
- if (c->split)
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL)
- rec_map_cells_post(c->progeny[k], full, fun, data);
-
- /* No progeny? */
- if (full || !c->split) fun(c, data);
-}
-
-/**
- * @brief Map a function to all particles in a aspace.
- *
- * @param s The #space we are working in.
- * @param full Map to all cells, including cells with sub-cells.
- * @param fun Function pointer to apply on the cells.
- * @param data Data passed to the function fun.
- */
-void space_map_cells_post(struct space *s, int full,
- void (*fun)(struct cell *c, void *data), void *data) {
-
- /* Call the recursive function on all higher-level cells. */
- for (int cid = 0; cid < s->nr_cells; cid++)
- rec_map_cells_post(&s->cells_top[cid], full, fun, data);
-}
-
-static void rec_map_cells_pre(struct cell *c, int full,
- void (*fun)(struct cell *c, void *data),
- void *data) {
-
- /* No progeny? */
- if (full || !c->split) fun(c, data);
-
- /* Recurse. */
- if (c->split)
- for (int k = 0; k < 8; k++)
- if (c->progeny[k] != NULL)
- rec_map_cells_pre(c->progeny[k], full, fun, data);
-}
-
-/**
- * @brief Calls function fun on the cells in the space s
- *
- * @param s The #space
- * @param full If true calls the function on all cells and not just on leaves
- * @param fun The function to call.
- * @param data Additional data passed to fun() when called
- */
-void space_map_cells_pre(struct space *s, int full,
- void (*fun)(struct cell *c, void *data), void *data) {
-
- /* Call the recursive function on all higher-level cells. */
- for (int cid = 0; cid < s->nr_cells; cid++)
- rec_map_cells_pre(&s->cells_top[cid], full, fun, data);
-}
-
-/**
- * @brief Recursively split a cell.
- *
- * @param s The #space in which the cell lives.
- * @param c The #cell to split recursively.
- * @param buff A buffer for particle sorting, should be of size at least
- * c->hydro.count or @c NULL.
- * @param sbuff A buffer for particle sorting, should be of size at least
- * c->stars.count or @c NULL.
- * @param bbuff A buffer for particle sorting, should be of size at least
- * c->black_holes.count or @c NULL.
- * @param gbuff A buffer for particle sorting, should be of size at least
- * c->grav.count or @c NULL.
- * @param sink_buff A buffer for particle sorting, should be of size at least
- * c->sinks.count or @c NULL.
- */
-void space_split_recursive(struct space *s, struct cell *c,
- struct cell_buff *restrict buff,
- struct cell_buff *restrict sbuff,
- struct cell_buff *restrict bbuff,
- struct cell_buff *restrict gbuff,
- struct cell_buff *restrict sink_buff) {
-
- const int count = c->hydro.count;
- const int gcount = c->grav.count;
- const int scount = c->stars.count;
- const int bcount = c->black_holes.count;
- const int sink_count = c->sinks.count;
- const int with_self_gravity = s->with_self_gravity;
- const int depth = c->depth;
- int maxdepth = 0;
- float h_max = 0.0f;
- float sinks_h_max = 0.f;
- float stars_h_max = 0.f;
- float black_holes_h_max = 0.f;
- integertime_t ti_hydro_end_min = max_nr_timesteps, ti_hydro_end_max = 0,
- ti_hydro_beg_max = 0;
- integertime_t ti_gravity_end_min = max_nr_timesteps, ti_gravity_end_max = 0,
- ti_gravity_beg_max = 0;
- integertime_t ti_stars_end_min = max_nr_timesteps, ti_stars_end_max = 0,
- ti_stars_beg_max = 0;
- integertime_t ti_sinks_end_min = max_nr_timesteps, ti_sinks_end_max = 0,
- ti_sinks_beg_max = 0;
- integertime_t ti_black_holes_end_min = max_nr_timesteps,
- ti_black_holes_end_max = 0, ti_black_holes_beg_max = 0;
- struct part *parts = c->hydro.parts;
- struct gpart *gparts = c->grav.parts;
- struct spart *sparts = c->stars.parts;
- struct bpart *bparts = c->black_holes.parts;
- struct xpart *xparts = c->hydro.xparts;
- struct sink *sinks = c->sinks.parts;
- struct engine *e = s->e;
- const integertime_t ti_current = e->ti_current;
-
- /* If the buff is NULL, allocate it, and remember to free it. */
- const int allocate_buffer = (buff == NULL && gbuff == NULL && sbuff == NULL &&
- bbuff == NULL && sink_buff == NULL);
- if (allocate_buffer) {
- if (count > 0) {
- if (swift_memalign("tempbuff", (void **)&buff, SWIFT_STRUCT_ALIGNMENT,
- sizeof(struct cell_buff) * count) != 0)
- error("Failed to allocate temporary indices.");
- for (int k = 0; k < count; k++) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (parts[k].time_bin == time_bin_inhibited)
- error("Inhibited particle present in space_split()");
- if (parts[k].time_bin == time_bin_not_created)
- error("Extra particle present in space_split()");
-#endif
- buff[k].x[0] = parts[k].x[0];
- buff[k].x[1] = parts[k].x[1];
- buff[k].x[2] = parts[k].x[2];
- }
- }
- if (gcount > 0) {
- if (swift_memalign("tempgbuff", (void **)&gbuff, SWIFT_STRUCT_ALIGNMENT,
- sizeof(struct cell_buff) * gcount) != 0)
- error("Failed to allocate temporary indices.");
- for (int k = 0; k < gcount; k++) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (gparts[k].time_bin == time_bin_inhibited)
- error("Inhibited particle present in space_split()");
- if (gparts[k].time_bin == time_bin_not_created)
- error("Extra particle present in space_split()");
-#endif
- gbuff[k].x[0] = gparts[k].x[0];
- gbuff[k].x[1] = gparts[k].x[1];
- gbuff[k].x[2] = gparts[k].x[2];
- }
- }
- if (scount > 0) {
- if (swift_memalign("tempsbuff", (void **)&sbuff, SWIFT_STRUCT_ALIGNMENT,
- sizeof(struct cell_buff) * scount) != 0)
- error("Failed to allocate temporary indices.");
- for (int k = 0; k < scount; k++) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (sparts[k].time_bin == time_bin_inhibited)
- error("Inhibited particle present in space_split()");
- if (sparts[k].time_bin == time_bin_not_created)
- error("Extra particle present in space_split()");
-#endif
- sbuff[k].x[0] = sparts[k].x[0];
- sbuff[k].x[1] = sparts[k].x[1];
- sbuff[k].x[2] = sparts[k].x[2];
- }
- }
- if (bcount > 0) {
- if (swift_memalign("tempbbuff", (void **)&bbuff, SWIFT_STRUCT_ALIGNMENT,
- sizeof(struct cell_buff) * bcount) != 0)
- error("Failed to allocate temporary indices.");
- for (int k = 0; k < bcount; k++) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (bparts[k].time_bin == time_bin_inhibited)
- error("Inhibited particle present in space_split()");
- if (bparts[k].time_bin == time_bin_not_created)
- error("Extra particle present in space_split()");
-#endif
- bbuff[k].x[0] = bparts[k].x[0];
- bbuff[k].x[1] = bparts[k].x[1];
- bbuff[k].x[2] = bparts[k].x[2];
- }
- }
- if (sink_count > 0) {
- if (swift_memalign("temp_sink_buff", (void **)&sink_buff,
- SWIFT_STRUCT_ALIGNMENT,
- sizeof(struct cell_buff) * sink_count) != 0)
- error("Failed to allocate temporary indices.");
- for (int k = 0; k < sink_count; k++) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (sinks[k].time_bin == time_bin_inhibited)
- error("Inhibited particle present in space_split()");
- if (sinks[k].time_bin == time_bin_not_created)
- error("Extra particle present in space_split()");
-#endif
- sink_buff[k].x[0] = sinks[k].x[0];
- sink_buff[k].x[1] = sinks[k].x[1];
- sink_buff[k].x[2] = sinks[k].x[2];
- }
- }
- }
-
- /* If the depth is too large, we have a problem and should stop. */
- if (depth > space_cell_maxdepth) {
- error(
- "Exceeded maximum depth (%d) when splitting cells, aborting. This is "
- "most likely due to having too many particles at the exact same "
- "position, making the construction of a tree impossible.",
- space_cell_maxdepth);
- }
-
- /* Split or let it be? */
- if ((with_self_gravity && gcount > space_splitsize) ||
- (!with_self_gravity &&
- (count > space_splitsize || scount > space_splitsize))) {
-
- /* No longer just a leaf. */
- c->split = 1;
-
- /* Create the cell's progeny. */
- space_getcells(s, 8, c->progeny);
- for (int k = 0; k < 8; k++) {
- struct cell *cp = c->progeny[k];
- cp->hydro.count = 0;
- cp->grav.count = 0;
- cp->stars.count = 0;
- cp->sinks.count = 0;
- cp->black_holes.count = 0;
- cp->hydro.count_total = 0;
- cp->grav.count_total = 0;
- cp->sinks.count_total = 0;
- cp->stars.count_total = 0;
- cp->black_holes.count_total = 0;
- cp->hydro.ti_old_part = c->hydro.ti_old_part;
- cp->grav.ti_old_part = c->grav.ti_old_part;
- cp->grav.ti_old_multipole = c->grav.ti_old_multipole;
- cp->stars.ti_old_part = c->stars.ti_old_part;
- cp->sinks.ti_old_part = c->sinks.ti_old_part;
- cp->black_holes.ti_old_part = c->black_holes.ti_old_part;
- cp->loc[0] = c->loc[0];
- cp->loc[1] = c->loc[1];
- cp->loc[2] = c->loc[2];
- cp->width[0] = c->width[0] / 2;
- cp->width[1] = c->width[1] / 2;
- cp->width[2] = c->width[2] / 2;
- cp->dmin = c->dmin / 2;
- if (k & 4) cp->loc[0] += cp->width[0];
- if (k & 2) cp->loc[1] += cp->width[1];
- if (k & 1) cp->loc[2] += cp->width[2];
- cp->depth = c->depth + 1;
- cp->split = 0;
- cp->hydro.h_max = 0.f;
- cp->hydro.dx_max_part = 0.f;
- cp->hydro.dx_max_sort = 0.f;
- cp->stars.h_max = 0.f;
- cp->stars.dx_max_part = 0.f;
- cp->stars.dx_max_sort = 0.f;
- cp->sinks.r_cut_max = 0.f;
- cp->sinks.dx_max_part = 0.f;
- cp->black_holes.h_max = 0.f;
- cp->black_holes.dx_max_part = 0.f;
- cp->nodeID = c->nodeID;
- cp->parent = c;
- cp->top = c->top;
- cp->super = NULL;
- cp->hydro.super = NULL;
- cp->grav.super = NULL;
- cp->flags = 0;
- star_formation_logger_init(&cp->stars.sfh);
-#ifdef WITH_MPI
- cp->mpi.tag = -1;
-#endif // WITH_MPI
-#if defined(SWIFT_DEBUG_CHECKS) || defined(SWIFT_CELL_GRAPH)
- cp->cellID = last_cell_id++;
-#endif
- }
-
- /* Split the cell's particle data. */
- cell_split(c, c->hydro.parts - s->parts, c->stars.parts - s->sparts,
- c->black_holes.parts - s->bparts, c->sinks.parts - s->sinks,
- buff, sbuff, bbuff, gbuff, sink_buff);
-
- /* Buffers for the progenitors */
- struct cell_buff *progeny_buff = buff, *progeny_gbuff = gbuff,
- *progeny_sbuff = sbuff, *progeny_bbuff = bbuff,
- *progeny_sink_buff = sink_buff;
-
- for (int k = 0; k < 8; k++) {
-
- /* Get the progenitor */
- struct cell *cp = c->progeny[k];
-
- /* Remove any progeny with zero particles. */
- if (cp->hydro.count == 0 && cp->grav.count == 0 && cp->stars.count == 0 &&
- cp->black_holes.count == 0 && cp->sinks.count == 0) {
-
- space_recycle(s, cp);
- c->progeny[k] = NULL;
-
- } else {
-
- /* Recurse */
- space_split_recursive(s, cp, progeny_buff, progeny_sbuff, progeny_bbuff,
- progeny_gbuff, progeny_sink_buff);
-
- /* Update the pointers in the buffers */
- progeny_buff += cp->hydro.count;
- progeny_gbuff += cp->grav.count;
- progeny_sbuff += cp->stars.count;
- progeny_bbuff += cp->black_holes.count;
- progeny_sink_buff += cp->sinks.count;
-
- /* Update the cell-wide properties */
- h_max = max(h_max, cp->hydro.h_max);
- stars_h_max = max(stars_h_max, cp->stars.h_max);
- black_holes_h_max = max(black_holes_h_max, cp->black_holes.h_max);
- sinks_h_max = max(sinks_h_max, cp->sinks.r_cut_max);
-
- ti_hydro_end_min = min(ti_hydro_end_min, cp->hydro.ti_end_min);
- ti_hydro_end_max = max(ti_hydro_end_max, cp->hydro.ti_end_max);
- ti_hydro_beg_max = max(ti_hydro_beg_max, cp->hydro.ti_beg_max);
- ti_gravity_end_min = min(ti_gravity_end_min, cp->grav.ti_end_min);
- ti_gravity_end_max = max(ti_gravity_end_max, cp->grav.ti_end_max);
- ti_gravity_beg_max = max(ti_gravity_beg_max, cp->grav.ti_beg_max);
- ti_stars_end_min = min(ti_stars_end_min, cp->stars.ti_end_min);
- ti_stars_end_max = max(ti_stars_end_max, cp->stars.ti_end_max);
- ti_stars_beg_max = max(ti_stars_beg_max, cp->stars.ti_beg_max);
- ti_sinks_end_min = min(ti_sinks_end_min, cp->sinks.ti_end_min);
- ti_sinks_end_max = max(ti_sinks_end_max, cp->sinks.ti_end_max);
- ti_sinks_beg_max = max(ti_sinks_beg_max, cp->sinks.ti_beg_max);
- ti_black_holes_end_min =
- min(ti_black_holes_end_min, cp->black_holes.ti_end_min);
- ti_black_holes_end_max =
- max(ti_black_holes_end_max, cp->black_holes.ti_end_max);
- ti_black_holes_beg_max =
- max(ti_black_holes_beg_max, cp->black_holes.ti_beg_max);
-
- star_formation_logger_add(&c->stars.sfh, &cp->stars.sfh);
-
- /* Increase the depth */
- maxdepth = max(maxdepth, cp->maxdepth);
- }
- }
-
- /* Deal with the multipole */
- if (s->with_self_gravity) {
-
- /* Reset everything */
- gravity_reset(c->grav.multipole);
-
- /* Compute CoM and bulk velocity from all progenies */
- double CoM[3] = {0., 0., 0.};
- double vel[3] = {0., 0., 0.};
- float max_delta_vel[3] = {0.f, 0.f, 0.f};
- float min_delta_vel[3] = {0.f, 0.f, 0.f};
- double mass = 0.;
-
- for (int k = 0; k < 8; ++k) {
- if (c->progeny[k] != NULL) {
- const struct gravity_tensors *m = c->progeny[k]->grav.multipole;
-
- mass += m->m_pole.M_000;
-
- CoM[0] += m->CoM[0] * m->m_pole.M_000;
- CoM[1] += m->CoM[1] * m->m_pole.M_000;
- CoM[2] += m->CoM[2] * m->m_pole.M_000;
-
- vel[0] += m->m_pole.vel[0] * m->m_pole.M_000;
- vel[1] += m->m_pole.vel[1] * m->m_pole.M_000;
- vel[2] += m->m_pole.vel[2] * m->m_pole.M_000;
-
- max_delta_vel[0] = max(m->m_pole.max_delta_vel[0], max_delta_vel[0]);
- max_delta_vel[1] = max(m->m_pole.max_delta_vel[1], max_delta_vel[1]);
- max_delta_vel[2] = max(m->m_pole.max_delta_vel[2], max_delta_vel[2]);
-
- min_delta_vel[0] = min(m->m_pole.min_delta_vel[0], min_delta_vel[0]);
- min_delta_vel[1] = min(m->m_pole.min_delta_vel[1], min_delta_vel[1]);
- min_delta_vel[2] = min(m->m_pole.min_delta_vel[2], min_delta_vel[2]);
- }
- }
-
- /* Final operation on the CoM and bulk velocity */
- const double inv_mass = 1. / mass;
- c->grav.multipole->CoM[0] = CoM[0] * inv_mass;
- c->grav.multipole->CoM[1] = CoM[1] * inv_mass;
- c->grav.multipole->CoM[2] = CoM[2] * inv_mass;
- c->grav.multipole->m_pole.vel[0] = vel[0] * inv_mass;
- c->grav.multipole->m_pole.vel[1] = vel[1] * inv_mass;
- c->grav.multipole->m_pole.vel[2] = vel[2] * inv_mass;
-
- /* Min max velocity along each axis */
- c->grav.multipole->m_pole.max_delta_vel[0] = max_delta_vel[0];
- c->grav.multipole->m_pole.max_delta_vel[1] = max_delta_vel[1];
- c->grav.multipole->m_pole.max_delta_vel[2] = max_delta_vel[2];
- c->grav.multipole->m_pole.min_delta_vel[0] = min_delta_vel[0];
- c->grav.multipole->m_pole.min_delta_vel[1] = min_delta_vel[1];
- c->grav.multipole->m_pole.min_delta_vel[2] = min_delta_vel[2];
-
- /* Now shift progeny multipoles and add them up */
- struct multipole temp;
- double r_max = 0.;
- for (int k = 0; k < 8; ++k) {
- if (c->progeny[k] != NULL) {
- const struct cell *cp = c->progeny[k];
- const struct multipole *m = &cp->grav.multipole->m_pole;
-
- /* Contribution to multipole */
- gravity_M2M(&temp, m, c->grav.multipole->CoM,
- cp->grav.multipole->CoM);
- gravity_multipole_add(&c->grav.multipole->m_pole, &temp);
-
- /* Upper limit of max CoM<->gpart distance */
- const double dx =
- c->grav.multipole->CoM[0] - cp->grav.multipole->CoM[0];
- const double dy =
- c->grav.multipole->CoM[1] - cp->grav.multipole->CoM[1];
- const double dz =
- c->grav.multipole->CoM[2] - cp->grav.multipole->CoM[2];
- const double r2 = dx * dx + dy * dy + dz * dz;
- r_max = max(r_max, cp->grav.multipole->r_max + sqrt(r2));
- }
- }
-
- /* Alternative upper limit of max CoM<->gpart distance */
- const double dx =
- c->grav.multipole->CoM[0] > c->loc[0] + c->width[0] / 2.
- ? c->grav.multipole->CoM[0] - c->loc[0]
- : c->loc[0] + c->width[0] - c->grav.multipole->CoM[0];
- const double dy =
- c->grav.multipole->CoM[1] > c->loc[1] + c->width[1] / 2.
- ? c->grav.multipole->CoM[1] - c->loc[1]
- : c->loc[1] + c->width[1] - c->grav.multipole->CoM[1];
- const double dz =
- c->grav.multipole->CoM[2] > c->loc[2] + c->width[2] / 2.
- ? c->grav.multipole->CoM[2] - c->loc[2]
- : c->loc[2] + c->width[2] - c->grav.multipole->CoM[2];
-
- /* Take minimum of both limits */
- c->grav.multipole->r_max = min(r_max, sqrt(dx * dx + dy * dy + dz * dz));
-
- /* Store the value at rebuild time */
- c->grav.multipole->r_max_rebuild = c->grav.multipole->r_max;
- c->grav.multipole->CoM_rebuild[0] = c->grav.multipole->CoM[0];
- c->grav.multipole->CoM_rebuild[1] = c->grav.multipole->CoM[1];
- c->grav.multipole->CoM_rebuild[2] = c->grav.multipole->CoM[2];
-
- /* Compute the multipole power */
- gravity_multipole_compute_power(&c->grav.multipole->m_pole);
-
- } /* Deal with gravity */
- } /* Split or let it be? */
-
- /* Otherwise, collect the data from the particles this cell. */
- else {
-
- /* Clear the progeny. */
- bzero(c->progeny, sizeof(struct cell *) * 8);
- c->split = 0;
- maxdepth = c->depth;
-
- ti_hydro_end_min = max_nr_timesteps;
- ti_hydro_end_max = 0;
- ti_hydro_beg_max = 0;
-
- ti_gravity_end_min = max_nr_timesteps;
- ti_gravity_end_max = 0;
- ti_gravity_beg_max = 0;
-
- ti_stars_end_min = max_nr_timesteps;
- ti_stars_end_max = 0;
- ti_stars_beg_max = 0;
-
- ti_black_holes_end_min = max_nr_timesteps;
- ti_black_holes_end_max = 0;
- ti_black_holes_beg_max = 0;
-
- /* parts: Get dt_min/dt_max and h_max. */
- for (int k = 0; k < count; k++) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (parts[k].time_bin == time_bin_not_created)
- error("Extra particle present in space_split()");
- if (parts[k].time_bin == time_bin_inhibited)
- error("Inhibited particle present in space_split()");
-#endif
-
- /* When does this particle's time-step start and end? */
- const timebin_t time_bin = parts[k].time_bin;
- const integertime_t ti_end = get_integer_time_end(ti_current, time_bin);
- const integertime_t ti_beg = get_integer_time_begin(ti_current, time_bin);
-
- ti_hydro_end_min = min(ti_hydro_end_min, ti_end);
- ti_hydro_end_max = max(ti_hydro_end_max, ti_end);
- ti_hydro_beg_max = max(ti_hydro_beg_max, ti_beg);
-
- h_max = max(h_max, parts[k].h);
-
- /* Collect SFR from the particles after rebuilt */
- star_formation_logger_log_inactive_part(&parts[k], &xparts[k],
- &c->stars.sfh);
- }
-
- /* xparts: Reset x_diff */
- for (int k = 0; k < count; k++) {
- xparts[k].x_diff[0] = 0.f;
- xparts[k].x_diff[1] = 0.f;
- xparts[k].x_diff[2] = 0.f;
- }
-
- /* gparts: Get dt_min/dt_max. */
- for (int k = 0; k < gcount; k++) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (gparts[k].time_bin == time_bin_not_created)
- error("Extra g-particle present in space_split()");
- if (gparts[k].time_bin == time_bin_inhibited)
- error("Inhibited g-particle present in space_split()");
-#endif
-
- /* When does this particle's time-step start and end? */
- const timebin_t time_bin = gparts[k].time_bin;
- const integertime_t ti_end = get_integer_time_end(ti_current, time_bin);
- const integertime_t ti_beg = get_integer_time_begin(ti_current, time_bin);
-
- ti_gravity_end_min = min(ti_gravity_end_min, ti_end);
- ti_gravity_end_max = max(ti_gravity_end_max, ti_end);
- ti_gravity_beg_max = max(ti_gravity_beg_max, ti_beg);
- }
-
- /* sparts: Get dt_min/dt_max */
- for (int k = 0; k < scount; k++) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (sparts[k].time_bin == time_bin_not_created)
- error("Extra s-particle present in space_split()");
- if (sparts[k].time_bin == time_bin_inhibited)
- error("Inhibited s-particle present in space_split()");
-#endif
-
- /* When does this particle's time-step start and end? */
- const timebin_t time_bin = sparts[k].time_bin;
- const integertime_t ti_end = get_integer_time_end(ti_current, time_bin);
- const integertime_t ti_beg = get_integer_time_begin(ti_current, time_bin);
-
- ti_stars_end_min = min(ti_stars_end_min, ti_end);
- ti_stars_end_max = max(ti_stars_end_max, ti_end);
- ti_stars_beg_max = max(ti_stars_beg_max, ti_beg);
-
- stars_h_max = max(stars_h_max, sparts[k].h);
-
- /* Reset x_diff */
- sparts[k].x_diff[0] = 0.f;
- sparts[k].x_diff[1] = 0.f;
- sparts[k].x_diff[2] = 0.f;
- }
-
- /* sinks: Get dt_min/dt_max */
- for (int k = 0; k < sink_count; k++) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (sinks[k].time_bin == time_bin_not_created)
- error("Extra sink-particle present in space_split()");
- if (sinks[k].time_bin == time_bin_inhibited)
- error("Inhibited sink-particle present in space_split()");
-#endif
-
- /* When does this particle's time-step start and end? */
- const timebin_t time_bin = sinks[k].time_bin;
- const integertime_t ti_end = get_integer_time_end(ti_current, time_bin);
- const integertime_t ti_beg = get_integer_time_begin(ti_current, time_bin);
-
- ti_sinks_end_min = min(ti_sinks_end_min, ti_end);
- ti_sinks_end_max = max(ti_sinks_end_max, ti_end);
- ti_sinks_beg_max = max(ti_sinks_beg_max, ti_beg);
-
- sinks_h_max = max(sinks_h_max, sinks[k].r_cut);
-
- /* Reset x_diff */
- sinks[k].x_diff[0] = 0.f;
- sinks[k].x_diff[1] = 0.f;
- sinks[k].x_diff[2] = 0.f;
- }
-
- /* bparts: Get dt_min/dt_max */
- for (int k = 0; k < bcount; k++) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (bparts[k].time_bin == time_bin_not_created)
- error("Extra b-particle present in space_split()");
- if (bparts[k].time_bin == time_bin_inhibited)
- error("Inhibited b-particle present in space_split()");
-#endif
-
- /* When does this particle's time-step start and end? */
- const timebin_t time_bin = bparts[k].time_bin;
- const integertime_t ti_end = get_integer_time_end(ti_current, time_bin);
- const integertime_t ti_beg = get_integer_time_begin(ti_current, time_bin);
-
- ti_black_holes_end_min = min(ti_black_holes_end_min, ti_end);
- ti_black_holes_end_max = max(ti_black_holes_end_max, ti_end);
- ti_black_holes_beg_max = max(ti_black_holes_beg_max, ti_beg);
-
- black_holes_h_max = max(black_holes_h_max, bparts[k].h);
-
- /* Reset x_diff */
- bparts[k].x_diff[0] = 0.f;
- bparts[k].x_diff[1] = 0.f;
- bparts[k].x_diff[2] = 0.f;
- }
-
- /* Construct the multipole and the centre of mass*/
- if (s->with_self_gravity) {
- if (gcount > 0) {
-
- gravity_P2M(c->grav.multipole, c->grav.parts, c->grav.count,
- e->gravity_properties);
-
- /* Compute the multipole power */
- gravity_multipole_compute_power(&c->grav.multipole->m_pole);
-
- } else {
-
- /* No gparts in that leaf cell */
-
- /* Set the values to something sensible */
- gravity_multipole_init(&c->grav.multipole->m_pole);
- if (c->nodeID == engine_rank) {
- c->grav.multipole->CoM[0] = c->loc[0] + c->width[0] / 2.;
- c->grav.multipole->CoM[1] = c->loc[1] + c->width[1] / 2.;
- c->grav.multipole->CoM[2] = c->loc[2] + c->width[2] / 2.;
- c->grav.multipole->r_max = 0.;
- }
- }
-
- /* Store the value at rebuild time */
- c->grav.multipole->r_max_rebuild = c->grav.multipole->r_max;
- c->grav.multipole->CoM_rebuild[0] = c->grav.multipole->CoM[0];
- c->grav.multipole->CoM_rebuild[1] = c->grav.multipole->CoM[1];
- c->grav.multipole->CoM_rebuild[2] = c->grav.multipole->CoM[2];
- }
- }
-
- /* Set the values for this cell. */
- c->hydro.h_max = h_max;
- c->hydro.ti_end_min = ti_hydro_end_min;
- c->hydro.ti_end_max = ti_hydro_end_max;
- c->hydro.ti_beg_max = ti_hydro_beg_max;
- c->grav.ti_end_min = ti_gravity_end_min;
- c->grav.ti_end_max = ti_gravity_end_max;
- c->grav.ti_beg_max = ti_gravity_beg_max;
- c->stars.ti_end_min = ti_stars_end_min;
- c->stars.ti_end_max = ti_stars_end_max;
- c->stars.ti_beg_max = ti_stars_beg_max;
- c->stars.h_max = stars_h_max;
- c->sinks.ti_end_min = ti_sinks_end_min;
- c->sinks.ti_end_max = ti_sinks_end_max;
- c->sinks.ti_beg_max = ti_sinks_beg_max;
- c->sinks.r_cut_max = sinks_h_max;
- c->black_holes.ti_end_min = ti_black_holes_end_min;
- c->black_holes.ti_end_max = ti_black_holes_end_max;
- c->black_holes.ti_beg_max = ti_black_holes_beg_max;
- c->black_holes.h_max = black_holes_h_max;
- c->maxdepth = maxdepth;
-
- /* Set ownership according to the start of the parts array. */
- if (s->nr_parts > 0)
- c->owner = ((c->hydro.parts - s->parts) % s->nr_parts) * s->nr_queues /
- s->nr_parts;
- else if (s->nr_sinks > 0)
- c->owner = ((c->sinks.parts - s->sinks) % s->nr_sinks) * s->nr_queues /
- s->nr_sinks;
- else if (s->nr_sparts > 0)
- c->owner = ((c->stars.parts - s->sparts) % s->nr_sparts) * s->nr_queues /
- s->nr_sparts;
- else if (s->nr_bparts > 0)
- c->owner = ((c->black_holes.parts - s->bparts) % s->nr_bparts) *
- s->nr_queues / s->nr_bparts;
- else if (s->nr_gparts > 0)
- c->owner = ((c->grav.parts - s->gparts) % s->nr_gparts) * s->nr_queues /
- s->nr_gparts;
- else
- c->owner = 0; /* Ok, there is really nothing on this rank... */
-
- /* Store the global max depth */
- if (c->depth == 0) atomic_max(&s->maxdepth, maxdepth);
-
- /* Clean up. */
- if (allocate_buffer) {
- if (buff != NULL) swift_free("tempbuff", buff);
- if (gbuff != NULL) swift_free("tempgbuff", gbuff);
- if (sbuff != NULL) swift_free("tempsbuff", sbuff);
- if (bbuff != NULL) swift_free("tempbbuff", bbuff);
- if (sink_buff != NULL) swift_free("temp_sink_buff", sink_buff);
- }
-}
-
-/**
- * @brief #threadpool mapper function to split cells if they contain
- * too many particles.
- *
- * @param map_data Pointer towards the top-cells.
- * @param num_cells The number of cells to treat.
- * @param extra_data Pointers to the #space.
- */
-void space_split_mapper(void *map_data, int num_cells, void *extra_data) {
-
- /* Unpack the inputs. */
- struct space *s = (struct space *)extra_data;
- struct cell *cells_top = s->cells_top;
- int *local_cells_with_particles = (int *)map_data;
-
- /* Loop over the non-empty cells */
- for (int ind = 0; ind < num_cells; ind++) {
- struct cell *c = &cells_top[local_cells_with_particles[ind]];
- space_split_recursive(s, c, NULL, NULL, NULL, NULL, NULL);
- }
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* All cells and particles should have consistent h_max values. */
- for (int ind = 0; ind < num_cells; ind++) {
- int depth = 0;
- const struct cell *c = &cells_top[local_cells_with_particles[ind]];
- if (!checkCellhdxmax(c, &depth)) message(" at cell depth %d", depth);
- }
-#endif
-}
-
-/**
- * @brief Return a used cell to the buffer of unused sub-cells.
- *
- * @param s The #space.
- * @param c The #cell.
- */
-void space_recycle(struct space *s, struct cell *c) {
-
- /* Clear the cell. */
- if (lock_destroy(&c->hydro.lock) != 0 || lock_destroy(&c->grav.plock) != 0 ||
- lock_destroy(&c->grav.mlock) != 0 || lock_destroy(&c->stars.lock) != 0 ||
- lock_destroy(&c->sinks.lock) != 0 ||
- lock_destroy(&c->sinks.sink_formation_lock) != 0 ||
- lock_destroy(&c->black_holes.lock) != 0 ||
- lock_destroy(&c->grav.star_formation_lock) != 0 ||
- lock_destroy(&c->stars.star_formation_lock) != 0)
- error("Failed to destroy spinlocks.");
-
- /* Lock the space. */
- lock_lock(&s->lock);
-
- /* Hook the multipole back in the buffer */
- if (s->with_self_gravity) {
- c->grav.multipole->next = s->multipoles_sub;
- s->multipoles_sub = c->grav.multipole;
- }
-
- /* Hook this cell into the buffer. */
- c->next = s->cells_sub;
- s->cells_sub = c;
- s->tot_cells -= 1;
-
- /* Unlock the space. */
- lock_unlock_blind(&s->lock);
-}
-
-/**
- * @brief Return a list of used cells to the buffer of unused sub-cells.
- *
- * @param s The #space.
- * @param cell_list_begin Pointer to the first #cell in the linked list of
- * cells joined by their @c next pointers.
- * @param cell_list_end Pointer to the last #cell in the linked list of
- * cells joined by their @c next pointers. It is assumed that this
- * cell's @c next pointer is @c NULL.
- * @param multipole_list_begin Pointer to the first #multipole in the linked
- * list of
- * multipoles joined by their @c next pointers.
- * @param multipole_list_end Pointer to the last #multipole in the linked list
- * of
- * multipoles joined by their @c next pointers. It is assumed that this
- * multipole's @c next pointer is @c NULL.
- */
-void space_recycle_list(struct space *s, struct cell *cell_list_begin,
- struct cell *cell_list_end,
- struct gravity_tensors *multipole_list_begin,
- struct gravity_tensors *multipole_list_end) {
-
- int count = 0;
-
- /* Clean up the list of cells. */
- for (struct cell *c = cell_list_begin; c != NULL; c = c->next) {
- /* Clear the cell. */
- if (lock_destroy(&c->hydro.lock) != 0 ||
- lock_destroy(&c->grav.plock) != 0 ||
- lock_destroy(&c->grav.mlock) != 0 ||
- lock_destroy(&c->stars.lock) != 0 ||
- lock_destroy(&c->sinks.lock) != 0 ||
- lock_destroy(&c->sinks.sink_formation_lock) != 0 ||
- lock_destroy(&c->black_holes.lock) != 0 ||
- lock_destroy(&c->stars.star_formation_lock) != 0 ||
- lock_destroy(&c->grav.star_formation_lock) != 0)
- error("Failed to destroy spinlocks.");
-
- /* Count this cell. */
- count += 1;
- }
-
- /* Lock the space. */
- lock_lock(&s->lock);
-
- /* Hook the cells into the buffer. */
- cell_list_end->next = s->cells_sub;
- s->cells_sub = cell_list_begin;
- s->tot_cells -= count;
-
- /* Hook the multipoles into the buffer. */
- if (s->with_self_gravity) {
- multipole_list_end->next = s->multipoles_sub;
- s->multipoles_sub = multipole_list_begin;
- }
-
- /* Unlock the space. */
- lock_unlock_blind(&s->lock);
-}
-
-/**
- * @brief Get a new empty (sub-)#cell.
- *
- * If there are cells in the buffer, use the one at the end of the linked list.
- * If we have no cells, allocate a new chunk of memory and pick one from there.
- *
- * @param s The #space.
- * @param nr_cells Number of #cell to pick up.
- * @param cells Array of @c nr_cells #cell pointers in which to store the
- * new cells.
- */
-void space_getcells(struct space *s, int nr_cells, struct cell **cells) {
-
- /* Lock the space. */
- lock_lock(&s->lock);
-
- /* For each requested cell... */
- for (int j = 0; j < nr_cells; j++) {
-
- /* Is the cell buffer empty? */
- if (s->cells_sub == NULL) {
- if (swift_memalign("cells_sub", (void **)&s->cells_sub, cell_align,
- space_cellallocchunk * sizeof(struct cell)) != 0)
- error("Failed to allocate more cells.");
-
- /* Clear the newly-allocated cells. */
- bzero(s->cells_sub, sizeof(struct cell) * space_cellallocchunk);
-
- /* Constructed a linked list */
- for (int k = 0; k < space_cellallocchunk - 1; k++)
- s->cells_sub[k].next = &s->cells_sub[k + 1];
- s->cells_sub[space_cellallocchunk - 1].next = NULL;
- }
-
- /* Is the multipole buffer empty? */
- if (s->with_self_gravity && s->multipoles_sub == NULL) {
- if (swift_memalign(
- "multipoles_sub", (void **)&s->multipoles_sub, multipole_align,
- space_cellallocchunk * sizeof(struct gravity_tensors)) != 0)
- error("Failed to allocate more multipoles.");
-
- /* Constructed a linked list */
- for (int k = 0; k < space_cellallocchunk - 1; k++)
- s->multipoles_sub[k].next = &s->multipoles_sub[k + 1];
- s->multipoles_sub[space_cellallocchunk - 1].next = NULL;
- }
-
- /* Pick off the next cell. */
- cells[j] = s->cells_sub;
- s->cells_sub = cells[j]->next;
- s->tot_cells += 1;
-
- /* Hook the multipole */
- if (s->with_self_gravity) {
- cells[j]->grav.multipole = s->multipoles_sub;
- s->multipoles_sub = cells[j]->grav.multipole->next;
- }
- }
-
- /* Unlock the space. */
- lock_unlock_blind(&s->lock);
-
- /* Init some things in the cell we just got. */
- for (int j = 0; j < nr_cells; j++) {
- cell_free_hydro_sorts(cells[j]);
- cell_free_stars_sorts(cells[j]);
-
- struct gravity_tensors *temp = cells[j]->grav.multipole;
- bzero(cells[j], sizeof(struct cell));
- cells[j]->grav.multipole = temp;
- cells[j]->nodeID = -1;
- if (lock_init(&cells[j]->hydro.lock) != 0 ||
- lock_init(&cells[j]->grav.plock) != 0 ||
- lock_init(&cells[j]->grav.mlock) != 0 ||
- lock_init(&cells[j]->stars.lock) != 0 ||
- lock_init(&cells[j]->sinks.lock) != 0 ||
- lock_init(&cells[j]->sinks.sink_formation_lock) != 0 ||
- lock_init(&cells[j]->black_holes.lock) != 0 ||
- lock_init(&cells[j]->stars.star_formation_lock) != 0 ||
- lock_init(&cells[j]->grav.star_formation_lock) != 0)
- error("Failed to initialize cell spinlocks.");
- }
+ /* Otherwise, recurse. */
+ else
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL) rec_map_parts_xparts(c->progeny[k], fun);
}
/**
- * @brief Free sort arrays in any cells in the cell buffer.
+ * @brief Map a function to all particles (#part and #xpart) in a space.
*
- * @param s The #space.
+ * @param s The #space we are working in.
+ * @param fun Function pointer to apply on the particles in the cells.
*/
-void space_free_buff_sort_indices(struct space *s) {
- for (struct cell *finger = s->cells_sub; finger != NULL;
- finger = finger->next) {
- cell_free_hydro_sorts(finger);
- cell_free_stars_sorts(finger);
- }
+void space_map_parts_xparts(struct space *s,
+ void (*fun)(struct part *p, struct xpart *xp,
+ struct cell *c)) {
+
+ /* Call the recursive function on all higher-level cells. */
+ for (int cid = 0; cid < s->nr_cells; cid++)
+ rec_map_parts_xparts(&s->cells_top[cid], fun);
}
/**
- * @brief Construct the list of top-level cells that have any tasks in
- * their hierarchy on this MPI rank. Also construct the list of top-level
- * cells on any rank that have > 0 particles (of any kind).
- *
- * This assumes the list has been pre-allocated at a regrid.
+ * @brief Map a function to all particles in a cell recursively.
*
- * @param s The #space.
+ * @param c The #cell we are working in.
+ * @param full Map to all cells, including cells with sub-cells.
+ * @param fun Function pointer to apply on the cells.
+ * @param data Data passed to the function fun.
*/
-void space_list_useful_top_level_cells(struct space *s) {
-
- const ticks tic = getticks();
-
- s->nr_local_cells_with_tasks = 0;
- s->nr_cells_with_particles = 0;
-
- for (int i = 0; i < s->nr_cells; ++i) {
- struct cell *c = &s->cells_top[i];
-
- if (cell_has_tasks(c)) {
- s->local_cells_with_tasks_top[s->nr_local_cells_with_tasks] = i;
- s->nr_local_cells_with_tasks++;
- }
-
- const int has_particles =
- (c->hydro.count > 0) || (c->grav.count > 0) || (c->stars.count > 0) ||
- (c->black_holes.count > 0) || (c->sinks.count > 0) ||
- (c->grav.multipole != NULL && c->grav.multipole->m_pole.M_000 > 0.f);
-
- if (has_particles) {
- s->cells_with_particles_top[s->nr_cells_with_particles] = i;
- s->nr_cells_with_particles++;
- }
- }
- if (s->e->verbose) {
- message("Have %d local top-level cells with tasks (total=%d)",
- s->nr_local_cells_with_tasks, s->nr_cells);
- message("Have %d top-level cells with particles (total=%d)",
- s->nr_cells_with_particles, s->nr_cells);
- }
-
- if (s->e->verbose)
- message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
- clocks_getunit());
-}
-
-void space_synchronize_part_positions_mapper(void *map_data, int nr_parts,
- void *extra_data) {
- /* Unpack the data */
- const struct part *parts = (struct part *)map_data;
- struct space *s = (struct space *)extra_data;
- const ptrdiff_t offset = parts - s->parts;
- const struct xpart *xparts = s->xparts + offset;
-
- for (int k = 0; k < nr_parts; k++) {
-
- /* Get the particle */
- const struct part *p = &parts[k];
- const struct xpart *xp = &xparts[k];
-
- /* Skip unimportant particles */
- if (p->time_bin == time_bin_not_created ||
- p->time_bin == time_bin_inhibited)
- continue;
-
- /* Get its gravity friend */
- struct gpart *gp = p->gpart;
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (gp == NULL) error("Unlinked particle!");
-#endif
-
- /* Synchronize positions, velocities and masses */
- gp->x[0] = p->x[0];
- gp->x[1] = p->x[1];
- gp->x[2] = p->x[2];
-
- gp->v_full[0] = xp->v_full[0];
- gp->v_full[1] = xp->v_full[1];
- gp->v_full[2] = xp->v_full[2];
-
- gp->mass = hydro_get_mass(p);
- }
-}
-
-void space_synchronize_spart_positions_mapper(void *map_data, int nr_sparts,
- void *extra_data) {
- /* Unpack the data */
- const struct spart *sparts = (struct spart *)map_data;
-
- for (int k = 0; k < nr_sparts; k++) {
-
- /* Get the particle */
- const struct spart *sp = &sparts[k];
-
- /* Skip unimportant particles */
- if (sp->time_bin == time_bin_not_created ||
- sp->time_bin == time_bin_inhibited)
- continue;
-
- /* Get its gravity friend */
- struct gpart *gp = sp->gpart;
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (gp == NULL) error("Unlinked particle!");
-#endif
-
- /* Synchronize positions, velocities and masses */
- gp->x[0] = sp->x[0];
- gp->x[1] = sp->x[1];
- gp->x[2] = sp->x[2];
-
- gp->v_full[0] = sp->v[0];
- gp->v_full[1] = sp->v[1];
- gp->v_full[2] = sp->v[2];
-
- gp->mass = sp->mass;
- }
-}
-
-void space_synchronize_bpart_positions_mapper(void *map_data, int nr_bparts,
- void *extra_data) {
- /* Unpack the data */
- const struct bpart *bparts = (struct bpart *)map_data;
-
- for (int k = 0; k < nr_bparts; k++) {
-
- /* Get the particle */
- const struct bpart *bp = &bparts[k];
-
- /* Skip unimportant particles */
- if (bp->time_bin == time_bin_not_created ||
- bp->time_bin == time_bin_inhibited)
- continue;
-
- /* Get its gravity friend */
- struct gpart *gp = bp->gpart;
-
-#ifdef SWIFT_DEBUG_CHECKS
- if (gp == NULL) error("Unlinked particle!");
-#endif
-
- /* Synchronize positions, velocities and masses */
- gp->x[0] = bp->x[0];
- gp->x[1] = bp->x[1];
- gp->x[2] = bp->x[2];
-
- gp->v_full[0] = bp->v[0];
- gp->v_full[1] = bp->v[1];
- gp->v_full[2] = bp->v[2];
+static void rec_map_cells_post(struct cell *c, int full,
+ void (*fun)(struct cell *c, void *data),
+ void *data) {
+ /* Recurse. */
+ if (c->split)
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL)
+ rec_map_cells_post(c->progeny[k], full, fun, data);
- gp->mass = bp->mass;
- }
+ /* No progeny? */
+ if (full || !c->split) fun(c, data);
}
-void space_synchronize_sink_positions_mapper(void *map_data, int nr_sinks,
- void *extra_data) {
- /* Unpack the data */
- const struct sink *sinks = (struct sink *)map_data;
-
- for (int k = 0; k < nr_sinks; k++) {
-
- /* Get the particle */
- const struct sink *sink = &sinks[k];
+/**
+ * @brief Map a function to all particles in a aspace.
+ *
+ * @param s The #space we are working in.
+ * @param full Map to all cells, including cells with sub-cells.
+ * @param fun Function pointer to apply on the cells.
+ * @param data Data passed to the function fun.
+ */
+void space_map_cells_post(struct space *s, int full,
+ void (*fun)(struct cell *c, void *data), void *data) {
- /* Skip unimportant particles */
- if (sink->time_bin == time_bin_not_created ||
- sink->time_bin == time_bin_inhibited)
- continue;
+ /* Call the recursive function on all higher-level cells. */
+ for (int cid = 0; cid < s->nr_cells; cid++)
+ rec_map_cells_post(&s->cells_top[cid], full, fun, data);
+}
- /* Get its gravity friend */
- struct gpart *gp = sink->gpart;
+static void rec_map_cells_pre(struct cell *c, int full,
+ void (*fun)(struct cell *c, void *data),
+ void *data) {
-#ifdef SWIFT_DEBUG_CHECKS
- if (gp == NULL) error("Unlinked particle!");
-#endif
+ /* No progeny? */
+ if (full || !c->split) fun(c, data);
- /* Synchronize positions, velocities and masses */
- gp->x[0] = sink->x[0];
- gp->x[1] = sink->x[1];
- gp->x[2] = sink->x[2];
+ /* Recurse. */
+ if (c->split)
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL)
+ rec_map_cells_pre(c->progeny[k], full, fun, data);
+}
- gp->v_full[0] = sink->v[0];
- gp->v_full[1] = sink->v[1];
- gp->v_full[2] = sink->v[2];
+/**
+ * @brief Calls function fun on the cells in the space s
+ *
+ * @param s The #space
+ * @param full If true calls the function on all cells and not just on leaves
+ * @param fun The function to call.
+ * @param data Additional data passed to fun() when called
+ */
+void space_map_cells_pre(struct space *s, int full,
+ void (*fun)(struct cell *c, void *data), void *data) {
- gp->mass = sink->mass;
- }
+ /* Call the recursive function on all higher-level cells. */
+ for (int cid = 0; cid < s->nr_cells; cid++)
+ rec_map_cells_pre(&s->cells_top[cid], full, fun, data);
}
/**
- * @brief Make sure the baryon particles are at the same position and
- * have the same velocity and mass as their #gpart friends.
+ * @brief Get a new empty (sub-)#cell.
*
- * We copy the baryon particle properties to the #gpart type-by-type.
+ * If there are cells in the buffer, use the one at the end of the linked list.
+ * If we have no cells, allocate a new chunk of memory and pick one from there.
*
* @param s The #space.
+ * @param nr_cells Number of #cell to pick up.
+ * @param cells Array of @c nr_cells #cell pointers in which to store the
+ * new cells.
*/
-void space_synchronize_particle_positions(struct space *s) {
-
- const ticks tic = getticks();
-
- if (s->nr_gparts > 0 && s->nr_parts > 0)
- threadpool_map(&s->e->threadpool, space_synchronize_part_positions_mapper,
- s->parts, s->nr_parts, sizeof(struct part),
- threadpool_auto_chunk_size, (void *)s);
-
- if (s->nr_gparts > 0 && s->nr_sparts > 0)
- threadpool_map(&s->e->threadpool, space_synchronize_spart_positions_mapper,
- s->sparts, s->nr_sparts, sizeof(struct spart),
- threadpool_auto_chunk_size, /*extra_data=*/NULL);
-
- if (s->nr_gparts > 0 && s->nr_bparts > 0)
- threadpool_map(&s->e->threadpool, space_synchronize_bpart_positions_mapper,
- s->bparts, s->nr_bparts, sizeof(struct bpart),
- threadpool_auto_chunk_size, /*extra_data=*/NULL);
-
- if (s->nr_gparts > 0 && s->nr_sinks > 0)
- threadpool_map(&s->e->threadpool, space_synchronize_sink_positions_mapper,
- s->sinks, s->nr_sinks, sizeof(struct sink),
- threadpool_auto_chunk_size, /*extra_data=*/NULL);
-
- if (s->e->verbose)
- message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
- clocks_getunit());
-}
+void space_getcells(struct space *s, int nr_cells, struct cell **cells) {
-void space_first_init_parts_mapper(void *restrict map_data, int count,
- void *restrict extra_data) {
+ /* Lock the space. */
+ lock_lock(&s->lock);
- struct part *restrict p = (struct part *)map_data;
- const struct space *restrict s = (struct space *)extra_data;
- const struct engine *e = s->e;
+ /* For each requested cell... */
+ for (int j = 0; j < nr_cells; j++) {
- const ptrdiff_t delta = p - s->parts;
- struct xpart *restrict xp = s->xparts + delta;
+ /* Is the cell buffer empty? */
+ if (s->cells_sub == NULL) {
+ if (swift_memalign("cells_sub", (void **)&s->cells_sub, cell_align,
+ space_cellallocchunk * sizeof(struct cell)) != 0)
+ error("Failed to allocate more cells.");
- /* Extract some constants */
- const struct cosmology *cosmo = s->e->cosmology;
- const struct phys_const *phys_const = s->e->physical_constants;
- const struct unit_system *us = s->e->internal_units;
- const float a_factor_vel = cosmo->a;
+ /* Clear the newly-allocated cells. */
+ bzero(s->cells_sub, sizeof(struct cell) * space_cellallocchunk);
- const struct hydro_props *hydro_props = s->e->hydro_properties;
- const float u_init = hydro_props->initial_internal_energy;
- const float hydro_h_min_ratio = e->hydro_properties->h_min_ratio;
-
- const struct gravity_props *grav_props = s->e->gravity_properties;
- const int with_gravity = e->policy & engine_policy_self_gravity;
-
- const struct chemistry_global_data *chemistry = e->chemistry;
- const struct star_formation *star_formation = e->star_formation;
- const struct cooling_function_data *cool_func = e->cooling_func;
-
- /* Check that the smoothing lengths are non-zero */
- for (int k = 0; k < count; k++) {
- if (p[k].h <= 0.)
- error("Invalid value of smoothing length for part %lld h=%e", p[k].id,
- p[k].h);
-
- if (with_gravity) {
- const struct gpart *gp = p[k].gpart;
- const float softening = gravity_get_softening(gp, grav_props);
- p->h = max(p->h, softening * hydro_h_min_ratio);
+ /* Constructed a linked list */
+ for (int k = 0; k < space_cellallocchunk - 1; k++)
+ s->cells_sub[k].next = &s->cells_sub[k + 1];
+ s->cells_sub[space_cellallocchunk - 1].next = NULL;
}
- }
- /* Convert velocities to internal units */
- for (int k = 0; k < count; k++) {
- p[k].v[0] *= a_factor_vel;
- p[k].v[1] *= a_factor_vel;
- p[k].v[2] *= a_factor_vel;
+ /* Is the multipole buffer empty? */
+ if (s->with_self_gravity && s->multipoles_sub == NULL) {
+ if (swift_memalign(
+ "multipoles_sub", (void **)&s->multipoles_sub, multipole_align,
+ space_cellallocchunk * sizeof(struct gravity_tensors)) != 0)
+ error("Failed to allocate more multipoles.");
-#ifdef HYDRO_DIMENSION_2D
- p[k].x[2] = 0.f;
- p[k].v[2] = 0.f;
-#endif
+ /* Constructed a linked list */
+ for (int k = 0; k < space_cellallocchunk - 1; k++)
+ s->multipoles_sub[k].next = &s->multipoles_sub[k + 1];
+ s->multipoles_sub[space_cellallocchunk - 1].next = NULL;
+ }
-#ifdef HYDRO_DIMENSION_1D
- p[k].x[1] = p[k].x[2] = 0.f;
- p[k].v[1] = p[k].v[2] = 0.f;
-#endif
- }
+ /* Pick off the next cell. */
+ cells[j] = s->cells_sub;
+ s->cells_sub = cells[j]->next;
+ s->tot_cells += 1;
- /* Overwrite the internal energy? */
- if (u_init > 0.f) {
- for (int k = 0; k < count; k++) {
- hydro_set_init_internal_energy(&p[k], u_init);
+ /* Hook the multipole */
+ if (s->with_self_gravity) {
+ cells[j]->grav.multipole = s->multipoles_sub;
+ s->multipoles_sub = cells[j]->grav.multipole->next;
}
}
- /* Initialise the rest */
- for (int k = 0; k < count; k++) {
-
- hydro_first_init_part(&p[k], &xp[k]);
- p[k].limiter_data.min_ngb_time_bin = num_time_bins + 1;
- p[k].limiter_data.wakeup = time_bin_not_awake;
- p[k].limiter_data.to_be_synchronized = 0;
-
-#ifdef WITH_LOGGER
- logger_part_data_init(&xp[k].logger_data);
-#endif
-
- /* Also initialise the chemistry */
- chemistry_first_init_part(phys_const, us, cosmo, chemistry, &p[k], &xp[k]);
-
- /* Also initialise the pressure floor */
- pressure_floor_first_init_part(phys_const, us, cosmo, &p[k], &xp[k]);
-
- /* Also initialise the star formation */
- star_formation_first_init_part(phys_const, us, cosmo, star_formation, &p[k],
- &xp[k]);
-
- /* And the cooling */
- cooling_first_init_part(phys_const, us, hydro_props, cosmo, cool_func,
- &p[k], &xp[k]);
-
- /* And the tracers */
- tracers_first_init_xpart(&p[k], &xp[k], us, phys_const, cosmo, hydro_props,
- cool_func);
-
- /* And the black hole markers */
- black_holes_mark_part_as_not_swallowed(&p[k].black_holes_data);
-
- /* And the radiative transfer */
- rt_first_init_part(&p[k]);
+ /* Unlock the space. */
+ lock_unlock_blind(&s->lock);
-#ifdef SWIFT_DEBUG_CHECKS
- /* Check part->gpart->part linkeage. */
- if (p[k].gpart && p[k].gpart->id_or_neg_offset != -(k + delta))
- error("Invalid gpart -> part link");
+ /* Init some things in the cell we just got. */
+ for (int j = 0; j < nr_cells; j++) {
+ cell_free_hydro_sorts(cells[j]);
+ cell_free_stars_sorts(cells[j]);
- /* Initialise the time-integration check variables */
- p[k].ti_drift = 0;
- p[k].ti_kick = 0;
-#endif
+ struct gravity_tensors *temp = cells[j]->grav.multipole;
+ bzero(cells[j], sizeof(struct cell));
+ cells[j]->grav.multipole = temp;
+ cells[j]->nodeID = -1;
+ if (lock_init(&cells[j]->hydro.lock) != 0 ||
+ lock_init(&cells[j]->grav.plock) != 0 ||
+ lock_init(&cells[j]->grav.mlock) != 0 ||
+ lock_init(&cells[j]->stars.lock) != 0 ||
+ lock_init(&cells[j]->sinks.lock) != 0 ||
+ lock_init(&cells[j]->sinks.sink_formation_lock) != 0 ||
+ lock_init(&cells[j]->black_holes.lock) != 0 ||
+ lock_init(&cells[j]->stars.star_formation_lock) != 0 ||
+ lock_init(&cells[j]->grav.star_formation_lock) != 0)
+ error("Failed to initialize cell spinlocks.");
}
}
/**
- * @brief Initialises all the particles by setting them into a valid state
+ * @brief Free sort arrays in any cells in the cell buffer.
*
- * Calls hydro_first_init_part() on all the particles
- * Calls chemistry_first_init_part() on all the particles
- * Calls cooling_first_init_part() on all the particles
+ * @param s The #space.
*/
-void space_first_init_parts(struct space *s, int verbose) {
-
- const ticks tic = getticks();
- if (s->nr_parts > 0)
- threadpool_map(&s->e->threadpool, space_first_init_parts_mapper, s->parts,
- s->nr_parts, sizeof(struct part), threadpool_auto_chunk_size,
- s);
-
- if (verbose)
- message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
- clocks_getunit());
-}
-
-void space_first_init_gparts_mapper(void *restrict map_data, int count,
- void *restrict extra_data) {
-
- struct gpart *restrict gp = (struct gpart *)map_data;
- const struct space *restrict s = (struct space *)extra_data;
-
- const struct cosmology *cosmo = s->e->cosmology;
- const float a_factor_vel = cosmo->a;
- const struct gravity_props *grav_props = s->e->gravity_properties;
-
- /* Convert velocities to internal units */
- for (int k = 0; k < count; k++) {
- gp[k].v_full[0] *= a_factor_vel;
- gp[k].v_full[1] *= a_factor_vel;
- gp[k].v_full[2] *= a_factor_vel;
-
-#ifdef HYDRO_DIMENSION_2D
- gp[k].x[2] = 0.f;
- gp[k].v_full[2] = 0.f;
-#endif
-
-#ifdef HYDRO_DIMENSION_1D
- gp[k].x[1] = gp[k].x[2] = 0.f;
- gp[k].v_full[1] = gp[k].v_full[2] = 0.f;
-#endif
- }
-
- /* Initialise the rest */
- for (int k = 0; k < count; k++) {
-
- gravity_first_init_gpart(&gp[k], grav_props);
-
-#ifdef WITH_LOGGER
- logger_part_data_init(&gp[k].logger_data);
-#endif
-
-#ifdef SWIFT_DEBUG_CHECKS
- /* Initialise the time-integration check variables */
- gp[k].ti_drift = 0;
- gp[k].ti_kick = 0;
- gp[k].ti_kick_mesh = 0;
-#endif
+void space_free_buff_sort_indices(struct space *s) {
+ for (struct cell *finger = s->cells_sub; finger != NULL;
+ finger = finger->next) {
+ cell_free_hydro_sorts(finger);
+ cell_free_stars_sorts(finger);
}
}
/**
- * @brief Initialises all the g-particles by setting them into a valid state
+ * @brief Construct the list of top-level cells that have any tasks in
+ * their hierarchy on this MPI rank. Also construct the list of top-level
+ * cells on any rank that have > 0 particles (of any kind).
*
- * Calls gravity_first_init_gpart() on all the particles
+ * This assumes the list has been pre-allocated at a regrid.
+ *
+ * @param s The #space.
*/
-void space_first_init_gparts(struct space *s, int verbose) {
+void space_list_useful_top_level_cells(struct space *s) {
const ticks tic = getticks();
- if (s->nr_gparts > 0)
- threadpool_map(&s->e->threadpool, space_first_init_gparts_mapper, s->gparts,
- s->nr_gparts, sizeof(struct gpart),
- threadpool_auto_chunk_size, s);
-
- if (verbose)
- message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
- clocks_getunit());
-}
-
-void space_first_init_sparts_mapper(void *restrict map_data, int count,
- void *restrict extra_data) {
-
- struct spart *restrict sp = (struct spart *)map_data;
- const struct space *restrict s = (struct space *)extra_data;
- const struct engine *e = s->e;
-
- const struct chemistry_global_data *chemistry = e->chemistry;
-
-#ifdef SWIFT_DEBUG_CHECKS
- const ptrdiff_t delta = sp - s->sparts;
-#endif
-
- const float initial_h = s->initial_spart_h;
-
- const int with_feedback = (e->policy & engine_policy_feedback);
- const int with_cosmology = (e->policy & engine_policy_cosmology);
-
- const struct cosmology *cosmo = e->cosmology;
- const struct stars_props *stars_properties = e->stars_properties;
- const float a_factor_vel = cosmo->a;
-
- /* Convert velocities to internal units */
- for (int k = 0; k < count; k++) {
-
- sp[k].v[0] *= a_factor_vel;
- sp[k].v[1] *= a_factor_vel;
- sp[k].v[2] *= a_factor_vel;
-
- /* Imposed smoothing length from parameter file */
- if (initial_h != -1.f) {
- sp[k].h = initial_h;
- }
-#ifdef HYDRO_DIMENSION_2D
- sp[k].x[2] = 0.f;
- sp[k].v[2] = 0.f;
-#endif
-
-#ifdef HYDRO_DIMENSION_1D
- sp[k].x[1] = sp[k].x[2] = 0.f;
- sp[k].v[1] = sp[k].v[2] = 0.f;
-#endif
- }
-
- /* Check that the smoothing lengths are non-zero */
- for (int k = 0; k < count; k++) {
- if (with_feedback && sp[k].h <= 0.)
- error("Invalid value of smoothing length for spart %lld h=%e", sp[k].id,
- sp[k].h);
- }
-
- /* Initialise the rest */
- for (int k = 0; k < count; k++) {
-
- stars_first_init_spart(&sp[k], stars_properties, with_cosmology, cosmo->a,
- e->time);
-
-#ifdef WITH_LOGGER
- logger_part_data_init(&sp[k].logger_data);
-#endif
+ s->nr_local_cells_with_tasks = 0;
+ s->nr_cells_with_particles = 0;
- /* Also initialise the chemistry */
- chemistry_first_init_spart(chemistry, &sp[k]);
+ for (int i = 0; i < s->nr_cells; ++i) {
+ struct cell *c = &s->cells_top[i];
- /* And radiative transfer data */
- rt_first_init_spart(&sp[k]);
+ if (cell_has_tasks(c)) {
+ s->local_cells_with_tasks_top[s->nr_local_cells_with_tasks] = i;
+ s->nr_local_cells_with_tasks++;
+ }
-#ifdef SWIFT_DEBUG_CHECKS
- if (sp[k].gpart && sp[k].gpart->id_or_neg_offset != -(k + delta))
- error("Invalid gpart -> spart link");
+ const int has_particles =
+ (c->hydro.count > 0) || (c->grav.count > 0) || (c->stars.count > 0) ||
+ (c->black_holes.count > 0) || (c->sinks.count > 0) ||
+ (c->grav.multipole != NULL && c->grav.multipole->m_pole.M_000 > 0.f);
- /* Initialise the time-integration check variables */
- sp[k].ti_drift = 0;
- sp[k].ti_kick = 0;
-#endif
+ if (has_particles) {
+ s->cells_with_particles_top[s->nr_cells_with_particles] = i;
+ s->nr_cells_with_particles++;
+ }
+ }
+ if (s->e->verbose) {
+ message("Have %d local top-level cells with tasks (total=%d)",
+ s->nr_local_cells_with_tasks, s->nr_cells);
+ message("Have %d top-level cells with particles (total=%d)",
+ s->nr_cells_with_particles, s->nr_cells);
}
-}
-
-/**
- * @brief Initialises all the s-particles by setting them into a valid state
- *
- * Calls stars_first_init_spart() on all the particles
- */
-void space_first_init_sparts(struct space *s, int verbose) {
- const ticks tic = getticks();
- if (s->nr_sparts > 0)
- threadpool_map(&s->e->threadpool, space_first_init_sparts_mapper, s->sparts,
- s->nr_sparts, sizeof(struct spart),
- threadpool_auto_chunk_size, s);
- if (verbose)
+ if (s->e->verbose)
message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
clocks_getunit());
}
-void space_first_init_bparts_mapper(void *restrict map_data, int count,
- void *restrict extra_data) {
-
- struct bpart *restrict bp = (struct bpart *)map_data;
- const struct space *restrict s = (struct space *)extra_data;
- const struct engine *e = s->e;
- const struct black_holes_props *props = e->black_holes_properties;
-
-#ifdef SWIFT_DEBUG_CHECKS
- const ptrdiff_t delta = bp - s->bparts;
-#endif
-
- const float initial_h = s->initial_bpart_h;
-
- const struct cosmology *cosmo = e->cosmology;
- const float a_factor_vel = cosmo->a;
+void space_synchronize_part_positions_mapper(void *map_data, int nr_parts,
+ void *extra_data) {
+ /* Unpack the data */
+ const struct part *parts = (struct part *)map_data;
+ struct space *s = (struct space *)extra_data;
+ const ptrdiff_t offset = parts - s->parts;
+ const struct xpart *xparts = s->xparts + offset;
- /* Convert velocities to internal units */
- for (int k = 0; k < count; k++) {
+ for (int k = 0; k < nr_parts; k++) {
- bp[k].v[0] *= a_factor_vel;
- bp[k].v[1] *= a_factor_vel;
- bp[k].v[2] *= a_factor_vel;
+ /* Get the particle */
+ const struct part *p = &parts[k];
+ const struct xpart *xp = &xparts[k];
- /* Imposed smoothing length from parameter file */
- if (initial_h != -1.f) {
- bp[k].h = initial_h;
- }
+ /* Skip unimportant particles */
+ if (p->time_bin == time_bin_not_created ||
+ p->time_bin == time_bin_inhibited)
+ continue;
-#ifdef HYDRO_DIMENSION_2D
- bp[k].x[2] = 0.f;
- bp[k].v[2] = 0.f;
-#endif
+ /* Get its gravity friend */
+ struct gpart *gp = p->gpart;
-#ifdef HYDRO_DIMENSION_1D
- bp[k].x[1] = bp[k].x[2] = 0.f;
- bp[k].v[1] = bp[k].v[2] = 0.f;
+#ifdef SWIFT_DEBUG_CHECKS
+ if (gp == NULL) error("Unlinked particle!");
#endif
- }
- /* Check that the smoothing lengths are non-zero */
- for (int k = 0; k < count; k++) {
- if (bp[k].h <= 0.)
- error("Invalid value of smoothing length for bpart %lld h=%e", bp[k].id,
- bp[k].h);
- }
-
- /* Initialise the rest */
- for (int k = 0; k < count; k++) {
-
- black_holes_first_init_bpart(&bp[k], props);
-
- /* And the black hole merger markers */
- black_holes_mark_bpart_as_not_swallowed(&bp[k].merger_data);
+ /* Synchronize positions, velocities and masses */
+ gp->x[0] = p->x[0];
+ gp->x[1] = p->x[1];
+ gp->x[2] = p->x[2];
-#ifdef SWIFT_DEBUG_CHECKS
- if (bp[k].gpart && bp[k].gpart->id_or_neg_offset != -(k + delta))
- error("Invalid gpart -> bpart link");
+ gp->v_full[0] = xp->v_full[0];
+ gp->v_full[1] = xp->v_full[1];
+ gp->v_full[2] = xp->v_full[2];
- /* Initialise the time-integration check variables */
- bp[k].ti_drift = 0;
- bp[k].ti_kick = 0;
-#endif
+ gp->mass = hydro_get_mass(p);
}
}
-/**
- * @brief Initialises all the b-particles by setting them into a valid state
- *
- * Calls stars_first_init_bpart() on all the particles
- */
-void space_first_init_bparts(struct space *s, int verbose) {
- const ticks tic = getticks();
- if (s->nr_bparts > 0)
- threadpool_map(&s->e->threadpool, space_first_init_bparts_mapper, s->bparts,
- s->nr_bparts, sizeof(struct bpart),
- threadpool_auto_chunk_size, s);
+void space_synchronize_spart_positions_mapper(void *map_data, int nr_sparts,
+ void *extra_data) {
+ /* Unpack the data */
+ const struct spart *sparts = (struct spart *)map_data;
- if (verbose)
- message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
- clocks_getunit());
-}
+ for (int k = 0; k < nr_sparts; k++) {
+
+ /* Get the particle */
+ const struct spart *sp = &sparts[k];
-void space_first_init_sinks_mapper(void *restrict map_data, int count,
- void *restrict extra_data) {
+ /* Skip unimportant particles */
+ if (sp->time_bin == time_bin_not_created ||
+ sp->time_bin == time_bin_inhibited)
+ continue;
- struct sink *restrict sink = (struct sink *)map_data;
- const struct space *restrict s = (struct space *)extra_data;
- const struct engine *e = s->e;
- const struct sink_props *props = e->sink_properties;
+ /* Get its gravity friend */
+ struct gpart *gp = sp->gpart;
#ifdef SWIFT_DEBUG_CHECKS
- const ptrdiff_t delta = sink - s->sinks;
+ if (gp == NULL) error("Unlinked particle!");
#endif
- const struct cosmology *cosmo = e->cosmology;
- const float a_factor_vel = cosmo->a;
+ /* Synchronize positions, velocities and masses */
+ gp->x[0] = sp->x[0];
+ gp->x[1] = sp->x[1];
+ gp->x[2] = sp->x[2];
+
+ gp->v_full[0] = sp->v[0];
+ gp->v_full[1] = sp->v[1];
+ gp->v_full[2] = sp->v[2];
- /* Convert velocities to internal units */
- for (int k = 0; k < count; k++) {
+ gp->mass = sp->mass;
+ }
+}
- sink[k].v[0] *= a_factor_vel;
- sink[k].v[1] *= a_factor_vel;
- sink[k].v[2] *= a_factor_vel;
+void space_synchronize_bpart_positions_mapper(void *map_data, int nr_bparts,
+ void *extra_data) {
+ /* Unpack the data */
+ const struct bpart *bparts = (struct bpart *)map_data;
-#ifdef HYDRO_DIMENSION_2D
- sink[k].x[2] = 0.f;
- sink[k].v[2] = 0.f;
-#endif
+ for (int k = 0; k < nr_bparts; k++) {
-#ifdef HYDRO_DIMENSION_1D
- sink[k].x[1] = sink[k].x[2] = 0.f;
- sink[k].v[1] = sink[k].v[2] = 0.f;
-#endif
- }
+ /* Get the particle */
+ const struct bpart *bp = &bparts[k];
- /* Initialise the rest */
- for (int k = 0; k < count; k++) {
+ /* Skip unimportant particles */
+ if (bp->time_bin == time_bin_not_created ||
+ bp->time_bin == time_bin_inhibited)
+ continue;
- sink_first_init_sink(&sink[k], props);
+ /* Get its gravity friend */
+ struct gpart *gp = bp->gpart;
#ifdef SWIFT_DEBUG_CHECKS
- if (sink[k].gpart && sink[k].gpart->id_or_neg_offset != -(k + delta))
- error("Invalid gpart -> sink link");
-
- /* Initialise the time-integration check variables */
- sink[k].ti_drift = 0;
- sink[k].ti_kick = 0;
+ if (gp == NULL) error("Unlinked particle!");
#endif
- }
-}
-
-/**
- * @brief Initialises all the sink-particles by setting them into a valid state
- *
- * Calls stars_first_init_sink() on all the particles
- */
-void space_first_init_sinks(struct space *s, int verbose) {
- const ticks tic = getticks();
- if (s->nr_sinks > 0)
- threadpool_map(&s->e->threadpool, space_first_init_sinks_mapper, s->sinks,
- s->nr_sinks, sizeof(struct sink), threadpool_auto_chunk_size,
- s);
- if (verbose)
- message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
- clocks_getunit());
-}
+ /* Synchronize positions, velocities and masses */
+ gp->x[0] = bp->x[0];
+ gp->x[1] = bp->x[1];
+ gp->x[2] = bp->x[2];
-void space_init_parts_mapper(void *restrict map_data, int count,
- void *restrict extra_data) {
+ gp->v_full[0] = bp->v[0];
+ gp->v_full[1] = bp->v[1];
+ gp->v_full[2] = bp->v[2];
- struct part *restrict parts = (struct part *)map_data;
- const struct engine *restrict e = (struct engine *)extra_data;
- const struct hydro_space *restrict hs = &e->s->hs;
- const int with_cosmology = (e->policy & engine_policy_cosmology);
-
- size_t ind = parts - e->s->parts;
- struct xpart *restrict xparts = e->s->xparts + ind;
-
- for (int k = 0; k < count; k++) {
- hydro_init_part(&parts[k], hs);
- black_holes_init_potential(&parts[k].black_holes_data);
- chemistry_init_part(&parts[k], e->chemistry);
- pressure_floor_init_part(&parts[k], &xparts[k]);
- rt_init_part(&parts[k]);
- star_formation_init_part(&parts[k], e->star_formation);
- tracers_after_init(&parts[k], &xparts[k], e->internal_units,
- e->physical_constants, with_cosmology, e->cosmology,
- e->hydro_properties, e->cooling_func, e->time);
+ gp->mass = bp->mass;
}
}
-/**
- * @brief Calls the #part initialisation function on all particles in the space.
- *
- * @param s The #space.
- * @param verbose Are we talkative?
- */
-void space_init_parts(struct space *s, int verbose) {
-
- const ticks tic = getticks();
+void space_synchronize_sink_positions_mapper(void *map_data, int nr_sinks,
+ void *extra_data) {
+ /* Unpack the data */
+ const struct sink *sinks = (struct sink *)map_data;
- if (s->nr_parts > 0)
- threadpool_map(&s->e->threadpool, space_init_parts_mapper, s->parts,
- s->nr_parts, sizeof(struct part), threadpool_auto_chunk_size,
- s->e);
- if (verbose)
- message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
- clocks_getunit());
-}
+ for (int k = 0; k < nr_sinks; k++) {
-void space_init_gparts_mapper(void *restrict map_data, int count,
- void *restrict extra_data) {
+ /* Get the particle */
+ const struct sink *sink = &sinks[k];
- struct gpart *gparts = (struct gpart *)map_data;
- for (int k = 0; k < count; k++) gravity_init_gpart(&gparts[k]);
-}
+ /* Skip unimportant particles */
+ if (sink->time_bin == time_bin_not_created ||
+ sink->time_bin == time_bin_inhibited)
+ continue;
-/**
- * @brief Calls the #gpart initialisation function on all particles in the
- * space.
- *
- * @param s The #space.
- * @param verbose Are we talkative?
- */
-void space_init_gparts(struct space *s, int verbose) {
+ /* Get its gravity friend */
+ struct gpart *gp = sink->gpart;
- const ticks tic = getticks();
+#ifdef SWIFT_DEBUG_CHECKS
+ if (gp == NULL) error("Unlinked particle!");
+#endif
- if (s->nr_gparts > 0)
- threadpool_map(&s->e->threadpool, space_init_gparts_mapper, s->gparts,
- s->nr_gparts, sizeof(struct gpart),
- threadpool_auto_chunk_size, /*extra_data=*/NULL);
- if (verbose)
- message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
- clocks_getunit());
-}
+ /* Synchronize positions, velocities and masses */
+ gp->x[0] = sink->x[0];
+ gp->x[1] = sink->x[1];
+ gp->x[2] = sink->x[2];
-void space_init_sparts_mapper(void *restrict map_data, int scount,
- void *restrict extra_data) {
+ gp->v_full[0] = sink->v[0];
+ gp->v_full[1] = sink->v[1];
+ gp->v_full[2] = sink->v[2];
- struct spart *restrict sparts = (struct spart *)map_data;
- for (int k = 0; k < scount; k++) {
- stars_init_spart(&sparts[k]);
- rt_init_spart(&sparts[k]);
+ gp->mass = sink->mass;
}
}
/**
- * @brief Calls the #spart initialisation function on all particles in the
- * space.
+ * @brief Make sure the baryon particles are at the same position and
+ * have the same velocity and mass as their #gpart friends.
*
- * @param s The #space.
- * @param verbose Are we talkative?
- */
-void space_init_sparts(struct space *s, int verbose) {
-
- const ticks tic = getticks();
-
- if (s->nr_sparts > 0)
- threadpool_map(&s->e->threadpool, space_init_sparts_mapper, s->sparts,
- s->nr_sparts, sizeof(struct spart),
- threadpool_auto_chunk_size, /*extra_data=*/NULL);
- if (verbose)
- message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
- clocks_getunit());
-}
-
-void space_init_bparts_mapper(void *restrict map_data, int bcount,
- void *restrict extra_data) {
-
- struct bpart *restrict bparts = (struct bpart *)map_data;
- for (int k = 0; k < bcount; k++) black_holes_init_bpart(&bparts[k]);
-}
-
-/**
- * @brief Calls the #bpart initialisation function on all particles in the
- * space.
+ * We copy the baryon particle properties to the #gpart type-by-type.
*
* @param s The #space.
- * @param verbose Are we talkative?
*/
-void space_init_bparts(struct space *s, int verbose) {
+void space_synchronize_particle_positions(struct space *s) {
const ticks tic = getticks();
- if (s->nr_bparts > 0)
- threadpool_map(&s->e->threadpool, space_init_bparts_mapper, s->bparts,
- s->nr_bparts, sizeof(struct bpart),
- threadpool_auto_chunk_size, /*extra_data=*/NULL);
- if (verbose)
- message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
- clocks_getunit());
-}
-
-void space_init_sinks_mapper(void *restrict map_data, int sink_count,
- void *restrict extra_data) {
+ if (s->nr_gparts > 0 && s->nr_parts > 0)
+ threadpool_map(&s->e->threadpool, space_synchronize_part_positions_mapper,
+ s->parts, s->nr_parts, sizeof(struct part),
+ threadpool_auto_chunk_size, (void *)s);
- struct sink *restrict sinks = (struct sink *)map_data;
- for (int k = 0; k < sink_count; k++) sink_init_sink(&sinks[k]);
-}
+ if (s->nr_gparts > 0 && s->nr_sparts > 0)
+ threadpool_map(&s->e->threadpool, space_synchronize_spart_positions_mapper,
+ s->sparts, s->nr_sparts, sizeof(struct spart),
+ threadpool_auto_chunk_size, /*extra_data=*/NULL);
-/**
- * @brief Calls the #sink initialisation function on all particles in the
- * space.
- *
- * @param s The #space.
- * @param verbose Are we talkative?
- */
-void space_init_sinks(struct space *s, int verbose) {
+ if (s->nr_gparts > 0 && s->nr_bparts > 0)
+ threadpool_map(&s->e->threadpool, space_synchronize_bpart_positions_mapper,
+ s->bparts, s->nr_bparts, sizeof(struct bpart),
+ threadpool_auto_chunk_size, /*extra_data=*/NULL);
- const ticks tic = getticks();
+ if (s->nr_gparts > 0 && s->nr_sinks > 0)
+ threadpool_map(&s->e->threadpool, space_synchronize_sink_positions_mapper,
+ s->sinks, s->nr_sinks, sizeof(struct sink),
+ threadpool_auto_chunk_size, /*extra_data=*/NULL);
- if (s->nr_sinks > 0)
- threadpool_map(&s->e->threadpool, space_init_sinks_mapper, s->sinks,
- s->nr_sinks, sizeof(struct sink), threadpool_auto_chunk_size,
- /*extra_data=*/NULL);
- if (verbose)
+ if (s->e->verbose)
message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
clocks_getunit());
}
diff --git a/src/space.h b/src/space.h
index a4db4f92f2df8173afcb612018ee41a868a8ba38..5af6783dfe2c99d2f2da01906ec9c646e3f9a380 100644
--- a/src/space.h
+++ b/src/space.h
@@ -359,9 +359,10 @@ void space_recycle_list(struct space *s, struct cell *cell_list_begin,
struct cell *cell_list_end,
struct gravity_tensors *multipole_list_begin,
struct gravity_tensors *multipole_list_end);
+void space_regrid(struct space *s, int verbose);
+void space_allocate_extras(struct space *s, int verbose);
void space_split(struct space *s, int verbose);
void space_reorder_extras(struct space *s, int verbose);
-void space_split_mapper(void *map_data, int num_elements, void *extra_data);
void space_list_useful_top_level_cells(struct space *s);
void space_parts_get_cell_index(struct space *s, int *ind, int *cell_counts,
size_t *count_inhibited_parts,
diff --git a/src/space_cell_index.c b/src/space_cell_index.c
new file mode 100644
index 0000000000000000000000000000000000000000..f99c382ac70f76f1b0386a124c00a90cf7620161
--- /dev/null
+++ b/src/space_cell_index.c
@@ -0,0 +1,937 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* This object's header. */
+#include "space.h"
+
+/* Local headers. */
+#include "cell.h"
+#include "engine.h"
+#include "error.h"
+#include "hydro.h"
+#include "threadpool.h"
+
+/* Some standard headers. */
+#include <float.h>
+
+/**
+ * @brief Information required to compute the particle cell indices.
+ */
+struct index_data {
+ struct space *s;
+ int *ind;
+ int *cell_counts;
+ size_t count_inhibited_part;
+ size_t count_inhibited_gpart;
+ size_t count_inhibited_spart;
+ size_t count_inhibited_bpart;
+ size_t count_inhibited_sink;
+ size_t count_extra_part;
+ size_t count_extra_gpart;
+ size_t count_extra_spart;
+ size_t count_extra_bpart;
+ size_t count_extra_sink;
+};
+
+/**
+ * @brief #threadpool mapper function to compute the particle cell indices.
+ *
+ * @param map_data Pointer towards the particles.
+ * @param nr_parts The number of particles to treat.
+ * @param extra_data Pointers to the space and index list
+ */
+void space_parts_get_cell_index_mapper(void *map_data, int nr_parts,
+ void *extra_data) {
+
+ /* Unpack the data */
+ struct part *restrict parts = (struct part *)map_data;
+ struct index_data *data = (struct index_data *)extra_data;
+ struct space *s = data->s;
+ int *const ind = data->ind + (ptrdiff_t)(parts - s->parts);
+
+ /* Get some constants */
+ const double dim_x = s->dim[0];
+ const double dim_y = s->dim[1];
+ const double dim_z = s->dim[2];
+ const int cdim[3] = {s->cdim[0], s->cdim[1], s->cdim[2]};
+ const double ih_x = s->iwidth[0];
+ const double ih_y = s->iwidth[1];
+ const double ih_z = s->iwidth[2];
+
+ /* Init the local count buffer. */
+ int *cell_counts = (int *)calloc(sizeof(int), s->nr_cells);
+ if (cell_counts == NULL)
+ error("Failed to allocate temporary cell count buffer.");
+
+ /* Init the local collectors */
+ float min_mass = FLT_MAX;
+ float sum_vel_norm = 0.f;
+ size_t count_inhibited_part = 0;
+ size_t count_extra_part = 0;
+
+ /* Loop over the parts. */
+ for (int k = 0; k < nr_parts; k++) {
+
+ /* Get the particle */
+ struct part *restrict p = &parts[k];
+
+ double old_pos_x = p->x[0];
+ double old_pos_y = p->x[1];
+ double old_pos_z = p->x[2];
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (!s->periodic && p->time_bin != time_bin_inhibited) {
+ if (old_pos_x < 0. || old_pos_x > dim_x)
+ error("Particle outside of volume along X.");
+ if (old_pos_y < 0. || old_pos_y > dim_y)
+ error("Particle outside of volume along Y.");
+ if (old_pos_z < 0. || old_pos_z > dim_z)
+ error("Particle outside of volume along Z.");
+ }
+#endif
+
+ /* Put it back into the simulation volume */
+ double pos_x = box_wrap(old_pos_x, 0.0, dim_x);
+ double pos_y = box_wrap(old_pos_y, 0.0, dim_y);
+ double pos_z = box_wrap(old_pos_z, 0.0, dim_z);
+
+ /* Treat the case where a particle was wrapped back exactly onto
+ * the edge because of rounding issues (more accuracy around 0
+ * than around dim) */
+ if (pos_x == dim_x) pos_x = 0.0;
+ if (pos_y == dim_y) pos_y = 0.0;
+ if (pos_z == dim_z) pos_z = 0.0;
+
+ /* Get its cell index */
+ const int index =
+ cell_getid(cdim, pos_x * ih_x, pos_y * ih_y, pos_z * ih_z);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (index < 0 || index >= cdim[0] * cdim[1] * cdim[2])
+ error("Invalid index=%d cdim=[%d %d %d] p->x=[%e %e %e]", index, cdim[0],
+ cdim[1], cdim[2], pos_x, pos_y, pos_z);
+
+ if (pos_x >= dim_x || pos_y >= dim_y || pos_z >= dim_z || pos_x < 0. ||
+ pos_y < 0. || pos_z < 0.)
+ error("Particle outside of simulation box. p->x=[%e %e %e]", pos_x, pos_y,
+ pos_z);
+#endif
+
+ if (p->time_bin == time_bin_inhibited) {
+ /* Is this particle to be removed? */
+ ind[k] = -1;
+ ++count_inhibited_part;
+ } else if (p->time_bin == time_bin_not_created) {
+ /* Is this a place-holder for on-the-fly creation? */
+ ind[k] = index;
+ cell_counts[index]++;
+ ++count_extra_part;
+
+ } else {
+ /* Normal case: list its top-level cell index */
+ ind[k] = index;
+ cell_counts[index]++;
+
+ /* Compute minimal mass */
+ min_mass = min(min_mass, hydro_get_mass(p));
+
+ /* Compute sum of velocity norm */
+ sum_vel_norm += p->v[0] * p->v[0] + p->v[1] * p->v[1] + p->v[2] * p->v[2];
+
+ /* Update the position */
+ p->x[0] = pos_x;
+ p->x[1] = pos_y;
+ p->x[2] = pos_z;
+ }
+ }
+
+ /* Write the counts back to the global array. */
+ for (int k = 0; k < s->nr_cells; k++)
+ if (cell_counts[k]) atomic_add(&data->cell_counts[k], cell_counts[k]);
+ free(cell_counts);
+
+ /* Write the count of inhibited and extra parts */
+ if (count_inhibited_part)
+ atomic_add(&data->count_inhibited_part, count_inhibited_part);
+ if (count_extra_part) atomic_add(&data->count_extra_part, count_extra_part);
+
+ /* Write back the minimal part mass and velocity sum */
+ atomic_min_f(&s->min_part_mass, min_mass);
+ atomic_add_f(&s->sum_part_vel_norm, sum_vel_norm);
+}
+
+/**
+ * @brief #threadpool mapper function to compute the g-particle cell indices.
+ *
+ * @param map_data Pointer towards the g-particles.
+ * @param nr_gparts The number of g-particles to treat.
+ * @param extra_data Pointers to the space and index list
+ */
+void space_gparts_get_cell_index_mapper(void *map_data, int nr_gparts,
+ void *extra_data) {
+
+ /* Unpack the data */
+ struct gpart *restrict gparts = (struct gpart *)map_data;
+ struct index_data *data = (struct index_data *)extra_data;
+ struct space *s = data->s;
+ int *const ind = data->ind + (ptrdiff_t)(gparts - s->gparts);
+
+ /* Get some constants */
+ const double dim_x = s->dim[0];
+ const double dim_y = s->dim[1];
+ const double dim_z = s->dim[2];
+ const int cdim[3] = {s->cdim[0], s->cdim[1], s->cdim[2]};
+ const double ih_x = s->iwidth[0];
+ const double ih_y = s->iwidth[1];
+ const double ih_z = s->iwidth[2];
+
+ /* Init the local count buffer. */
+ int *cell_counts = (int *)calloc(sizeof(int), s->nr_cells);
+ if (cell_counts == NULL)
+ error("Failed to allocate temporary cell count buffer.");
+
+ /* Init the local collectors */
+ float min_mass = FLT_MAX;
+ float sum_vel_norm = 0.f;
+ size_t count_inhibited_gpart = 0;
+ size_t count_extra_gpart = 0;
+
+ for (int k = 0; k < nr_gparts; k++) {
+
+ /* Get the particle */
+ struct gpart *restrict gp = &gparts[k];
+
+ double old_pos_x = gp->x[0];
+ double old_pos_y = gp->x[1];
+ double old_pos_z = gp->x[2];
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (!s->periodic && gp->time_bin != time_bin_inhibited) {
+ if (old_pos_x < 0. || old_pos_x > dim_x)
+ error("Particle outside of volume along X.");
+ if (old_pos_y < 0. || old_pos_y > dim_y)
+ error("Particle outside of volume along Y.");
+ if (old_pos_z < 0. || old_pos_z > dim_z)
+ error("Particle outside of volume along Z.");
+ }
+#endif
+
+ /* Put it back into the simulation volume */
+ double pos_x = box_wrap(old_pos_x, 0.0, dim_x);
+ double pos_y = box_wrap(old_pos_y, 0.0, dim_y);
+ double pos_z = box_wrap(old_pos_z, 0.0, dim_z);
+
+ /* Treat the case where a particle was wrapped back exactly onto
+ * the edge because of rounding issues (more accuracy around 0
+ * than around dim) */
+ if (pos_x == dim_x) pos_x = 0.0;
+ if (pos_y == dim_y) pos_y = 0.0;
+ if (pos_z == dim_z) pos_z = 0.0;
+
+ /* Get its cell index */
+ const int index =
+ cell_getid(cdim, pos_x * ih_x, pos_y * ih_y, pos_z * ih_z);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (index < 0 || index >= cdim[0] * cdim[1] * cdim[2])
+ error("Invalid index=%d cdim=[%d %d %d] p->x=[%e %e %e]", index, cdim[0],
+ cdim[1], cdim[2], pos_x, pos_y, pos_z);
+
+ if (pos_x >= dim_x || pos_y >= dim_y || pos_z >= dim_z || pos_x < 0. ||
+ pos_y < 0. || pos_z < 0.)
+ error("Particle outside of simulation box. p->x=[%e %e %e]", pos_x, pos_y,
+ pos_z);
+#endif
+
+ if (gp->time_bin == time_bin_inhibited) {
+ /* Is this particle to be removed? */
+ ind[k] = -1;
+ ++count_inhibited_gpart;
+ } else if (gp->time_bin == time_bin_not_created) {
+ /* Is this a place-holder for on-the-fly creation? */
+ ind[k] = index;
+ cell_counts[index]++;
+ ++count_extra_gpart;
+
+ } else {
+ /* List its top-level cell index */
+ ind[k] = index;
+ cell_counts[index]++;
+
+ if (gp->type == swift_type_dark_matter) {
+
+ /* Compute minimal mass */
+ min_mass = min(min_mass, gp->mass);
+
+ /* Compute sum of velocity norm */
+ sum_vel_norm += gp->v_full[0] * gp->v_full[0] +
+ gp->v_full[1] * gp->v_full[1] +
+ gp->v_full[2] * gp->v_full[2];
+ }
+
+ /* Update the position */
+ gp->x[0] = pos_x;
+ gp->x[1] = pos_y;
+ gp->x[2] = pos_z;
+ }
+ }
+
+ /* Write the counts back to the global array. */
+ for (int k = 0; k < s->nr_cells; k++)
+ if (cell_counts[k]) atomic_add(&data->cell_counts[k], cell_counts[k]);
+ free(cell_counts);
+
+ /* Write the count of inhibited and extra gparts */
+ if (count_inhibited_gpart)
+ atomic_add(&data->count_inhibited_gpart, count_inhibited_gpart);
+ if (count_extra_gpart)
+ atomic_add(&data->count_extra_gpart, count_extra_gpart);
+
+ /* Write back the minimal part mass and velocity sum */
+ atomic_min_f(&s->min_gpart_mass, min_mass);
+ atomic_add_f(&s->sum_gpart_vel_norm, sum_vel_norm);
+}
+
+/**
+ * @brief #threadpool mapper function to compute the s-particle cell indices.
+ *
+ * @param map_data Pointer towards the s-particles.
+ * @param nr_sparts The number of s-particles to treat.
+ * @param extra_data Pointers to the space and index list
+ */
+void space_sparts_get_cell_index_mapper(void *map_data, int nr_sparts,
+ void *extra_data) {
+
+ /* Unpack the data */
+ struct spart *restrict sparts = (struct spart *)map_data;
+ struct index_data *data = (struct index_data *)extra_data;
+ struct space *s = data->s;
+ int *const ind = data->ind + (ptrdiff_t)(sparts - s->sparts);
+
+ /* Get some constants */
+ const double dim_x = s->dim[0];
+ const double dim_y = s->dim[1];
+ const double dim_z = s->dim[2];
+ const int cdim[3] = {s->cdim[0], s->cdim[1], s->cdim[2]};
+ const double ih_x = s->iwidth[0];
+ const double ih_y = s->iwidth[1];
+ const double ih_z = s->iwidth[2];
+
+ /* Init the local count buffer. */
+ int *cell_counts = (int *)calloc(sizeof(int), s->nr_cells);
+ if (cell_counts == NULL)
+ error("Failed to allocate temporary cell count buffer.");
+
+ /* Init the local collectors */
+ float min_mass = FLT_MAX;
+ float sum_vel_norm = 0.f;
+ size_t count_inhibited_spart = 0;
+ size_t count_extra_spart = 0;
+
+ for (int k = 0; k < nr_sparts; k++) {
+
+ /* Get the particle */
+ struct spart *restrict sp = &sparts[k];
+
+ double old_pos_x = sp->x[0];
+ double old_pos_y = sp->x[1];
+ double old_pos_z = sp->x[2];
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (!s->periodic && sp->time_bin != time_bin_inhibited) {
+ if (old_pos_x < 0. || old_pos_x > dim_x)
+ error("Particle outside of volume along X.");
+ if (old_pos_y < 0. || old_pos_y > dim_y)
+ error("Particle outside of volume along Y.");
+ if (old_pos_z < 0. || old_pos_z > dim_z)
+ error("Particle outside of volume along Z.");
+ }
+#endif
+
+ /* Put it back into the simulation volume */
+ double pos_x = box_wrap(old_pos_x, 0.0, dim_x);
+ double pos_y = box_wrap(old_pos_y, 0.0, dim_y);
+ double pos_z = box_wrap(old_pos_z, 0.0, dim_z);
+
+ /* Treat the case where a particle was wrapped back exactly onto
+ * the edge because of rounding issues (more accuracy around 0
+ * than around dim) */
+ if (pos_x == dim_x) pos_x = 0.0;
+ if (pos_y == dim_y) pos_y = 0.0;
+ if (pos_z == dim_z) pos_z = 0.0;
+
+ /* Get its cell index */
+ const int index =
+ cell_getid(cdim, pos_x * ih_x, pos_y * ih_y, pos_z * ih_z);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (index < 0 || index >= cdim[0] * cdim[1] * cdim[2])
+ error("Invalid index=%d cdim=[%d %d %d] p->x=[%e %e %e]", index, cdim[0],
+ cdim[1], cdim[2], pos_x, pos_y, pos_z);
+
+ if (pos_x >= dim_x || pos_y >= dim_y || pos_z >= dim_z || pos_x < 0. ||
+ pos_y < 0. || pos_z < 0.)
+ error("Particle outside of simulation box. p->x=[%e %e %e]", pos_x, pos_y,
+ pos_z);
+#endif
+
+ /* Is this particle to be removed? */
+ if (sp->time_bin == time_bin_inhibited) {
+ ind[k] = -1;
+ ++count_inhibited_spart;
+ } else if (sp->time_bin == time_bin_not_created) {
+ /* Is this a place-holder for on-the-fly creation? */
+ ind[k] = index;
+ cell_counts[index]++;
+ ++count_extra_spart;
+
+ } else {
+ /* List its top-level cell index */
+ ind[k] = index;
+ cell_counts[index]++;
+
+ /* Compute minimal mass */
+ min_mass = min(min_mass, sp->mass);
+
+ /* Compute sum of velocity norm */
+ sum_vel_norm +=
+ sp->v[0] * sp->v[0] + sp->v[1] * sp->v[1] + sp->v[2] * sp->v[2];
+
+ /* Update the position */
+ sp->x[0] = pos_x;
+ sp->x[1] = pos_y;
+ sp->x[2] = pos_z;
+ }
+ }
+
+ /* Write the counts back to the global array. */
+ for (int k = 0; k < s->nr_cells; k++)
+ if (cell_counts[k]) atomic_add(&data->cell_counts[k], cell_counts[k]);
+ free(cell_counts);
+
+ /* Write the count of inhibited and extra sparts */
+ if (count_inhibited_spart)
+ atomic_add(&data->count_inhibited_spart, count_inhibited_spart);
+ if (count_extra_spart)
+ atomic_add(&data->count_extra_spart, count_extra_spart);
+
+ /* Write back the minimal part mass and velocity sum */
+ atomic_min_f(&s->min_spart_mass, min_mass);
+ atomic_add_f(&s->sum_spart_vel_norm, sum_vel_norm);
+}
+
+/**
+ * @brief #threadpool mapper function to compute the b-particle cell indices.
+ *
+ * @param map_data Pointer towards the b-particles.
+ * @param nr_bparts The number of b-particles to treat.
+ * @param extra_data Pointers to the space and index list
+ */
+void space_bparts_get_cell_index_mapper(void *map_data, int nr_bparts,
+ void *extra_data) {
+
+ /* Unpack the data */
+ struct bpart *restrict bparts = (struct bpart *)map_data;
+ struct index_data *data = (struct index_data *)extra_data;
+ struct space *s = data->s;
+ int *const ind = data->ind + (ptrdiff_t)(bparts - s->bparts);
+
+ /* Get some constants */
+ const double dim_x = s->dim[0];
+ const double dim_y = s->dim[1];
+ const double dim_z = s->dim[2];
+ const int cdim[3] = {s->cdim[0], s->cdim[1], s->cdim[2]};
+ const double ih_x = s->iwidth[0];
+ const double ih_y = s->iwidth[1];
+ const double ih_z = s->iwidth[2];
+
+ /* Init the local count buffer. */
+ int *cell_counts = (int *)calloc(sizeof(int), s->nr_cells);
+ if (cell_counts == NULL)
+ error("Failed to allocate temporary cell count buffer.");
+
+ /* Init the local collectors */
+ float min_mass = FLT_MAX;
+ float sum_vel_norm = 0.f;
+ size_t count_inhibited_bpart = 0;
+ size_t count_extra_bpart = 0;
+
+ for (int k = 0; k < nr_bparts; k++) {
+
+ /* Get the particle */
+ struct bpart *restrict bp = &bparts[k];
+
+ double old_pos_x = bp->x[0];
+ double old_pos_y = bp->x[1];
+ double old_pos_z = bp->x[2];
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (!s->periodic && bp->time_bin != time_bin_inhibited) {
+ if (old_pos_x < 0. || old_pos_x > dim_x)
+ error("Particle outside of volume along X.");
+ if (old_pos_y < 0. || old_pos_y > dim_y)
+ error("Particle outside of volume along Y.");
+ if (old_pos_z < 0. || old_pos_z > dim_z)
+ error("Particle outside of volume along Z.");
+ }
+#endif
+
+ /* Put it back into the simulation volume */
+ double pos_x = box_wrap(old_pos_x, 0.0, dim_x);
+ double pos_y = box_wrap(old_pos_y, 0.0, dim_y);
+ double pos_z = box_wrap(old_pos_z, 0.0, dim_z);
+
+ /* Treat the case where a particle was wrapped back exactly onto
+ * the edge because of rounding issues (more accuracy around 0
+ * than around dim) */
+ if (pos_x == dim_x) pos_x = 0.0;
+ if (pos_y == dim_y) pos_y = 0.0;
+ if (pos_z == dim_z) pos_z = 0.0;
+
+ /* Get its cell index */
+ const int index =
+ cell_getid(cdim, pos_x * ih_x, pos_y * ih_y, pos_z * ih_z);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (index < 0 || index >= cdim[0] * cdim[1] * cdim[2])
+ error("Invalid index=%d cdim=[%d %d %d] p->x=[%e %e %e]", index, cdim[0],
+ cdim[1], cdim[2], pos_x, pos_y, pos_z);
+
+ if (pos_x >= dim_x || pos_y >= dim_y || pos_z >= dim_z || pos_x < 0. ||
+ pos_y < 0. || pos_z < 0.)
+ error("Particle outside of simulation box. p->x=[%e %e %e]", pos_x, pos_y,
+ pos_z);
+#endif
+
+ /* Is this particle to be removed? */
+ if (bp->time_bin == time_bin_inhibited) {
+ ind[k] = -1;
+ ++count_inhibited_bpart;
+ } else if (bp->time_bin == time_bin_not_created) {
+ /* Is this a place-holder for on-the-fly creation? */
+ ind[k] = index;
+ cell_counts[index]++;
+ ++count_extra_bpart;
+
+ } else {
+ /* List its top-level cell index */
+ ind[k] = index;
+ cell_counts[index]++;
+
+ /* Compute minimal mass */
+ min_mass = min(min_mass, bp->mass);
+
+ /* Compute sum of velocity norm */
+ sum_vel_norm +=
+ bp->v[0] * bp->v[0] + bp->v[1] * bp->v[1] + bp->v[2] * bp->v[2];
+
+ /* Update the position */
+ bp->x[0] = pos_x;
+ bp->x[1] = pos_y;
+ bp->x[2] = pos_z;
+ }
+ }
+
+ /* Write the counts back to the global array. */
+ for (int k = 0; k < s->nr_cells; k++)
+ if (cell_counts[k]) atomic_add(&data->cell_counts[k], cell_counts[k]);
+ free(cell_counts);
+
+ /* Write the count of inhibited and extra bparts */
+ if (count_inhibited_bpart)
+ atomic_add(&data->count_inhibited_bpart, count_inhibited_bpart);
+ if (count_extra_bpart)
+ atomic_add(&data->count_extra_bpart, count_extra_bpart);
+
+ /* Write back the minimal part mass and velocity sum */
+ atomic_min_f(&s->min_bpart_mass, min_mass);
+ atomic_add_f(&s->sum_bpart_vel_norm, sum_vel_norm);
+}
+
+/**
+ * @brief #threadpool mapper function to compute the sink-particle cell indices.
+ *
+ * @param map_data Pointer towards the sink-particles.
+ * @param nr_sinks The number of sink-particles to treat.
+ * @param extra_data Pointers to the space and index list
+ */
+void space_sinks_get_cell_index_mapper(void *map_data, int nr_sinks,
+ void *extra_data) {
+
+ /* Unpack the data */
+ struct sink *restrict sinks = (struct sink *)map_data;
+ struct index_data *data = (struct index_data *)extra_data;
+ struct space *s = data->s;
+ int *const ind = data->ind + (ptrdiff_t)(sinks - s->sinks);
+
+ /* Get some constants */
+ const double dim_x = s->dim[0];
+ const double dim_y = s->dim[1];
+ const double dim_z = s->dim[2];
+ const int cdim[3] = {s->cdim[0], s->cdim[1], s->cdim[2]};
+ const double ih_x = s->iwidth[0];
+ const double ih_y = s->iwidth[1];
+ const double ih_z = s->iwidth[2];
+
+ /* Init the local count buffer. */
+ int *cell_counts = (int *)calloc(sizeof(int), s->nr_cells);
+ if (cell_counts == NULL)
+ error("Failed to allocate temporary cell count buffer.");
+
+ /* Init the local collectors */
+ size_t count_inhibited_sink = 0;
+ size_t count_extra_sink = 0;
+
+ for (int k = 0; k < nr_sinks; k++) {
+
+ /* Get the particle */
+ struct sink *restrict sink = &sinks[k];
+
+ double old_pos_x = sink->x[0];
+ double old_pos_y = sink->x[1];
+ double old_pos_z = sink->x[2];
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (!s->periodic && sink->time_bin != time_bin_inhibited) {
+ if (old_pos_x < 0. || old_pos_x > dim_x)
+ error("Particle outside of volume along X.");
+ if (old_pos_y < 0. || old_pos_y > dim_y)
+ error("Particle outside of volume along Y.");
+ if (old_pos_z < 0. || old_pos_z > dim_z)
+ error("Particle outside of volume along Z.");
+ }
+#endif
+
+ /* Put it back into the simulation volume */
+ double pos_x = box_wrap(old_pos_x, 0.0, dim_x);
+ double pos_y = box_wrap(old_pos_y, 0.0, dim_y);
+ double pos_z = box_wrap(old_pos_z, 0.0, dim_z);
+
+ /* Treat the case where a particle was wrapped back exactly onto
+ * the edge because of rounding issues (more accuracy around 0
+ * than around dim) */
+ if (pos_x == dim_x) pos_x = 0.0;
+ if (pos_y == dim_y) pos_y = 0.0;
+ if (pos_z == dim_z) pos_z = 0.0;
+
+ /* Get its cell index */
+ const int index =
+ cell_getid(cdim, pos_x * ih_x, pos_y * ih_y, pos_z * ih_z);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (index < 0 || index >= cdim[0] * cdim[1] * cdim[2])
+ error("Invalid index=%d cdim=[%d %d %d] p->x=[%e %e %e]", index, cdim[0],
+ cdim[1], cdim[2], pos_x, pos_y, pos_z);
+
+ if (pos_x >= dim_x || pos_y >= dim_y || pos_z >= dim_z || pos_x < 0. ||
+ pos_y < 0. || pos_z < 0.)
+ error("Particle outside of simulation box. p->x=[%e %e %e]", pos_x, pos_y,
+ pos_z);
+#endif
+
+ /* Is this particle to be removed? */
+ if (sink->time_bin == time_bin_inhibited) {
+ ind[k] = -1;
+ ++count_inhibited_sink;
+ } else if (sink->time_bin == time_bin_not_created) {
+ /* Is this a place-holder for on-the-fly creation? */
+ ind[k] = index;
+ cell_counts[index]++;
+ ++count_extra_sink;
+
+ } else {
+ /* List its top-level cell index */
+ ind[k] = index;
+ cell_counts[index]++;
+
+ /* Update the position */
+ sink->x[0] = pos_x;
+ sink->x[1] = pos_y;
+ sink->x[2] = pos_z;
+ }
+ }
+
+ /* Write the counts back to the global array. */
+ for (int k = 0; k < s->nr_cells; k++)
+ if (cell_counts[k]) atomic_add(&data->cell_counts[k], cell_counts[k]);
+ free(cell_counts);
+
+ /* Write the count of inhibited and extra sinks */
+ if (count_inhibited_sink)
+ atomic_add(&data->count_inhibited_sink, count_inhibited_sink);
+ if (count_extra_sink) atomic_add(&data->count_extra_sink, count_extra_sink);
+}
+
+/**
+ * @brief Computes the cell index of all the particles.
+ *
+ * Also computes the minimal mass of all #part.
+ *
+ * @param s The #space.
+ * @param ind The array of indices to fill.
+ * @param cell_counts The cell counters to update.
+ * @param count_inhibited_parts (return) The number of #part to remove.
+ * @param count_extra_parts (return) The number of #part for on-the-fly
+ * creation.
+ * @param verbose Are we talkative ?
+ */
+void space_parts_get_cell_index(struct space *s, int *ind, int *cell_counts,
+ size_t *count_inhibited_parts,
+ size_t *count_extra_parts, int verbose) {
+
+ const ticks tic = getticks();
+
+ /* Re-set the counters */
+ s->min_part_mass = FLT_MAX;
+ s->sum_part_vel_norm = 0.f;
+
+ /* Pack the extra information */
+ struct index_data data;
+ data.s = s;
+ data.ind = ind;
+ data.cell_counts = cell_counts;
+ data.count_inhibited_part = 0;
+ data.count_inhibited_gpart = 0;
+ data.count_inhibited_spart = 0;
+ data.count_inhibited_bpart = 0;
+ data.count_inhibited_sink = 0;
+ data.count_extra_part = 0;
+ data.count_extra_gpart = 0;
+ data.count_extra_spart = 0;
+ data.count_extra_bpart = 0;
+ data.count_extra_sink = 0;
+
+ threadpool_map(&s->e->threadpool, space_parts_get_cell_index_mapper, s->parts,
+ s->nr_parts, sizeof(struct part), threadpool_auto_chunk_size,
+ &data);
+
+ *count_inhibited_parts = data.count_inhibited_part;
+ *count_extra_parts = data.count_extra_part;
+
+ if (verbose)
+ message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+ clocks_getunit());
+}
+
+/**
+ * @brief Computes the cell index of all the g-particles.
+ *
+ * Also computes the minimal mass of all dark-matter #gpart.
+ *
+ * @param s The #space.
+ * @param gind The array of indices to fill.
+ * @param cell_counts The cell counters to update.
+ * @param count_inhibited_gparts (return) The number of #gpart to remove.
+ * @param count_extra_gparts (return) The number of #gpart for on-the-fly
+ * creation.
+ * @param verbose Are we talkative ?
+ */
+void space_gparts_get_cell_index(struct space *s, int *gind, int *cell_counts,
+ size_t *count_inhibited_gparts,
+ size_t *count_extra_gparts, int verbose) {
+
+ const ticks tic = getticks();
+
+ /* Re-set the counters */
+ s->min_gpart_mass = FLT_MAX;
+ s->sum_gpart_vel_norm = 0.f;
+
+ /* Pack the extra information */
+ struct index_data data;
+ data.s = s;
+ data.ind = gind;
+ data.cell_counts = cell_counts;
+ data.count_inhibited_part = 0;
+ data.count_inhibited_gpart = 0;
+ data.count_inhibited_spart = 0;
+ data.count_inhibited_bpart = 0;
+ data.count_inhibited_sink = 0;
+ data.count_extra_part = 0;
+ data.count_extra_gpart = 0;
+ data.count_extra_spart = 0;
+ data.count_extra_bpart = 0;
+ data.count_extra_sink = 0;
+
+ threadpool_map(&s->e->threadpool, space_gparts_get_cell_index_mapper,
+ s->gparts, s->nr_gparts, sizeof(struct gpart),
+ threadpool_auto_chunk_size, &data);
+
+ *count_inhibited_gparts = data.count_inhibited_gpart;
+ *count_extra_gparts = data.count_extra_gpart;
+
+ if (verbose)
+ message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+ clocks_getunit());
+}
+
+/**
+ * @brief Computes the cell index of all the s-particles.
+ *
+ * Also computes the minimal mass of all #spart.
+ *
+ * @param s The #space.
+ * @param sind The array of indices to fill.
+ * @param cell_counts The cell counters to update.
+ * @param count_inhibited_sparts (return) The number of #spart to remove.
+ * @param count_extra_sparts (return) The number of #spart for on-the-fly
+ * creation.
+ * @param verbose Are we talkative ?
+ */
+void space_sparts_get_cell_index(struct space *s, int *sind, int *cell_counts,
+ size_t *count_inhibited_sparts,
+ size_t *count_extra_sparts, int verbose) {
+
+ const ticks tic = getticks();
+
+ /* Re-set the counters */
+ s->min_spart_mass = FLT_MAX;
+ s->sum_spart_vel_norm = 0.f;
+
+ /* Pack the extra information */
+ struct index_data data;
+ data.s = s;
+ data.ind = sind;
+ data.cell_counts = cell_counts;
+ data.count_inhibited_part = 0;
+ data.count_inhibited_gpart = 0;
+ data.count_inhibited_spart = 0;
+ data.count_inhibited_sink = 0;
+ data.count_inhibited_bpart = 0;
+ data.count_extra_part = 0;
+ data.count_extra_gpart = 0;
+ data.count_extra_spart = 0;
+ data.count_extra_bpart = 0;
+ data.count_extra_sink = 0;
+
+ threadpool_map(&s->e->threadpool, space_sparts_get_cell_index_mapper,
+ s->sparts, s->nr_sparts, sizeof(struct spart),
+ threadpool_auto_chunk_size, &data);
+
+ *count_inhibited_sparts = data.count_inhibited_spart;
+ *count_extra_sparts = data.count_extra_spart;
+
+ if (verbose)
+ message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+ clocks_getunit());
+}
+
+/**
+ * @brief Computes the cell index of all the sink-particles.
+ *
+ * @param s The #space.
+ * @param sink_ind The array of indices to fill.
+ * @param cell_counts The cell counters to update.
+ * @param count_inhibited_sinks (return) The number of #sink to remove.
+ * @param count_extra_sinks (return) The number of #sink for on-the-fly
+ * creation.
+ * @param verbose Are we talkative ?
+ */
+void space_sinks_get_cell_index(struct space *s, int *sink_ind,
+ int *cell_counts, size_t *count_inhibited_sinks,
+ size_t *count_extra_sinks, int verbose) {
+
+ const ticks tic = getticks();
+
+ /* Re-set the counters */
+ s->min_sink_mass = FLT_MAX;
+ s->sum_sink_vel_norm = 0.f;
+
+ /* Pack the extra information */
+ struct index_data data;
+ data.s = s;
+ data.ind = sink_ind;
+ data.cell_counts = cell_counts;
+ data.count_inhibited_part = 0;
+ data.count_inhibited_gpart = 0;
+ data.count_inhibited_spart = 0;
+ data.count_inhibited_bpart = 0;
+ data.count_inhibited_sink = 0;
+ data.count_extra_part = 0;
+ data.count_extra_gpart = 0;
+ data.count_extra_spart = 0;
+ data.count_extra_bpart = 0;
+ data.count_extra_sink = 0;
+
+ threadpool_map(&s->e->threadpool, space_sinks_get_cell_index_mapper, s->sinks,
+ s->nr_sinks, sizeof(struct sink), threadpool_auto_chunk_size,
+ &data);
+
+ *count_inhibited_sinks = data.count_inhibited_sink;
+ *count_extra_sinks = data.count_extra_sink;
+
+ if (verbose)
+ message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+ clocks_getunit());
+}
+
+/**
+ * @brief Computes the cell index of all the b-particles.
+ *
+ * Also computes the minimal mass of all #bpart.
+ *
+ * @param s The #space.
+ * @param bind The array of indices to fill.
+ * @param cell_counts The cell counters to update.
+ * @param count_inhibited_bparts (return) The number of #bpart to remove.
+ * @param count_extra_bparts (return) The number of #bpart for on-the-fly
+ * creation.
+ * @param verbose Are we talkative ?
+ */
+void space_bparts_get_cell_index(struct space *s, int *bind, int *cell_counts,
+ size_t *count_inhibited_bparts,
+ size_t *count_extra_bparts, int verbose) {
+
+ const ticks tic = getticks();
+
+ /* Re-set the counters */
+ s->min_bpart_mass = FLT_MAX;
+ s->sum_bpart_vel_norm = 0.f;
+
+ /* Pack the extra information */
+ struct index_data data;
+ data.s = s;
+ data.ind = bind;
+ data.cell_counts = cell_counts;
+ data.count_inhibited_part = 0;
+ data.count_inhibited_gpart = 0;
+ data.count_inhibited_spart = 0;
+ data.count_inhibited_bpart = 0;
+ data.count_inhibited_sink = 0;
+ data.count_extra_part = 0;
+ data.count_extra_gpart = 0;
+ data.count_extra_spart = 0;
+ data.count_extra_bpart = 0;
+ data.count_extra_sink = 0;
+
+ threadpool_map(&s->e->threadpool, space_bparts_get_cell_index_mapper,
+ s->bparts, s->nr_bparts, sizeof(struct bpart),
+ threadpool_auto_chunk_size, &data);
+
+ *count_inhibited_bparts = data.count_inhibited_bpart;
+ *count_extra_bparts = data.count_extra_bpart;
+
+ if (verbose)
+ message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+ clocks_getunit());
+}
diff --git a/src/space_extras.c b/src/space_extras.c
new file mode 100644
index 0000000000000000000000000000000000000000..5eda872340a14534b1f561ee9b81337574aac384
--- /dev/null
+++ b/src/space_extras.c
@@ -0,0 +1,576 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* This object's header. */
+#include "space.h"
+
+/* Local headers. */
+#include "cell.h"
+#include "engine.h"
+
+/* Some standard headers. */
+#include <string.h>
+
+/**
+ * @brief Allocate memory for the extra particles used for on-the-fly creation.
+ *
+ * This rarely actually allocates memory. Most of the time, we convert
+ * pre-allocated memory inot extra particles.
+ *
+ * This function also sets the extra particles' location to their top-level
+ * cells. They can then be sorted into their correct memory position later on.
+ *
+ * @param s The current #space.
+ * @param verbose Are we talkative?
+ */
+void space_allocate_extras(struct space *s, int verbose) {
+
+ const int local_nodeID = s->e->nodeID;
+
+ /* Anything to do here? (Abort if we don't want extras)*/
+ if (space_extra_parts == 0 && space_extra_gparts == 0 &&
+ space_extra_sparts == 0 && space_extra_bparts == 0 &&
+ space_extra_sinks == 0)
+ return;
+
+ /* The top-level cells */
+ const struct cell *cells = s->cells_top;
+ const double half_cell_width[3] = {0.5 * cells[0].width[0],
+ 0.5 * cells[0].width[1],
+ 0.5 * cells[0].width[2]};
+
+ /* The current number of particles (including spare ones) */
+ size_t nr_parts = s->nr_parts;
+ size_t nr_gparts = s->nr_gparts;
+ size_t nr_sparts = s->nr_sparts;
+ size_t nr_bparts = s->nr_bparts;
+ size_t nr_sinks = s->nr_sinks;
+
+ /* The current number of actual particles */
+ size_t nr_actual_parts = nr_parts - s->nr_extra_parts;
+ size_t nr_actual_gparts = nr_gparts - s->nr_extra_gparts;
+ size_t nr_actual_sparts = nr_sparts - s->nr_extra_sparts;
+ size_t nr_actual_bparts = nr_bparts - s->nr_extra_bparts;
+ size_t nr_actual_sinks = nr_sinks - s->nr_extra_sinks;
+
+ /* The number of particles we allocated memory for (MPI overhead) */
+ size_t size_parts = s->size_parts;
+ size_t size_gparts = s->size_gparts;
+ size_t size_sparts = s->size_sparts;
+ size_t size_bparts = s->size_bparts;
+ size_t size_sinks = s->size_sinks;
+
+ int *local_cells = (int *)malloc(sizeof(int) * s->nr_cells);
+ if (local_cells == NULL)
+ error("Failed to allocate list of local top-level cells");
+
+ /* List the local cells */
+ size_t nr_local_cells = 0;
+ for (int i = 0; i < s->nr_cells; ++i) {
+ if (s->cells_top[i].nodeID == local_nodeID) {
+ local_cells[nr_local_cells] = i;
+ ++nr_local_cells;
+ }
+ }
+
+ /* Number of extra particles we want for each type */
+ const size_t expected_num_extra_parts = nr_local_cells * space_extra_parts;
+ const size_t expected_num_extra_gparts = nr_local_cells * space_extra_gparts;
+ const size_t expected_num_extra_sparts = nr_local_cells * space_extra_sparts;
+ const size_t expected_num_extra_bparts = nr_local_cells * space_extra_bparts;
+ const size_t expected_num_extra_sinks = nr_local_cells * space_extra_sinks;
+
+ if (verbose) {
+ message("Currently have %zd/%zd/%zd/%zd/%zd real particles.",
+ nr_actual_parts, nr_actual_gparts, nr_actual_sinks,
+ nr_actual_sparts, nr_actual_bparts);
+ message("Currently have %zd/%zd/%zd/%zd/%zd spaces for extra particles.",
+ s->nr_extra_parts, s->nr_extra_gparts, s->nr_extra_sinks,
+ s->nr_extra_sparts, s->nr_extra_bparts);
+ message(
+ "Requesting space for future %zd/%zd/%zd/%zd/%zd "
+ "part/gpart/sinks/sparts/bparts.",
+ expected_num_extra_parts, expected_num_extra_gparts,
+ expected_num_extra_sinks, expected_num_extra_sparts,
+ expected_num_extra_bparts);
+ }
+
+ if (expected_num_extra_parts < s->nr_extra_parts)
+ error("Reduction in top-level cells number not handled.");
+ if (expected_num_extra_gparts < s->nr_extra_gparts)
+ error("Reduction in top-level cells number not handled.");
+ if (expected_num_extra_sparts < s->nr_extra_sparts)
+ error("Reduction in top-level cells number not handled.");
+ if (expected_num_extra_bparts < s->nr_extra_bparts)
+ error("Reduction in top-level cells number not handled.");
+ if (expected_num_extra_sinks < s->nr_extra_sinks)
+ error("Reduction in top-level cells number not handled.");
+
+ /* Do we have enough space for the extra gparts (i.e. we haven't used up any)
+ * ? */
+ if (nr_actual_gparts + expected_num_extra_gparts > nr_gparts) {
+
+ /* Ok... need to put some more in the game */
+
+ /* Do we need to reallocate? */
+ if (nr_actual_gparts + expected_num_extra_gparts > size_gparts) {
+
+ size_gparts = (nr_actual_gparts + expected_num_extra_gparts) *
+ engine_redistribute_alloc_margin;
+
+ if (verbose)
+ message("Re-allocating gparts array from %zd to %zd", s->size_gparts,
+ size_gparts);
+
+ /* Create more space for parts */
+ struct gpart *gparts_new = NULL;
+ if (swift_memalign("gparts", (void **)&gparts_new, gpart_align,
+ sizeof(struct gpart) * size_gparts) != 0)
+ error("Failed to allocate new gpart data");
+ const ptrdiff_t delta = gparts_new - s->gparts;
+ memcpy(gparts_new, s->gparts, sizeof(struct gpart) * s->size_gparts);
+ swift_free("gparts", s->gparts);
+ s->gparts = gparts_new;
+
+ /* Update the counter */
+ s->size_gparts = size_gparts;
+
+ /* We now need to reset all the part and spart pointers */
+ for (size_t i = 0; i < nr_parts; ++i) {
+ if (s->parts[i].time_bin != time_bin_not_created)
+ s->parts[i].gpart += delta;
+ }
+ for (size_t i = 0; i < nr_sparts; ++i) {
+ if (s->sparts[i].time_bin != time_bin_not_created)
+ s->sparts[i].gpart += delta;
+ }
+ for (size_t i = 0; i < nr_bparts; ++i) {
+ if (s->bparts[i].time_bin != time_bin_not_created)
+ s->bparts[i].gpart += delta;
+ }
+ }
+
+ /* Turn some of the allocated spares into particles we can use */
+ for (size_t i = nr_gparts; i < nr_actual_gparts + expected_num_extra_gparts;
+ ++i) {
+ bzero(&s->gparts[i], sizeof(struct gpart));
+ s->gparts[i].time_bin = time_bin_not_created;
+ s->gparts[i].type = swift_type_dark_matter;
+ s->gparts[i].id_or_neg_offset = -1;
+ }
+
+ /* Put the spare particles in their correct cell */
+ size_t local_cell_id = 0;
+ int current_cell = local_cells[local_cell_id];
+ int count_in_cell = 0;
+ size_t count_extra_gparts = 0;
+ for (size_t i = 0; i < nr_actual_gparts + expected_num_extra_gparts; ++i) {
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (current_cell == s->nr_cells)
+ error("Cell counter beyond the maximal nr. cells.");
+#endif
+
+ if (s->gparts[i].time_bin == time_bin_not_created) {
+
+ /* We want the extra particles to be at the centre of their cell */
+ s->gparts[i].x[0] = cells[current_cell].loc[0] + half_cell_width[0];
+ s->gparts[i].x[1] = cells[current_cell].loc[1] + half_cell_width[1];
+ s->gparts[i].x[2] = cells[current_cell].loc[2] + half_cell_width[2];
+ ++count_in_cell;
+ count_extra_gparts++;
+ }
+
+ /* Once we have reached the number of extra gpart per cell, we move to the
+ * next */
+ if (count_in_cell == space_extra_gparts) {
+ ++local_cell_id;
+
+ if (local_cell_id == nr_local_cells) break;
+
+ current_cell = local_cells[local_cell_id];
+ count_in_cell = 0;
+ }
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (count_extra_gparts != expected_num_extra_gparts)
+ error("Constructed the wrong number of extra gparts (%zd vs. %zd)",
+ count_extra_gparts, expected_num_extra_gparts);
+#endif
+
+ /* Update the counters */
+ s->nr_gparts = nr_actual_gparts + expected_num_extra_gparts;
+ s->nr_extra_gparts = expected_num_extra_gparts;
+ }
+
+ /* Do we have enough space for the extra parts (i.e. we haven't used up any) ?
+ */
+ if (nr_actual_parts + expected_num_extra_parts > nr_parts) {
+
+ /* Ok... need to put some more in the game */
+
+ /* Do we need to reallocate? */
+ if (nr_actual_parts + expected_num_extra_parts > size_parts) {
+
+ size_parts = (nr_actual_parts + expected_num_extra_parts) *
+ engine_redistribute_alloc_margin;
+
+ if (verbose)
+ message("Re-allocating parts array from %zd to %zd", s->size_parts,
+ size_parts);
+
+ /* Create more space for parts */
+ struct part *parts_new = NULL;
+ if (swift_memalign("parts", (void **)&parts_new, part_align,
+ sizeof(struct part) * size_parts) != 0)
+ error("Failed to allocate new part data");
+ memcpy(parts_new, s->parts, sizeof(struct part) * s->size_parts);
+ swift_free("parts", s->parts);
+ s->parts = parts_new;
+
+ /* Same for xparts */
+ struct xpart *xparts_new = NULL;
+ if (swift_memalign("xparts", (void **)&xparts_new, xpart_align,
+ sizeof(struct xpart) * size_parts) != 0)
+ error("Failed to allocate new xpart data");
+ memcpy(xparts_new, s->xparts, sizeof(struct xpart) * s->size_parts);
+ swift_free("xparts", s->xparts);
+ s->xparts = xparts_new;
+
+ /* Update the counter */
+ s->size_parts = size_parts;
+ }
+
+ /* Turn some of the allocated spares into particles we can use */
+ for (size_t i = nr_parts; i < nr_actual_parts + expected_num_extra_parts;
+ ++i) {
+ bzero(&s->parts[i], sizeof(struct part));
+ bzero(&s->xparts[i], sizeof(struct xpart));
+ s->parts[i].time_bin = time_bin_not_created;
+ s->parts[i].id = -42;
+ }
+
+ /* Put the spare particles in their correct cell */
+ size_t local_cell_id = 0;
+ int current_cell = local_cells[local_cell_id];
+ int count_in_cell = 0;
+ size_t count_extra_parts = 0;
+ for (size_t i = 0; i < nr_actual_parts + expected_num_extra_parts; ++i) {
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (current_cell == s->nr_cells)
+ error("Cell counter beyond the maximal nr. cells.");
+#endif
+
+ if (s->parts[i].time_bin == time_bin_not_created) {
+
+ /* We want the extra particles to be at the centre of their cell */
+ s->parts[i].x[0] = cells[current_cell].loc[0] + half_cell_width[0];
+ s->parts[i].x[1] = cells[current_cell].loc[1] + half_cell_width[1];
+ s->parts[i].x[2] = cells[current_cell].loc[2] + half_cell_width[2];
+ ++count_in_cell;
+ count_extra_parts++;
+ }
+
+ /* Once we have reached the number of extra part per cell, we move to the
+ * next */
+ if (count_in_cell == space_extra_parts) {
+ ++local_cell_id;
+
+ if (local_cell_id == nr_local_cells) break;
+
+ current_cell = local_cells[local_cell_id];
+ count_in_cell = 0;
+ }
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (count_extra_parts != expected_num_extra_parts)
+ error("Constructed the wrong number of extra parts (%zd vs. %zd)",
+ count_extra_parts, expected_num_extra_parts);
+#endif
+
+ /* Update the counters */
+ s->nr_parts = nr_actual_parts + expected_num_extra_parts;
+ s->nr_extra_parts = expected_num_extra_parts;
+ }
+
+ /* Do we have enough space for the extra sinks (i.e. we haven't used up any)
+ * ? */
+ if (nr_actual_sinks + expected_num_extra_sinks > nr_sinks) {
+ /* Ok... need to put some more in the game */
+
+ /* Do we need to reallocate? */
+ if (nr_actual_sinks + expected_num_extra_sinks > size_sinks) {
+
+ size_sinks = (nr_actual_sinks + expected_num_extra_sinks) *
+ engine_redistribute_alloc_margin;
+
+ if (verbose)
+ message("Re-allocating sinks array from %zd to %zd", s->size_sinks,
+ size_sinks);
+
+ /* Create more space for parts */
+ struct sink *sinks_new = NULL;
+ if (swift_memalign("sinks", (void **)&sinks_new, sink_align,
+ sizeof(struct sink) * size_sinks) != 0)
+ error("Failed to allocate new sink data");
+ memcpy(sinks_new, s->sinks, sizeof(struct sink) * s->size_sinks);
+ swift_free("sinks", s->sinks);
+ s->sinks = sinks_new;
+
+ /* Update the counter */
+ s->size_sinks = size_sinks;
+ }
+
+ /* Turn some of the allocated spares into particles we can use */
+ for (size_t i = nr_sinks; i < nr_actual_sinks + expected_num_extra_sinks;
+ ++i) {
+ bzero(&s->sinks[i], sizeof(struct sink));
+ s->sinks[i].time_bin = time_bin_not_created;
+ s->sinks[i].id = -42;
+ }
+
+ /* Put the spare particles in their correct cell */
+ size_t local_cell_id = 0;
+ int current_cell = local_cells[local_cell_id];
+ int count_in_cell = 0;
+ size_t count_extra_sinks = 0;
+ for (size_t i = 0; i < nr_actual_sinks + expected_num_extra_sinks; ++i) {
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (current_cell == s->nr_cells)
+ error("Cell counter beyond the maximal nr. cells.");
+#endif
+
+ if (s->sinks[i].time_bin == time_bin_not_created) {
+
+ /* We want the extra particles to be at the centre of their cell */
+ s->sinks[i].x[0] = cells[current_cell].loc[0] + half_cell_width[0];
+ s->sinks[i].x[1] = cells[current_cell].loc[1] + half_cell_width[1];
+ s->sinks[i].x[2] = cells[current_cell].loc[2] + half_cell_width[2];
+ ++count_in_cell;
+ count_extra_sinks++;
+ }
+
+ /* Once we have reached the number of extra sink per cell, we move to the
+ * next */
+ if (count_in_cell == space_extra_sinks) {
+ ++local_cell_id;
+
+ if (local_cell_id == nr_local_cells) break;
+
+ current_cell = local_cells[local_cell_id];
+ count_in_cell = 0;
+ }
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (count_extra_sinks != expected_num_extra_sinks)
+ error("Constructed the wrong number of extra sinks (%zd vs. %zd)",
+ count_extra_sinks, expected_num_extra_sinks);
+#endif
+
+ /* Update the counters */
+ s->nr_sinks = nr_actual_sinks + expected_num_extra_sinks;
+ s->nr_extra_sinks = expected_num_extra_sinks;
+ }
+
+ /* Do we have enough space for the extra sparts (i.e. we haven't used up any)
+ * ? */
+ if (nr_actual_sparts + expected_num_extra_sparts > nr_sparts) {
+
+ /* Ok... need to put some more in the game */
+
+ /* Do we need to reallocate? */
+ if (nr_actual_sparts + expected_num_extra_sparts > size_sparts) {
+
+ size_sparts = (nr_actual_sparts + expected_num_extra_sparts) *
+ engine_redistribute_alloc_margin;
+
+ if (verbose)
+ message("Re-allocating sparts array from %zd to %zd", s->size_sparts,
+ size_sparts);
+
+ /* Create more space for parts */
+ struct spart *sparts_new = NULL;
+ if (swift_memalign("sparts", (void **)&sparts_new, spart_align,
+ sizeof(struct spart) * size_sparts) != 0)
+ error("Failed to allocate new spart data");
+ memcpy(sparts_new, s->sparts, sizeof(struct spart) * s->size_sparts);
+ swift_free("sparts", s->sparts);
+ s->sparts = sparts_new;
+
+ /* Update the counter */
+ s->size_sparts = size_sparts;
+ }
+
+ /* Turn some of the allocated spares into particles we can use */
+ for (size_t i = nr_sparts; i < nr_actual_sparts + expected_num_extra_sparts;
+ ++i) {
+ bzero(&s->sparts[i], sizeof(struct spart));
+ s->sparts[i].time_bin = time_bin_not_created;
+ s->sparts[i].id = -42;
+ }
+
+ /* Put the spare particles in their correct cell */
+ size_t local_cell_id = 0;
+ int current_cell = local_cells[local_cell_id];
+ int count_in_cell = 0;
+ size_t count_extra_sparts = 0;
+ for (size_t i = 0; i < nr_actual_sparts + expected_num_extra_sparts; ++i) {
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (current_cell == s->nr_cells)
+ error("Cell counter beyond the maximal nr. cells.");
+#endif
+
+ if (s->sparts[i].time_bin == time_bin_not_created) {
+
+ /* We want the extra particles to be at the centre of their cell */
+ s->sparts[i].x[0] = cells[current_cell].loc[0] + half_cell_width[0];
+ s->sparts[i].x[1] = cells[current_cell].loc[1] + half_cell_width[1];
+ s->sparts[i].x[2] = cells[current_cell].loc[2] + half_cell_width[2];
+ ++count_in_cell;
+ count_extra_sparts++;
+ }
+
+ /* Once we have reached the number of extra spart per cell, we move to the
+ * next */
+ if (count_in_cell == space_extra_sparts) {
+ ++local_cell_id;
+
+ if (local_cell_id == nr_local_cells) break;
+
+ current_cell = local_cells[local_cell_id];
+ count_in_cell = 0;
+ }
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (count_extra_sparts != expected_num_extra_sparts)
+ error("Constructed the wrong number of extra sparts (%zd vs. %zd)",
+ count_extra_sparts, expected_num_extra_sparts);
+#endif
+
+ /* Update the counters */
+ s->nr_sparts = nr_actual_sparts + expected_num_extra_sparts;
+ s->nr_extra_sparts = expected_num_extra_sparts;
+ }
+
+ /* Do we have enough space for the extra bparts (i.e. we haven't used up any)
+ * ? */
+ if (nr_actual_bparts + expected_num_extra_bparts > nr_bparts) {
+
+ /* Ok... need to put some more in the game */
+
+ /* Do we need to reallocate? */
+ if (nr_actual_bparts + expected_num_extra_bparts > size_bparts) {
+
+ size_bparts = (nr_actual_bparts + expected_num_extra_bparts) *
+ engine_redistribute_alloc_margin;
+
+ if (verbose)
+ message("Re-allocating bparts array from %zd to %zd", s->size_bparts,
+ size_bparts);
+
+ /* Create more space for parts */
+ struct bpart *bparts_new = NULL;
+ if (swift_memalign("bparts", (void **)&bparts_new, bpart_align,
+ sizeof(struct bpart) * size_bparts) != 0)
+ error("Failed to allocate new bpart data");
+ memcpy(bparts_new, s->bparts, sizeof(struct bpart) * s->size_bparts);
+ swift_free("bparts", s->bparts);
+ s->bparts = bparts_new;
+
+ /* Update the counter */
+ s->size_bparts = size_bparts;
+ }
+
+ /* Turn some of the allocated spares into particles we can use */
+ for (size_t i = nr_bparts; i < nr_actual_bparts + expected_num_extra_bparts;
+ ++i) {
+ bzero(&s->bparts[i], sizeof(struct bpart));
+ s->bparts[i].time_bin = time_bin_not_created;
+ s->bparts[i].id = -42;
+ }
+
+ /* Put the spare particles in their correct cell */
+ size_t local_cell_id = 0;
+ int current_cell = local_cells[local_cell_id];
+ int count_in_cell = 0;
+ size_t count_extra_bparts = 0;
+ for (size_t i = 0; i < nr_actual_bparts + expected_num_extra_bparts; ++i) {
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (current_cell == s->nr_cells)
+ error("Cell counter beyond the maximal nr. cells.");
+#endif
+
+ if (s->bparts[i].time_bin == time_bin_not_created) {
+
+ /* We want the extra particles to be at the centre of their cell */
+ s->bparts[i].x[0] = cells[current_cell].loc[0] + half_cell_width[0];
+ s->bparts[i].x[1] = cells[current_cell].loc[1] + half_cell_width[1];
+ s->bparts[i].x[2] = cells[current_cell].loc[2] + half_cell_width[2];
+ ++count_in_cell;
+ count_extra_bparts++;
+ }
+
+ /* Once we have reached the number of extra bpart per cell, we move to the
+ * next */
+ if (count_in_cell == space_extra_bparts) {
+ ++local_cell_id;
+
+ if (local_cell_id == nr_local_cells) break;
+
+ current_cell = local_cells[local_cell_id];
+ count_in_cell = 0;
+ }
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (count_extra_bparts != expected_num_extra_bparts)
+ error("Constructed the wrong number of extra bparts (%zd vs. %zd)",
+ count_extra_bparts, expected_num_extra_bparts);
+#endif
+
+ /* Update the counters */
+ s->nr_bparts = nr_actual_bparts + expected_num_extra_bparts;
+ s->nr_extra_bparts = expected_num_extra_bparts;
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Verify that the links are correct */
+ if ((nr_gparts > 0 && nr_parts > 0) || (nr_gparts > 0 && nr_sparts > 0) ||
+ (nr_gparts > 0 && nr_bparts > 0) || (nr_gparts > 0 && nr_sinks > 0))
+ part_verify_links(s->parts, s->gparts, s->sinks, s->sparts, s->bparts,
+ nr_parts, nr_gparts, nr_sinks, nr_sparts, nr_bparts,
+ verbose);
+#endif
+
+ /* Free the list of local cells */
+ free(local_cells);
+}
diff --git a/src/space_first_init.c b/src/space_first_init.c
new file mode 100644
index 0000000000000000000000000000000000000000..46c9d5f573d066e5e3629323dd30fee90292b44b
--- /dev/null
+++ b/src/space_first_init.c
@@ -0,0 +1,479 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* This object's header. */
+#include "space.h"
+
+/* Local headers. */
+#include "black_holes.h"
+#include "chemistry.h"
+#include "engine.h"
+#include "gravity.h"
+#include "pressure_floor.h"
+#include "rt.h"
+#include "sink.h"
+#include "star_formation.h"
+#include "stars.h"
+#include "threadpool.h"
+#include "tracers.h"
+
+void space_first_init_parts_mapper(void *restrict map_data, int count,
+ void *restrict extra_data) {
+
+ struct part *restrict p = (struct part *)map_data;
+ const struct space *restrict s = (struct space *)extra_data;
+ const struct engine *e = s->e;
+
+ const ptrdiff_t delta = p - s->parts;
+ struct xpart *restrict xp = s->xparts + delta;
+
+ /* Extract some constants */
+ const struct cosmology *cosmo = s->e->cosmology;
+ const struct phys_const *phys_const = s->e->physical_constants;
+ const struct unit_system *us = s->e->internal_units;
+ const float a_factor_vel = cosmo->a;
+
+ const struct hydro_props *hydro_props = s->e->hydro_properties;
+ const float u_init = hydro_props->initial_internal_energy;
+ const float hydro_h_min_ratio = e->hydro_properties->h_min_ratio;
+
+ const struct gravity_props *grav_props = s->e->gravity_properties;
+ const int with_gravity = e->policy & engine_policy_self_gravity;
+
+ const struct chemistry_global_data *chemistry = e->chemistry;
+ const struct star_formation *star_formation = e->star_formation;
+ const struct cooling_function_data *cool_func = e->cooling_func;
+
+ /* Check that the smoothing lengths are non-zero */
+ for (int k = 0; k < count; k++) {
+ if (p[k].h <= 0.)
+ error("Invalid value of smoothing length for part %lld h=%e", p[k].id,
+ p[k].h);
+
+ if (with_gravity) {
+ const struct gpart *gp = p[k].gpart;
+ const float softening = gravity_get_softening(gp, grav_props);
+ p->h = max(p->h, softening * hydro_h_min_ratio);
+ }
+ }
+
+ /* Convert velocities to internal units */
+ for (int k = 0; k < count; k++) {
+ p[k].v[0] *= a_factor_vel;
+ p[k].v[1] *= a_factor_vel;
+ p[k].v[2] *= a_factor_vel;
+
+#ifdef HYDRO_DIMENSION_2D
+ p[k].x[2] = 0.f;
+ p[k].v[2] = 0.f;
+#endif
+
+#ifdef HYDRO_DIMENSION_1D
+ p[k].x[1] = p[k].x[2] = 0.f;
+ p[k].v[1] = p[k].v[2] = 0.f;
+#endif
+ }
+
+ /* Overwrite the internal energy? */
+ if (u_init > 0.f) {
+ for (int k = 0; k < count; k++) {
+ hydro_set_init_internal_energy(&p[k], u_init);
+ }
+ }
+
+ /* Initialise the rest */
+ for (int k = 0; k < count; k++) {
+
+ hydro_first_init_part(&p[k], &xp[k]);
+ p[k].limiter_data.min_ngb_time_bin = num_time_bins + 1;
+ p[k].limiter_data.wakeup = time_bin_not_awake;
+ p[k].limiter_data.to_be_synchronized = 0;
+
+#ifdef WITH_LOGGER
+ logger_part_data_init(&xp[k].logger_data);
+#endif
+
+ /* Also initialise the chemistry */
+ chemistry_first_init_part(phys_const, us, cosmo, chemistry, &p[k], &xp[k]);
+
+ /* Also initialise the pressure floor */
+ pressure_floor_first_init_part(phys_const, us, cosmo, &p[k], &xp[k]);
+
+ /* Also initialise the star formation */
+ star_formation_first_init_part(phys_const, us, cosmo, star_formation, &p[k],
+ &xp[k]);
+
+ /* And the cooling */
+ cooling_first_init_part(phys_const, us, hydro_props, cosmo, cool_func,
+ &p[k], &xp[k]);
+
+ /* And the tracers */
+ tracers_first_init_xpart(&p[k], &xp[k], us, phys_const, cosmo, hydro_props,
+ cool_func);
+
+ /* And the black hole markers */
+ black_holes_mark_part_as_not_swallowed(&p[k].black_holes_data);
+
+ /* And the radiative transfer */
+ rt_first_init_part(&p[k]);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Check part->gpart->part linkeage. */
+ if (p[k].gpart && p[k].gpart->id_or_neg_offset != -(k + delta))
+ error("Invalid gpart -> part link");
+
+ /* Initialise the time-integration check variables */
+ p[k].ti_drift = 0;
+ p[k].ti_kick = 0;
+#endif
+ }
+}
+
+/**
+ * @brief Initialises all the particles by setting them into a valid state
+ *
+ * Calls hydro_first_init_part() on all the particles
+ * Calls chemistry_first_init_part() on all the particles
+ * Calls cooling_first_init_part() on all the particles
+ */
+void space_first_init_parts(struct space *s, int verbose) {
+
+ const ticks tic = getticks();
+ if (s->nr_parts > 0)
+ threadpool_map(&s->e->threadpool, space_first_init_parts_mapper, s->parts,
+ s->nr_parts, sizeof(struct part), threadpool_auto_chunk_size,
+ s);
+
+ if (verbose)
+ message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+ clocks_getunit());
+}
+
+void space_first_init_gparts_mapper(void *restrict map_data, int count,
+ void *restrict extra_data) {
+
+ struct gpart *restrict gp = (struct gpart *)map_data;
+ const struct space *restrict s = (struct space *)extra_data;
+
+ const struct cosmology *cosmo = s->e->cosmology;
+ const float a_factor_vel = cosmo->a;
+ const struct gravity_props *grav_props = s->e->gravity_properties;
+
+ /* Convert velocities to internal units */
+ for (int k = 0; k < count; k++) {
+ gp[k].v_full[0] *= a_factor_vel;
+ gp[k].v_full[1] *= a_factor_vel;
+ gp[k].v_full[2] *= a_factor_vel;
+
+#ifdef HYDRO_DIMENSION_2D
+ gp[k].x[2] = 0.f;
+ gp[k].v_full[2] = 0.f;
+#endif
+
+#ifdef HYDRO_DIMENSION_1D
+ gp[k].x[1] = gp[k].x[2] = 0.f;
+ gp[k].v_full[1] = gp[k].v_full[2] = 0.f;
+#endif
+ }
+
+ /* Initialise the rest */
+ for (int k = 0; k < count; k++) {
+
+ gravity_first_init_gpart(&gp[k], grav_props);
+
+#ifdef WITH_LOGGER
+ logger_part_data_init(&gp[k].logger_data);
+#endif
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Initialise the time-integration check variables */
+ gp[k].ti_drift = 0;
+ gp[k].ti_kick = 0;
+ gp[k].ti_kick_mesh = 0;
+#endif
+ }
+}
+
+/**
+ * @brief Initialises all the g-particles by setting them into a valid state
+ *
+ * Calls gravity_first_init_gpart() on all the particles
+ */
+void space_first_init_gparts(struct space *s, int verbose) {
+
+ const ticks tic = getticks();
+ if (s->nr_gparts > 0)
+ threadpool_map(&s->e->threadpool, space_first_init_gparts_mapper, s->gparts,
+ s->nr_gparts, sizeof(struct gpart),
+ threadpool_auto_chunk_size, s);
+
+ if (verbose)
+ message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+ clocks_getunit());
+}
+
+void space_first_init_sparts_mapper(void *restrict map_data, int count,
+ void *restrict extra_data) {
+
+ struct spart *restrict sp = (struct spart *)map_data;
+ const struct space *restrict s = (struct space *)extra_data;
+ const struct engine *e = s->e;
+
+ const struct chemistry_global_data *chemistry = e->chemistry;
+
+#ifdef SWIFT_DEBUG_CHECKS
+ const ptrdiff_t delta = sp - s->sparts;
+#endif
+
+ const float initial_h = s->initial_spart_h;
+
+ const int with_feedback = (e->policy & engine_policy_feedback);
+ const int with_cosmology = (e->policy & engine_policy_cosmology);
+
+ const struct cosmology *cosmo = e->cosmology;
+ const struct stars_props *stars_properties = e->stars_properties;
+ const float a_factor_vel = cosmo->a;
+
+ /* Convert velocities to internal units */
+ for (int k = 0; k < count; k++) {
+
+ sp[k].v[0] *= a_factor_vel;
+ sp[k].v[1] *= a_factor_vel;
+ sp[k].v[2] *= a_factor_vel;
+
+ /* Imposed smoothing length from parameter file */
+ if (initial_h != -1.f) {
+ sp[k].h = initial_h;
+ }
+
+#ifdef HYDRO_DIMENSION_2D
+ sp[k].x[2] = 0.f;
+ sp[k].v[2] = 0.f;
+#endif
+
+#ifdef HYDRO_DIMENSION_1D
+ sp[k].x[1] = sp[k].x[2] = 0.f;
+ sp[k].v[1] = sp[k].v[2] = 0.f;
+#endif
+ }
+
+ /* Check that the smoothing lengths are non-zero */
+ for (int k = 0; k < count; k++) {
+ if (with_feedback && sp[k].h <= 0.)
+ error("Invalid value of smoothing length for spart %lld h=%e", sp[k].id,
+ sp[k].h);
+ }
+
+ /* Initialise the rest */
+ for (int k = 0; k < count; k++) {
+
+ stars_first_init_spart(&sp[k], stars_properties, with_cosmology, cosmo->a,
+ e->time);
+
+#ifdef WITH_LOGGER
+ logger_part_data_init(&sp[k].logger_data);
+#endif
+
+ /* Also initialise the chemistry */
+ chemistry_first_init_spart(chemistry, &sp[k]);
+
+ /* And radiative transfer data */
+ rt_first_init_spart(&sp[k]);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (sp[k].gpart && sp[k].gpart->id_or_neg_offset != -(k + delta))
+ error("Invalid gpart -> spart link");
+
+ /* Initialise the time-integration check variables */
+ sp[k].ti_drift = 0;
+ sp[k].ti_kick = 0;
+#endif
+ }
+}
+
+/**
+ * @brief Initialises all the s-particles by setting them into a valid state
+ *
+ * Calls stars_first_init_spart() on all the particles
+ */
+void space_first_init_sparts(struct space *s, int verbose) {
+ const ticks tic = getticks();
+ if (s->nr_sparts > 0)
+ threadpool_map(&s->e->threadpool, space_first_init_sparts_mapper, s->sparts,
+ s->nr_sparts, sizeof(struct spart),
+ threadpool_auto_chunk_size, s);
+
+ if (verbose)
+ message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+ clocks_getunit());
+}
+
+void space_first_init_bparts_mapper(void *restrict map_data, int count,
+ void *restrict extra_data) {
+
+ struct bpart *restrict bp = (struct bpart *)map_data;
+ const struct space *restrict s = (struct space *)extra_data;
+ const struct engine *e = s->e;
+ const struct black_holes_props *props = e->black_holes_properties;
+
+#ifdef SWIFT_DEBUG_CHECKS
+ const ptrdiff_t delta = bp - s->bparts;
+#endif
+
+ const float initial_h = s->initial_bpart_h;
+
+ const struct cosmology *cosmo = e->cosmology;
+ const float a_factor_vel = cosmo->a;
+
+ /* Convert velocities to internal units */
+ for (int k = 0; k < count; k++) {
+
+ bp[k].v[0] *= a_factor_vel;
+ bp[k].v[1] *= a_factor_vel;
+ bp[k].v[2] *= a_factor_vel;
+
+ /* Imposed smoothing length from parameter file */
+ if (initial_h != -1.f) {
+ bp[k].h = initial_h;
+ }
+
+#ifdef HYDRO_DIMENSION_2D
+ bp[k].x[2] = 0.f;
+ bp[k].v[2] = 0.f;
+#endif
+
+#ifdef HYDRO_DIMENSION_1D
+ bp[k].x[1] = bp[k].x[2] = 0.f;
+ bp[k].v[1] = bp[k].v[2] = 0.f;
+#endif
+ }
+
+ /* Check that the smoothing lengths are non-zero */
+ for (int k = 0; k < count; k++) {
+ if (bp[k].h <= 0.)
+ error("Invalid value of smoothing length for bpart %lld h=%e", bp[k].id,
+ bp[k].h);
+ }
+
+ /* Initialise the rest */
+ for (int k = 0; k < count; k++) {
+
+ black_holes_first_init_bpart(&bp[k], props);
+
+ /* And the black hole merger markers */
+ black_holes_mark_bpart_as_not_swallowed(&bp[k].merger_data);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (bp[k].gpart && bp[k].gpart->id_or_neg_offset != -(k + delta))
+ error("Invalid gpart -> bpart link");
+
+ /* Initialise the time-integration check variables */
+ bp[k].ti_drift = 0;
+ bp[k].ti_kick = 0;
+#endif
+ }
+}
+
+/**
+ * @brief Initialises all the b-particles by setting them into a valid state
+ *
+ * Calls stars_first_init_bpart() on all the particles
+ */
+void space_first_init_bparts(struct space *s, int verbose) {
+ const ticks tic = getticks();
+ if (s->nr_bparts > 0)
+ threadpool_map(&s->e->threadpool, space_first_init_bparts_mapper, s->bparts,
+ s->nr_bparts, sizeof(struct bpart),
+ threadpool_auto_chunk_size, s);
+
+ if (verbose)
+ message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+ clocks_getunit());
+}
+
+void space_first_init_sinks_mapper(void *restrict map_data, int count,
+ void *restrict extra_data) {
+
+ struct sink *restrict sink = (struct sink *)map_data;
+ const struct space *restrict s = (struct space *)extra_data;
+ const struct engine *e = s->e;
+ const struct sink_props *props = e->sink_properties;
+
+#ifdef SWIFT_DEBUG_CHECKS
+ const ptrdiff_t delta = sink - s->sinks;
+#endif
+
+ const struct cosmology *cosmo = e->cosmology;
+ const float a_factor_vel = cosmo->a;
+
+ /* Convert velocities to internal units */
+ for (int k = 0; k < count; k++) {
+
+ sink[k].v[0] *= a_factor_vel;
+ sink[k].v[1] *= a_factor_vel;
+ sink[k].v[2] *= a_factor_vel;
+
+#ifdef HYDRO_DIMENSION_2D
+ sink[k].x[2] = 0.f;
+ sink[k].v[2] = 0.f;
+#endif
+
+#ifdef HYDRO_DIMENSION_1D
+ sink[k].x[1] = sink[k].x[2] = 0.f;
+ sink[k].v[1] = sink[k].v[2] = 0.f;
+#endif
+ }
+
+ /* Initialise the rest */
+ for (int k = 0; k < count; k++) {
+
+ sink_first_init_sink(&sink[k], props);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ if (sink[k].gpart && sink[k].gpart->id_or_neg_offset != -(k + delta))
+ error("Invalid gpart -> sink link");
+
+ /* Initialise the time-integration check variables */
+ sink[k].ti_drift = 0;
+ sink[k].ti_kick = 0;
+#endif
+ }
+}
+
+/**
+ * @brief Initialises all the sink-particles by setting them into a valid state
+ *
+ * Calls stars_first_init_sink() on all the particles
+ */
+void space_first_init_sinks(struct space *s, int verbose) {
+ const ticks tic = getticks();
+ if (s->nr_sinks > 0)
+ threadpool_map(&s->e->threadpool, space_first_init_sinks_mapper, s->sinks,
+ s->nr_sinks, sizeof(struct sink), threadpool_auto_chunk_size,
+ s);
+
+ if (verbose)
+ message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+ clocks_getunit());
+}
diff --git a/src/space_init.c b/src/space_init.c
new file mode 100644
index 0000000000000000000000000000000000000000..b0e710ca7308bd04db0585aad84c214b1c50d9d9
--- /dev/null
+++ b/src/space_init.c
@@ -0,0 +1,193 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* This object's header. */
+#include "space.h"
+
+/* Local headers. */
+#include "black_holes.h"
+#include "chemistry.h"
+#include "engine.h"
+#include "gravity.h"
+#include "pressure_floor.h"
+#include "rt.h"
+#include "sink.h"
+#include "star_formation.h"
+#include "stars.h"
+#include "threadpool.h"
+#include "tracers.h"
+
+void space_init_parts_mapper(void *restrict map_data, int count,
+ void *restrict extra_data) {
+
+ struct part *restrict parts = (struct part *)map_data;
+ const struct engine *restrict e = (struct engine *)extra_data;
+ const struct hydro_space *restrict hs = &e->s->hs;
+ const int with_cosmology = (e->policy & engine_policy_cosmology);
+
+ size_t ind = parts - e->s->parts;
+ struct xpart *restrict xparts = e->s->xparts + ind;
+
+ for (int k = 0; k < count; k++) {
+ hydro_init_part(&parts[k], hs);
+ black_holes_init_potential(&parts[k].black_holes_data);
+ chemistry_init_part(&parts[k], e->chemistry);
+ pressure_floor_init_part(&parts[k], &xparts[k]);
+ rt_init_part(&parts[k]);
+ star_formation_init_part(&parts[k], e->star_formation);
+ tracers_after_init(&parts[k], &xparts[k], e->internal_units,
+ e->physical_constants, with_cosmology, e->cosmology,
+ e->hydro_properties, e->cooling_func, e->time);
+ }
+}
+
+/**
+ * @brief Calls the #part initialisation function on all particles in the space.
+ *
+ * @param s The #space.
+ * @param verbose Are we talkative?
+ */
+void space_init_parts(struct space *s, int verbose) {
+
+ const ticks tic = getticks();
+
+ if (s->nr_parts > 0)
+ threadpool_map(&s->e->threadpool, space_init_parts_mapper, s->parts,
+ s->nr_parts, sizeof(struct part), threadpool_auto_chunk_size,
+ s->e);
+ if (verbose)
+ message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+ clocks_getunit());
+}
+
+void space_init_gparts_mapper(void *restrict map_data, int count,
+ void *restrict extra_data) {
+
+ struct gpart *gparts = (struct gpart *)map_data;
+ for (int k = 0; k < count; k++) gravity_init_gpart(&gparts[k]);
+}
+
+/**
+ * @brief Calls the #gpart initialisation function on all particles in the
+ * space.
+ *
+ * @param s The #space.
+ * @param verbose Are we talkative?
+ */
+void space_init_gparts(struct space *s, int verbose) {
+
+ const ticks tic = getticks();
+
+ if (s->nr_gparts > 0)
+ threadpool_map(&s->e->threadpool, space_init_gparts_mapper, s->gparts,
+ s->nr_gparts, sizeof(struct gpart),
+ threadpool_auto_chunk_size, /*extra_data=*/NULL);
+ if (verbose)
+ message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+ clocks_getunit());
+}
+
+void space_init_sparts_mapper(void *restrict map_data, int scount,
+ void *restrict extra_data) {
+
+ struct spart *restrict sparts = (struct spart *)map_data;
+ for (int k = 0; k < scount; k++) {
+ stars_init_spart(&sparts[k]);
+ rt_init_spart(&sparts[k]);
+ }
+}
+
+/**
+ * @brief Calls the #spart initialisation function on all particles in the
+ * space.
+ *
+ * @param s The #space.
+ * @param verbose Are we talkative?
+ */
+void space_init_sparts(struct space *s, int verbose) {
+
+ const ticks tic = getticks();
+
+ if (s->nr_sparts > 0)
+ threadpool_map(&s->e->threadpool, space_init_sparts_mapper, s->sparts,
+ s->nr_sparts, sizeof(struct spart),
+ threadpool_auto_chunk_size, /*extra_data=*/NULL);
+ if (verbose)
+ message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+ clocks_getunit());
+}
+
+void space_init_bparts_mapper(void *restrict map_data, int bcount,
+ void *restrict extra_data) {
+
+ struct bpart *restrict bparts = (struct bpart *)map_data;
+ for (int k = 0; k < bcount; k++) black_holes_init_bpart(&bparts[k]);
+}
+
+/**
+ * @brief Calls the #bpart initialisation function on all particles in the
+ * space.
+ *
+ * @param s The #space.
+ * @param verbose Are we talkative?
+ */
+void space_init_bparts(struct space *s, int verbose) {
+
+ const ticks tic = getticks();
+
+ if (s->nr_bparts > 0)
+ threadpool_map(&s->e->threadpool, space_init_bparts_mapper, s->bparts,
+ s->nr_bparts, sizeof(struct bpart),
+ threadpool_auto_chunk_size, /*extra_data=*/NULL);
+ if (verbose)
+ message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+ clocks_getunit());
+}
+
+void space_init_sinks_mapper(void *restrict map_data, int sink_count,
+ void *restrict extra_data) {
+
+ struct sink *restrict sinks = (struct sink *)map_data;
+ for (int k = 0; k < sink_count; k++) sink_init_sink(&sinks[k]);
+}
+
+/**
+ * @brief Calls the #sink initialisation function on all particles in the
+ * space.
+ *
+ * @param s The #space.
+ * @param verbose Are we talkative?
+ */
+void space_init_sinks(struct space *s, int verbose) {
+
+ const ticks tic = getticks();
+
+ if (s->nr_sinks > 0)
+ threadpool_map(&s->e->threadpool, space_init_sinks_mapper, s->sinks,
+ s->nr_sinks, sizeof(struct sink), threadpool_auto_chunk_size,
+ /*extra_data=*/NULL);
+ if (verbose)
+ message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+ clocks_getunit());
+}
diff --git a/src/space_rebuild.c b/src/space_rebuild.c
new file mode 100644
index 0000000000000000000000000000000000000000..f456be124731830a7c91f65caca4d1517644d701
--- /dev/null
+++ b/src/space_rebuild.c
@@ -0,0 +1,991 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* This object's header. */
+#include "space.h"
+
+/* Local headers. */
+#include "cell.h"
+#include "engine.h"
+#include "memswap.h"
+
+/*! Expected maximal number of strays received at a rebuild */
+extern int space_expected_max_nr_strays;
+
+/*! Counter for cell IDs (when debugging) */
+#if defined(SWIFT_DEBUG_CHECKS) || defined(SWIFT_CELL_GRAPH)
+extern int last_cell_id;
+#endif
+
+/**
+ * @brief Re-build the top-level cells as well as the whole hierarchy.
+ *
+ * @param s The #space in which to update the cells.
+ * @param repartitioned Did we just repartition?
+ * @param verbose Print messages to stdout or not
+ */
+void space_rebuild(struct space *s, int repartitioned, int verbose) {
+
+ const ticks tic = getticks();
+
+/* Be verbose about this. */
+#ifdef SWIFT_DEBUG_CHECKS
+ if (s->e->nodeID == 0 || verbose) message("(re)building space");
+ fflush(stdout);
+#endif
+#if defined(SWIFT_DEBUG_CHECKS) || defined(SWIFT_CELL_GRAPH)
+ /* Reset the cell counter */
+ last_cell_id = 1;
+#endif
+
+ /* Re-grid if necessary, or just re-set the cell data. */
+ space_regrid(s, verbose);
+
+ /* Allocate extra space for particles that will be created */
+ if (s->with_star_formation || s->e->policy & engine_policy_sinks)
+ space_allocate_extras(s, verbose);
+
+ struct cell *cells_top = s->cells_top;
+ const integertime_t ti_current = (s->e != NULL) ? s->e->ti_current : 0;
+ const int local_nodeID = s->e->nodeID;
+
+ /* The current number of particles */
+ size_t nr_parts = s->nr_parts;
+ size_t nr_gparts = s->nr_gparts;
+ size_t nr_sparts = s->nr_sparts;
+ size_t nr_bparts = s->nr_bparts;
+ size_t nr_sinks = s->nr_sinks;
+
+ /* The number of particles we allocated memory for */
+ size_t size_parts = s->size_parts;
+ size_t size_gparts = s->size_gparts;
+ size_t size_sparts = s->size_sparts;
+ size_t size_bparts = s->size_bparts;
+ size_t size_sinks = s->size_sinks;
+
+ /* Counter for the number of inhibited particles found on the node */
+ size_t count_inhibited_parts = 0;
+ size_t count_inhibited_gparts = 0;
+ size_t count_inhibited_sparts = 0;
+ size_t count_inhibited_bparts = 0;
+ size_t count_inhibited_sinks = 0;
+
+ /* Counter for the number of extra particles found on the node */
+ size_t count_extra_parts = 0;
+ size_t count_extra_gparts = 0;
+ size_t count_extra_sparts = 0;
+ size_t count_extra_bparts = 0;
+ size_t count_extra_sinks = 0;
+
+ /* Number of particles we expect to have after strays exchange */
+ const size_t h_index_size = size_parts + space_expected_max_nr_strays;
+ const size_t g_index_size = size_gparts + space_expected_max_nr_strays;
+ const size_t s_index_size = size_sparts + space_expected_max_nr_strays;
+ const size_t b_index_size = size_bparts + space_expected_max_nr_strays;
+ const size_t sink_index_size = size_sinks + space_expected_max_nr_strays;
+
+ /* Allocate arrays to store the indices of the cells where particles
+ belong. We allocate extra space to allow for particles we may
+ receive from other nodes */
+ int *h_index = (int *)swift_malloc("h_index", sizeof(int) * h_index_size);
+ int *g_index = (int *)swift_malloc("g_index", sizeof(int) * g_index_size);
+ int *s_index = (int *)swift_malloc("s_index", sizeof(int) * s_index_size);
+ int *b_index = (int *)swift_malloc("b_index", sizeof(int) * b_index_size);
+ int *sink_index =
+ (int *)swift_malloc("sink_index", sizeof(int) * sink_index_size);
+ if (h_index == NULL || g_index == NULL || s_index == NULL ||
+ b_index == NULL || sink_index == NULL)
+ error("Failed to allocate temporary particle indices.");
+
+ /* Allocate counters of particles that will land in each cell */
+ int *cell_part_counts =
+ (int *)swift_malloc("cell_part_counts", sizeof(int) * s->nr_cells);
+ int *cell_gpart_counts =
+ (int *)swift_malloc("cell_gpart_counts", sizeof(int) * s->nr_cells);
+ int *cell_spart_counts =
+ (int *)swift_malloc("cell_spart_counts", sizeof(int) * s->nr_cells);
+ int *cell_bpart_counts =
+ (int *)swift_malloc("cell_bpart_counts", sizeof(int) * s->nr_cells);
+ int *cell_sink_counts =
+ (int *)swift_malloc("cell_sink_counts", sizeof(int) * s->nr_cells);
+
+ if (cell_part_counts == NULL || cell_gpart_counts == NULL ||
+ cell_spart_counts == NULL || cell_bpart_counts == NULL ||
+ cell_sink_counts == NULL)
+ error("Failed to allocate cell particle count buffer.");
+
+ /* Initialise the counters, including buffer space for future particles */
+ for (int i = 0; i < s->nr_cells; ++i) {
+ cell_part_counts[i] = 0;
+ cell_gpart_counts[i] = 0;
+ cell_spart_counts[i] = 0;
+ cell_bpart_counts[i] = 0;
+ cell_sink_counts[i] = 0;
+ }
+
+ /* Run through the particles and get their cell index. */
+ if (nr_parts > 0)
+ space_parts_get_cell_index(s, h_index, cell_part_counts,
+ &count_inhibited_parts, &count_extra_parts,
+ verbose);
+ if (nr_gparts > 0)
+ space_gparts_get_cell_index(s, g_index, cell_gpart_counts,
+ &count_inhibited_gparts, &count_extra_gparts,
+ verbose);
+ if (nr_sparts > 0)
+ space_sparts_get_cell_index(s, s_index, cell_spart_counts,
+ &count_inhibited_sparts, &count_extra_sparts,
+ verbose);
+ if (nr_bparts > 0)
+ space_bparts_get_cell_index(s, b_index, cell_bpart_counts,
+ &count_inhibited_bparts, &count_extra_bparts,
+ verbose);
+ if (nr_sinks > 0)
+ space_sinks_get_cell_index(s, sink_index, cell_sink_counts,
+ &count_inhibited_sinks, &count_extra_sinks,
+ verbose);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Some safety checks */
+ if (repartitioned && count_inhibited_parts)
+ error("We just repartitioned but still found inhibited parts.");
+ if (repartitioned && count_inhibited_sparts)
+ error("We just repartitioned but still found inhibited sparts.");
+ if (repartitioned && count_inhibited_gparts)
+ error("We just repartitioned but still found inhibited gparts.");
+ if (repartitioned && count_inhibited_bparts)
+ error("We just repartitioned but still found inhibited bparts.");
+ if (repartitioned && count_inhibited_sinks)
+ error("We just repartitioned but still found inhibited sinks.");
+
+ if (count_extra_parts != s->nr_extra_parts)
+ error(
+ "Number of extra parts in the part array not matching the space "
+ "counter.");
+ if (count_extra_gparts != s->nr_extra_gparts)
+ error(
+ "Number of extra gparts in the gpart array not matching the space "
+ "counter.");
+ if (count_extra_sparts != s->nr_extra_sparts)
+ error(
+ "Number of extra sparts in the spart array not matching the space "
+ "counter.");
+ if (count_extra_bparts != s->nr_extra_bparts)
+ error(
+ "Number of extra bparts in the bpart array not matching the space "
+ "counter.");
+ if (count_extra_sinks != s->nr_extra_sinks)
+ error(
+ "Number of extra sinks in the sink array not matching the space "
+ "counter.");
+#endif
+
+ const ticks tic2 = getticks();
+
+ /* Move non-local parts and inhibited parts to the end of the list. */
+ if (!repartitioned && (s->e->nr_nodes > 1 || count_inhibited_parts > 0)) {
+
+ for (size_t k = 0; k < nr_parts; /* void */) {
+
+ /* Inhibited particle or foreign particle */
+ if (h_index[k] == -1 || cells_top[h_index[k]].nodeID != local_nodeID) {
+
+ /* One fewer particle */
+ nr_parts -= 1;
+
+ /* Swap the particle */
+ memswap(&s->parts[k], &s->parts[nr_parts], sizeof(struct part));
+
+ /* Swap the link with the gpart */
+ if (s->parts[k].gpart != NULL) {
+ s->parts[k].gpart->id_or_neg_offset = -k;
+ }
+ if (s->parts[nr_parts].gpart != NULL) {
+ s->parts[nr_parts].gpart->id_or_neg_offset = -nr_parts;
+ }
+
+ /* Swap the xpart */
+ memswap(&s->xparts[k], &s->xparts[nr_parts], sizeof(struct xpart));
+ /* Swap the index */
+ memswap(&h_index[k], &h_index[nr_parts], sizeof(int));
+
+ } else {
+ /* Increment when not exchanging otherwise we need to retest "k".*/
+ k++;
+ }
+ }
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Check that all parts are in the correct places. */
+ size_t check_count_inhibited_part = 0;
+ for (size_t k = 0; k < nr_parts; k++) {
+ if (h_index[k] == -1 || cells_top[h_index[k]].nodeID != local_nodeID) {
+ error("Failed to move all non-local parts to send list");
+ }
+ }
+ for (size_t k = nr_parts; k < s->nr_parts; k++) {
+ if (h_index[k] != -1 && cells_top[h_index[k]].nodeID == local_nodeID) {
+ error("Failed to remove local parts from send list");
+ }
+ if (h_index[k] == -1) ++check_count_inhibited_part;
+ }
+ if (check_count_inhibited_part != count_inhibited_parts)
+ error("Counts of inhibited particles do not match!");
+#endif /* SWIFT_DEBUG_CHECKS */
+
+ /* Move non-local sparts and inhibited sparts to the end of the list. */
+ if (!repartitioned && (s->e->nr_nodes > 1 || count_inhibited_sparts > 0)) {
+
+ for (size_t k = 0; k < nr_sparts; /* void */) {
+
+ /* Inhibited particle or foreign particle */
+ if (s_index[k] == -1 || cells_top[s_index[k]].nodeID != local_nodeID) {
+
+ /* One fewer particle */
+ nr_sparts -= 1;
+
+ /* Swap the particle */
+ memswap(&s->sparts[k], &s->sparts[nr_sparts], sizeof(struct spart));
+
+ /* Swap the link with the gpart */
+ if (s->sparts[k].gpart != NULL) {
+ s->sparts[k].gpart->id_or_neg_offset = -k;
+ }
+ if (s->sparts[nr_sparts].gpart != NULL) {
+ s->sparts[nr_sparts].gpart->id_or_neg_offset = -nr_sparts;
+ }
+
+ /* Swap the index */
+ memswap(&s_index[k], &s_index[nr_sparts], sizeof(int));
+
+ } else {
+ /* Increment when not exchanging otherwise we need to retest "k".*/
+ k++;
+ }
+ }
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Check that all sparts are in the correct place. */
+ size_t check_count_inhibited_spart = 0;
+ for (size_t k = 0; k < nr_sparts; k++) {
+ if (s_index[k] == -1 || cells_top[s_index[k]].nodeID != local_nodeID) {
+ error("Failed to move all non-local sparts to send list");
+ }
+ }
+ for (size_t k = nr_sparts; k < s->nr_sparts; k++) {
+ if (s_index[k] != -1 && cells_top[s_index[k]].nodeID == local_nodeID) {
+ error("Failed to remove local sparts from send list");
+ }
+ if (s_index[k] == -1) ++check_count_inhibited_spart;
+ }
+ if (check_count_inhibited_spart != count_inhibited_sparts)
+ error("Counts of inhibited s-particles do not match!");
+#endif /* SWIFT_DEBUG_CHECKS */
+
+ /* Move non-local bparts and inhibited bparts to the end of the list. */
+ if (!repartitioned && (s->e->nr_nodes > 1 || count_inhibited_bparts > 0)) {
+
+ for (size_t k = 0; k < nr_bparts; /* void */) {
+
+ /* Inhibited particle or foreign particle */
+ if (b_index[k] == -1 || cells_top[b_index[k]].nodeID != local_nodeID) {
+
+ /* One fewer particle */
+ nr_bparts -= 1;
+
+ /* Swap the particle */
+ memswap(&s->bparts[k], &s->bparts[nr_bparts], sizeof(struct bpart));
+
+ /* Swap the link with the gpart */
+ if (s->bparts[k].gpart != NULL) {
+ s->bparts[k].gpart->id_or_neg_offset = -k;
+ }
+ if (s->bparts[nr_bparts].gpart != NULL) {
+ s->bparts[nr_bparts].gpart->id_or_neg_offset = -nr_bparts;
+ }
+
+ /* Swap the index */
+ memswap(&b_index[k], &b_index[nr_bparts], sizeof(int));
+
+ } else {
+ /* Increment when not exchanging otherwise we need to retest "k".*/
+ k++;
+ }
+ }
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Check that all bparts are in the correct place. */
+ size_t check_count_inhibited_bpart = 0;
+ for (size_t k = 0; k < nr_bparts; k++) {
+ if (b_index[k] == -1 || cells_top[b_index[k]].nodeID != local_nodeID) {
+ error("Failed to move all non-local bparts to send list");
+ }
+ }
+ for (size_t k = nr_bparts; k < s->nr_bparts; k++) {
+ if (b_index[k] != -1 && cells_top[b_index[k]].nodeID == local_nodeID) {
+ error("Failed to remove local bparts from send list");
+ }
+ if (b_index[k] == -1) ++check_count_inhibited_bpart;
+ }
+ if (check_count_inhibited_bpart != count_inhibited_bparts)
+ error("Counts of inhibited b-particles do not match!");
+#endif /* SWIFT_DEBUG_CHECKS */
+
+ /* Move non-local sinks and inhibited sinks to the end of the list. */
+ if (!repartitioned && (s->e->nr_nodes > 1 || count_inhibited_sinks > 0)) {
+
+ for (size_t k = 0; k < nr_sinks; /* void */) {
+
+ /* Inhibited particle or foreign particle */
+ if (sink_index[k] == -1 ||
+ cells_top[sink_index[k]].nodeID != local_nodeID) {
+
+ /* One fewer particle */
+ nr_sinks -= 1;
+
+ /* Swap the particle */
+ memswap(&s->sinks[k], &s->sinks[nr_sinks], sizeof(struct sink));
+
+ /* Swap the link with the gpart */
+ if (s->sinks[k].gpart != NULL) {
+ s->sinks[k].gpart->id_or_neg_offset = -k;
+ }
+ if (s->sinks[nr_sinks].gpart != NULL) {
+ s->sinks[nr_sinks].gpart->id_or_neg_offset = -nr_sinks;
+ }
+
+ /* Swap the index */
+ memswap(&sink_index[k], &sink_index[nr_sinks], sizeof(int));
+
+ } else {
+ /* Increment when not exchanging otherwise we need to retest "k".*/
+ k++;
+ }
+ }
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Check that all sinks are in the correct place. */
+ size_t check_count_inhibited_sinks = 0;
+ for (size_t k = 0; k < nr_sinks; k++) {
+ if (sink_index[k] == -1 ||
+ cells_top[sink_index[k]].nodeID != local_nodeID) {
+ error("Failed to move all non-local sinks to send list");
+ }
+ }
+ for (size_t k = nr_sinks; k < s->nr_sinks; k++) {
+ if (sink_index[k] != -1 &&
+ cells_top[sink_index[k]].nodeID == local_nodeID) {
+ error("Failed to remove local sinks from send list");
+ }
+ if (sink_index[k] == -1) ++check_count_inhibited_sinks;
+ }
+ if (check_count_inhibited_sinks != count_inhibited_sinks)
+ error("Counts of inhibited sink-particles do not match!");
+#endif /* SWIFT_DEBUG_CHECKS */
+
+ /* Move non-local gparts and inhibited parts to the end of the list. */
+ if (!repartitioned && (s->e->nr_nodes > 1 || count_inhibited_gparts > 0)) {
+
+ for (size_t k = 0; k < nr_gparts; /* void */) {
+
+ /* Inhibited particle or foreign particle */
+ if (g_index[k] == -1 || cells_top[g_index[k]].nodeID != local_nodeID) {
+
+ /* One fewer particle */
+ nr_gparts -= 1;
+
+ /* Swap the particle */
+ memswap_unaligned(&s->gparts[k], &s->gparts[nr_gparts],
+ sizeof(struct gpart));
+
+ /* Swap the link with part/spart */
+ if (s->gparts[k].type == swift_type_gas) {
+ s->parts[-s->gparts[k].id_or_neg_offset].gpart = &s->gparts[k];
+ } else if (s->gparts[k].type == swift_type_stars) {
+ s->sparts[-s->gparts[k].id_or_neg_offset].gpart = &s->gparts[k];
+ } else if (s->gparts[k].type == swift_type_sink) {
+ s->sparts[-s->gparts[k].id_or_neg_offset].gpart = &s->gparts[k];
+ } else if (s->gparts[k].type == swift_type_black_hole) {
+ s->bparts[-s->gparts[k].id_or_neg_offset].gpart = &s->gparts[k];
+ }
+
+ if (s->gparts[nr_gparts].type == swift_type_gas) {
+ s->parts[-s->gparts[nr_gparts].id_or_neg_offset].gpart =
+ &s->gparts[nr_gparts];
+ } else if (s->gparts[nr_gparts].type == swift_type_stars) {
+ s->sparts[-s->gparts[nr_gparts].id_or_neg_offset].gpart =
+ &s->gparts[nr_gparts];
+ } else if (s->gparts[nr_gparts].type == swift_type_sink) {
+ s->sparts[-s->gparts[nr_gparts].id_or_neg_offset].gpart =
+ &s->gparts[nr_gparts];
+ } else if (s->gparts[nr_gparts].type == swift_type_black_hole) {
+ s->bparts[-s->gparts[nr_gparts].id_or_neg_offset].gpart =
+ &s->gparts[nr_gparts];
+ }
+
+ /* Swap the index */
+ memswap(&g_index[k], &g_index[nr_gparts], sizeof(int));
+ } else {
+ /* Increment when not exchanging otherwise we need to retest "k".*/
+ k++;
+ }
+ }
+ }
+
+ if (verbose)
+ message("Moving non-local particles took %.3f %s.",
+ clocks_from_ticks(getticks() - tic2), clocks_getunit());
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Check that all gparts are in the correct place. */
+ size_t check_count_inhibited_gpart = 0;
+ for (size_t k = 0; k < nr_gparts; k++) {
+ if (g_index[k] == -1 || cells_top[g_index[k]].nodeID != local_nodeID) {
+ error("Failed to move all non-local gparts to send list");
+ }
+ }
+ for (size_t k = nr_gparts; k < s->nr_gparts; k++) {
+ if (g_index[k] != -1 && cells_top[g_index[k]].nodeID == local_nodeID) {
+ error("Failed to remove local gparts from send list");
+ }
+ if (g_index[k] == -1) ++check_count_inhibited_gpart;
+ }
+ if (check_count_inhibited_gpart != count_inhibited_gparts)
+ error("Counts of inhibited g-particles do not match!");
+#endif /* SWIFT_DEBUG_CHECKS */
+
+#ifdef WITH_MPI
+
+ /* Exchange the strays, note that this potentially re-allocates
+ the parts arrays. This can be skipped if we just repartitioned space
+ as there should be no strays in that case */
+ if (!repartitioned) {
+
+ size_t nr_parts_exchanged = s->nr_parts - nr_parts;
+ size_t nr_gparts_exchanged = s->nr_gparts - nr_gparts;
+ size_t nr_sparts_exchanged = s->nr_sparts - nr_sparts;
+ size_t nr_bparts_exchanged = s->nr_bparts - nr_bparts;
+ engine_exchange_strays(s->e, nr_parts, &h_index[nr_parts],
+ &nr_parts_exchanged, nr_gparts, &g_index[nr_gparts],
+ &nr_gparts_exchanged, nr_sparts, &s_index[nr_sparts],
+ &nr_sparts_exchanged, nr_bparts, &b_index[nr_bparts],
+ &nr_bparts_exchanged);
+
+ /* Set the new particle counts. */
+ s->nr_parts = nr_parts + nr_parts_exchanged;
+ s->nr_gparts = nr_gparts + nr_gparts_exchanged;
+ s->nr_sparts = nr_sparts + nr_sparts_exchanged;
+ s->nr_bparts = nr_bparts + nr_bparts_exchanged;
+
+ } else {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (s->nr_parts != nr_parts)
+ error("Number of parts changing after repartition");
+ if (s->nr_sparts != nr_sparts)
+ error("Number of sparts changing after repartition");
+ if (s->nr_gparts != nr_gparts)
+ error("Number of gparts changing after repartition");
+#endif
+ }
+
+ /* Clear non-local cell counts. */
+ for (int k = 0; k < s->nr_cells; k++) {
+ if (s->cells_top[k].nodeID != local_nodeID) {
+ cell_part_counts[k] = 0;
+ cell_spart_counts[k] = 0;
+ cell_gpart_counts[k] = 0;
+ cell_bpart_counts[k] = 0;
+ }
+ }
+
+ /* Re-allocate the index array for the parts if needed.. */
+ if (s->nr_parts + 1 > h_index_size) {
+ int *ind_new;
+ if ((ind_new = (int *)swift_malloc(
+ "h_index", sizeof(int) * (s->nr_parts + 1))) == NULL)
+ error("Failed to allocate temporary particle indices.");
+ memcpy(ind_new, h_index, sizeof(int) * nr_parts);
+ swift_free("h_index", h_index);
+ h_index = ind_new;
+ }
+
+ /* Re-allocate the index array for the sparts if needed.. */
+ if (s->nr_sparts + 1 > s_index_size) {
+ int *sind_new;
+ if ((sind_new = (int *)swift_malloc(
+ "s_index", sizeof(int) * (s->nr_sparts + 1))) == NULL)
+ error("Failed to allocate temporary s-particle indices.");
+ memcpy(sind_new, s_index, sizeof(int) * nr_sparts);
+ swift_free("s_index", s_index);
+ s_index = sind_new;
+ }
+
+ /* Re-allocate the index array for the bparts if needed.. */
+ if (s->nr_bparts + 1 > s_index_size) {
+ int *bind_new;
+ if ((bind_new = (int *)swift_malloc(
+ "b_index", sizeof(int) * (s->nr_bparts + 1))) == NULL)
+ error("Failed to allocate temporary s-particle indices.");
+ memcpy(bind_new, b_index, sizeof(int) * nr_bparts);
+ swift_free("b_index", b_index);
+ b_index = bind_new;
+ }
+
+ const int cdim[3] = {s->cdim[0], s->cdim[1], s->cdim[2]};
+ const double ih[3] = {s->iwidth[0], s->iwidth[1], s->iwidth[2]};
+
+ /* Assign each received part to its cell. */
+ for (size_t k = nr_parts; k < s->nr_parts; k++) {
+ const struct part *const p = &s->parts[k];
+ h_index[k] =
+ cell_getid(cdim, p->x[0] * ih[0], p->x[1] * ih[1], p->x[2] * ih[2]);
+ cell_part_counts[h_index[k]]++;
+#ifdef SWIFT_DEBUG_CHECKS
+ if (cells_top[h_index[k]].nodeID != local_nodeID)
+ error("Received part that does not belong to me (nodeID=%i).",
+ cells_top[h_index[k]].nodeID);
+#endif
+ }
+ nr_parts = s->nr_parts;
+
+ /* Assign each received spart to its cell. */
+ for (size_t k = nr_sparts; k < s->nr_sparts; k++) {
+ const struct spart *const sp = &s->sparts[k];
+ s_index[k] =
+ cell_getid(cdim, sp->x[0] * ih[0], sp->x[1] * ih[1], sp->x[2] * ih[2]);
+ cell_spart_counts[s_index[k]]++;
+#ifdef SWIFT_DEBUG_CHECKS
+ if (cells_top[s_index[k]].nodeID != local_nodeID)
+ error("Received s-part that does not belong to me (nodeID=%i).",
+ cells_top[s_index[k]].nodeID);
+#endif
+ }
+ nr_sparts = s->nr_sparts;
+
+ /* Assign each received bpart to its cell. */
+ for (size_t k = nr_bparts; k < s->nr_bparts; k++) {
+ const struct bpart *const bp = &s->bparts[k];
+ b_index[k] =
+ cell_getid(cdim, bp->x[0] * ih[0], bp->x[1] * ih[1], bp->x[2] * ih[2]);
+ cell_bpart_counts[b_index[k]]++;
+#ifdef SWIFT_DEBUG_CHECKS
+ if (cells_top[b_index[k]].nodeID != local_nodeID)
+ error("Received s-part that does not belong to me (nodeID=%i).",
+ cells_top[b_index[k]].nodeID);
+#endif
+ }
+ nr_bparts = s->nr_bparts;
+
+#else /* WITH_MPI */
+
+ /* Update the part, spart and bpart counters */
+ s->nr_parts = nr_parts;
+ s->nr_sparts = nr_sparts;
+ s->nr_bparts = nr_bparts;
+ s->nr_sinks = nr_sinks;
+
+#endif /* WITH_MPI */
+
+ /* Sort the parts according to their cells. */
+ if (nr_parts > 0)
+ space_parts_sort(s->parts, s->xparts, h_index, cell_part_counts,
+ s->nr_cells, 0);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Verify that the part have been sorted correctly. */
+ for (size_t k = 0; k < nr_parts; k++) {
+ const struct part *p = &s->parts[k];
+
+ if (p->time_bin == time_bin_inhibited)
+ error("Inhibited particle sorted into a cell!");
+
+ /* New cell index */
+ const int new_ind =
+ cell_getid(s->cdim, p->x[0] * s->iwidth[0], p->x[1] * s->iwidth[1],
+ p->x[2] * s->iwidth[2]);
+
+ /* New cell of this part */
+ const struct cell *c = &s->cells_top[new_ind];
+
+ if (h_index[k] != new_ind)
+ error("part's new cell index not matching sorted index.");
+
+ if (p->x[0] < c->loc[0] || p->x[0] > c->loc[0] + c->width[0] ||
+ p->x[1] < c->loc[1] || p->x[1] > c->loc[1] + c->width[1] ||
+ p->x[2] < c->loc[2] || p->x[2] > c->loc[2] + c->width[2])
+ error("part not sorted into the right top-level cell!");
+ }
+#endif /* SWIFT_DEBUG_CHECKS */
+
+ /* Sort the sparts according to their cells. */
+ if (nr_sparts > 0)
+ space_sparts_sort(s->sparts, s_index, cell_spart_counts, s->nr_cells, 0);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Verify that the spart have been sorted correctly. */
+ for (size_t k = 0; k < nr_sparts; k++) {
+ const struct spart *sp = &s->sparts[k];
+
+ if (sp->time_bin == time_bin_inhibited)
+ error("Inhibited particle sorted into a cell!");
+
+ /* New cell index */
+ const int new_sind =
+ cell_getid(s->cdim, sp->x[0] * s->iwidth[0], sp->x[1] * s->iwidth[1],
+ sp->x[2] * s->iwidth[2]);
+
+ /* New cell of this spart */
+ const struct cell *c = &s->cells_top[new_sind];
+
+ if (s_index[k] != new_sind)
+ error("spart's new cell index not matching sorted index.");
+
+ if (sp->x[0] < c->loc[0] || sp->x[0] > c->loc[0] + c->width[0] ||
+ sp->x[1] < c->loc[1] || sp->x[1] > c->loc[1] + c->width[1] ||
+ sp->x[2] < c->loc[2] || sp->x[2] > c->loc[2] + c->width[2])
+ error("spart not sorted into the right top-level cell!");
+ }
+#endif /* SWIFT_DEBUG_CHECKS */
+
+ /* Sort the bparts according to their cells. */
+ if (nr_bparts > 0)
+ space_bparts_sort(s->bparts, b_index, cell_bpart_counts, s->nr_cells, 0);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Verify that the bpart have been sorted correctly. */
+ for (size_t k = 0; k < nr_bparts; k++) {
+ const struct bpart *bp = &s->bparts[k];
+
+ if (bp->time_bin == time_bin_inhibited)
+ error("Inhibited particle sorted into a cell!");
+
+ /* New cell index */
+ const int new_bind =
+ cell_getid(s->cdim, bp->x[0] * s->iwidth[0], bp->x[1] * s->iwidth[1],
+ bp->x[2] * s->iwidth[2]);
+
+ /* New cell of this bpart */
+ const struct cell *c = &s->cells_top[new_bind];
+
+ if (b_index[k] != new_bind)
+ error("bpart's new cell index not matching sorted index.");
+
+ if (bp->x[0] < c->loc[0] || bp->x[0] > c->loc[0] + c->width[0] ||
+ bp->x[1] < c->loc[1] || bp->x[1] > c->loc[1] + c->width[1] ||
+ bp->x[2] < c->loc[2] || bp->x[2] > c->loc[2] + c->width[2])
+ error("bpart not sorted into the right top-level cell!");
+ }
+#endif /* SWIFT_DEBUG_CHECKS */
+
+ /* Sort the sink according to their cells. */
+ if (nr_sinks > 0)
+ space_sinks_sort(s->sinks, sink_index, cell_sink_counts, s->nr_cells, 0);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Verify that the sink have been sorted correctly. */
+ for (size_t k = 0; k < nr_sinks; k++) {
+ const struct sink *sink = &s->sinks[k];
+
+ if (sink->time_bin == time_bin_inhibited)
+ error("Inhibited particle sorted into a cell!");
+
+ /* New cell index */
+ const int new_bind =
+ cell_getid(s->cdim, sink->x[0] * s->iwidth[0],
+ sink->x[1] * s->iwidth[1], sink->x[2] * s->iwidth[2]);
+
+ /* New cell of this sink */
+ const struct cell *c = &s->cells_top[new_bind];
+
+ if (sink_index[k] != new_bind)
+ error("sink's new cell index not matching sorted index.");
+
+ if (sink->x[0] < c->loc[0] || sink->x[0] > c->loc[0] + c->width[0] ||
+ sink->x[1] < c->loc[1] || sink->x[1] > c->loc[1] + c->width[1] ||
+ sink->x[2] < c->loc[2] || sink->x[2] > c->loc[2] + c->width[2])
+ error("sink not sorted into the right top-level cell!");
+ }
+#endif /* SWIFT_DEBUG_CHECKS */
+
+ /* Extract the cell counts from the sorted indices. Deduct the extra
+ * particles. */
+ size_t last_index = 0;
+ h_index[nr_parts] = s->nr_cells; // sentinel.
+ for (size_t k = 0; k < nr_parts; k++) {
+ if (h_index[k] < h_index[k + 1]) {
+ cells_top[h_index[k]].hydro.count =
+ k - last_index + 1 - space_extra_parts;
+ last_index = k + 1;
+ }
+ }
+
+ /* Extract the cell counts from the sorted indices. Deduct the extra
+ * particles. */
+ size_t last_sindex = 0;
+ s_index[nr_sparts] = s->nr_cells; // sentinel.
+ for (size_t k = 0; k < nr_sparts; k++) {
+ if (s_index[k] < s_index[k + 1]) {
+ cells_top[s_index[k]].stars.count =
+ k - last_sindex + 1 - space_extra_sparts;
+ last_sindex = k + 1;
+ }
+ }
+
+ /* Extract the cell counts from the sorted indices. Deduct the extra
+ * particles. */
+ size_t last_bindex = 0;
+ b_index[nr_bparts] = s->nr_cells; // sentinel.
+ for (size_t k = 0; k < nr_bparts; k++) {
+ if (b_index[k] < b_index[k + 1]) {
+ cells_top[b_index[k]].black_holes.count =
+ k - last_bindex + 1 - space_extra_bparts;
+ last_bindex = k + 1;
+ }
+ }
+
+ /* Extract the cell counts from the sorted indices. Deduct the extra
+ * particles. */
+ size_t last_sink_index = 0;
+ sink_index[nr_sinks] = s->nr_cells; // sentinel.
+ for (size_t k = 0; k < nr_sinks; k++) {
+ if (sink_index[k] < sink_index[k + 1]) {
+ cells_top[sink_index[k]].sinks.count =
+ k - last_sink_index + 1 - space_extra_sinks;
+ last_sink_index = k + 1;
+ }
+ }
+
+ /* We no longer need the indices as of here. */
+ swift_free("h_index", h_index);
+ swift_free("cell_part_counts", cell_part_counts);
+ swift_free("s_index", s_index);
+ swift_free("cell_spart_counts", cell_spart_counts);
+ swift_free("b_index", b_index);
+ swift_free("cell_bpart_counts", cell_bpart_counts);
+ swift_free("sink_index", sink_index);
+ swift_free("cell_sink_counts", cell_sink_counts);
+
+ /* Update the slice of unique IDs. */
+ space_update_unique_id(s);
+
+#ifdef WITH_MPI
+
+ /* Re-allocate the index array for the gparts if needed.. */
+ if (s->nr_gparts + 1 > g_index_size) {
+ int *gind_new;
+ if ((gind_new = (int *)swift_malloc(
+ "g_index", sizeof(int) * (s->nr_gparts + 1))) == NULL)
+ error("Failed to allocate temporary g-particle indices.");
+ memcpy(gind_new, g_index, sizeof(int) * nr_gparts);
+ swift_free("g_index", g_index);
+ g_index = gind_new;
+ }
+
+ /* Assign each received gpart to its cell. */
+ for (size_t k = nr_gparts; k < s->nr_gparts; k++) {
+ const struct gpart *const p = &s->gparts[k];
+ g_index[k] =
+ cell_getid(cdim, p->x[0] * ih[0], p->x[1] * ih[1], p->x[2] * ih[2]);
+ cell_gpart_counts[g_index[k]]++;
+#ifdef SWIFT_DEBUG_CHECKS
+ if (cells_top[g_index[k]].nodeID != s->e->nodeID)
+ error("Received g-part that does not belong to me (nodeID=%i).",
+ cells_top[g_index[k]].nodeID);
+#endif
+ }
+ nr_gparts = s->nr_gparts;
+
+#else /* WITH_MPI */
+
+ /* Update the gpart counter */
+ s->nr_gparts = nr_gparts;
+
+#endif /* WITH_MPI */
+
+ /* Mark that there are no inhibited particles left */
+ s->nr_inhibited_parts = 0;
+ s->nr_inhibited_gparts = 0;
+ s->nr_inhibited_sparts = 0;
+ s->nr_inhibited_bparts = 0;
+ s->nr_inhibited_sinks = 0;
+
+ /* Sort the gparts according to their cells. */
+ if (nr_gparts > 0)
+ space_gparts_sort(s->gparts, s->parts, s->sinks, s->sparts, s->bparts,
+ g_index, cell_gpart_counts, s->nr_cells);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Verify that the gpart have been sorted correctly. */
+ for (size_t k = 0; k < nr_gparts; k++) {
+ const struct gpart *gp = &s->gparts[k];
+
+ if (gp->time_bin == time_bin_inhibited)
+ error("Inhibited particle sorted into a cell!");
+
+ /* New cell index */
+ const int new_gind =
+ cell_getid(s->cdim, gp->x[0] * s->iwidth[0], gp->x[1] * s->iwidth[1],
+ gp->x[2] * s->iwidth[2]);
+
+ /* New cell of this gpart */
+ const struct cell *c = &s->cells_top[new_gind];
+
+ if (g_index[k] != new_gind)
+ error("gpart's new cell index not matching sorted index.");
+
+ if (gp->x[0] < c->loc[0] || gp->x[0] > c->loc[0] + c->width[0] ||
+ gp->x[1] < c->loc[1] || gp->x[1] > c->loc[1] + c->width[1] ||
+ gp->x[2] < c->loc[2] || gp->x[2] > c->loc[2] + c->width[2])
+ error("gpart not sorted into the right top-level cell!");
+ }
+#endif /* SWIFT_DEBUG_CHECKS */
+
+ /* Extract the cell counts from the sorted indices. Deduct the extra
+ * particles. */
+ size_t last_gindex = 0;
+ g_index[nr_gparts] = s->nr_cells;
+ for (size_t k = 0; k < nr_gparts; k++) {
+ if (g_index[k] < g_index[k + 1]) {
+ cells_top[g_index[k]].grav.count =
+ k - last_gindex + 1 - space_extra_gparts;
+ last_gindex = k + 1;
+ }
+ }
+
+ /* We no longer need the indices as of here. */
+ swift_free("g_index", g_index);
+ swift_free("cell_gpart_counts", cell_gpart_counts);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Verify that the links are correct */
+ if ((nr_gparts > 0 && nr_parts > 0) || (nr_gparts > 0 && nr_sparts > 0) ||
+ (nr_gparts > 0 && nr_bparts > 0) || (nr_gparts > 0 && nr_sinks > 0))
+ part_verify_links(s->parts, s->gparts, s->sinks, s->sparts, s->bparts,
+ nr_parts, nr_gparts, nr_sinks, nr_sparts, nr_bparts,
+ verbose);
+#endif
+
+ /* Hook the cells up to the parts. Make list of local and non-empty cells */
+ const ticks tic3 = getticks();
+ struct part *finger = s->parts;
+ struct xpart *xfinger = s->xparts;
+ struct gpart *gfinger = s->gparts;
+ struct spart *sfinger = s->sparts;
+ struct bpart *bfinger = s->bparts;
+ struct sink *sink_finger = s->sinks;
+ s->nr_cells_with_particles = 0;
+ s->nr_local_cells_with_particles = 0;
+ s->nr_local_cells = 0;
+ for (int k = 0; k < s->nr_cells; k++) {
+ struct cell *restrict c = &cells_top[k];
+ c->hydro.ti_old_part = ti_current;
+ c->grav.ti_old_part = ti_current;
+ c->grav.ti_old_multipole = ti_current;
+ c->stars.ti_old_part = ti_current;
+ c->sinks.ti_old_part = ti_current;
+ c->black_holes.ti_old_part = ti_current;
+
+#if defined(SWIFT_DEBUG_CHECKS) || defined(SWIFT_CELL_GRAPH)
+ c->cellID = -last_cell_id;
+ last_cell_id++;
+#endif
+
+ const int is_local = (c->nodeID == engine_rank);
+ const int has_particles =
+ (c->hydro.count > 0) || (c->grav.count > 0) || (c->stars.count > 0) ||
+ (c->black_holes.count > 0) || (c->sinks.count > 0);
+
+ if (is_local) {
+ c->hydro.parts = finger;
+ c->hydro.xparts = xfinger;
+ c->grav.parts = gfinger;
+ c->stars.parts = sfinger;
+ c->black_holes.parts = bfinger;
+ c->sinks.parts = sink_finger;
+
+ /* Store the state at rebuild time */
+ c->stars.parts_rebuild = c->stars.parts;
+ c->grav.parts_rebuild = c->grav.parts;
+
+ c->hydro.count_total = c->hydro.count + space_extra_parts;
+ c->grav.count_total = c->grav.count + space_extra_gparts;
+ c->stars.count_total = c->stars.count + space_extra_sparts;
+ c->sinks.count_total = c->sinks.count + space_extra_sinks;
+ c->black_holes.count_total = c->black_holes.count + space_extra_bparts;
+
+ finger = &finger[c->hydro.count_total];
+ xfinger = &xfinger[c->hydro.count_total];
+ gfinger = &gfinger[c->grav.count_total];
+ sfinger = &sfinger[c->stars.count_total];
+ bfinger = &bfinger[c->black_holes.count_total];
+ sink_finger = &sink_finger[c->sinks.count_total];
+
+ /* Add this cell to the list of local cells */
+ s->local_cells_top[s->nr_local_cells] = k;
+ s->nr_local_cells++;
+ }
+
+ if (is_local && has_particles) {
+
+ /* Add this cell to the list of non-empty cells */
+ s->local_cells_with_particles_top[s->nr_local_cells_with_particles] = k;
+ s->nr_local_cells_with_particles++;
+ }
+ }
+ if (verbose) {
+ message("Have %d local top-level cells with particles (total=%d)",
+ s->nr_local_cells_with_particles, s->nr_cells);
+ message("Have %d local top-level cells (total=%d)", s->nr_local_cells,
+ s->nr_cells);
+ message("hooking up cells took %.3f %s.",
+ clocks_from_ticks(getticks() - tic3), clocks_getunit());
+ }
+
+ /* Re-order the extra particles such that they are at the end of their cell's
+ memory pool. */
+ if (s->with_star_formation || s->e->policy & engine_policy_sinks)
+ space_reorder_extras(s, verbose);
+
+ /* At this point, we have the upper-level cells. Now recursively split each
+ cell to get the full AMR grid. */
+ space_split(s, verbose);
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Check that the multipole construction went OK */
+ if (s->with_self_gravity)
+ for (int k = 0; k < s->nr_cells; k++)
+ cell_check_multipole(&s->cells_top[k], s->e->gravity_properties);
+#endif
+
+ /* Clean up any stray sort indices in the cell buffer. */
+ space_free_buff_sort_indices(s);
+
+ if (verbose)
+ message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+ clocks_getunit());
+}
diff --git a/src/space_recycle.c b/src/space_recycle.c
new file mode 100644
index 0000000000000000000000000000000000000000..f98e83d158e1eb572efd5f6f9cde235bb56a31d8
--- /dev/null
+++ b/src/space_recycle.c
@@ -0,0 +1,326 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* This object's header. */
+#include "space.h"
+
+/* Local headers. */
+#include "cell.h"
+#include "engine.h"
+#include "star_formation_logger.h"
+#include "threadpool.h"
+
+/**
+ * @brief Recursively dismantle a cell tree.
+ *
+ * @param s The #space.
+ * @param c The #cell to recycle.
+ * @param cell_rec_begin Pointer to the start of the list of cells to recycle.
+ * @param cell_rec_end Pointer to the end of the list of cells to recycle.
+ * @param multipole_rec_begin Pointer to the start of the list of multipoles to
+ * recycle.
+ * @param multipole_rec_end Pointer to the end of the list of multipoles to
+ * recycle.
+ */
+void space_rebuild_recycle_rec(struct space *s, struct cell *c,
+ struct cell **cell_rec_begin,
+ struct cell **cell_rec_end,
+ struct gravity_tensors **multipole_rec_begin,
+ struct gravity_tensors **multipole_rec_end) {
+ if (c->split)
+ for (int k = 0; k < 8; k++)
+ if (c->progeny[k] != NULL) {
+ space_rebuild_recycle_rec(s, c->progeny[k], cell_rec_begin,
+ cell_rec_end, multipole_rec_begin,
+ multipole_rec_end);
+
+ c->progeny[k]->next = *cell_rec_begin;
+ *cell_rec_begin = c->progeny[k];
+
+ if (s->with_self_gravity) {
+ c->progeny[k]->grav.multipole->next = *multipole_rec_begin;
+ *multipole_rec_begin = c->progeny[k]->grav.multipole;
+ }
+
+ if (*cell_rec_end == NULL) *cell_rec_end = *cell_rec_begin;
+ if (s->with_self_gravity && *multipole_rec_end == NULL)
+ *multipole_rec_end = *multipole_rec_begin;
+
+ c->progeny[k]->grav.multipole = NULL;
+ c->progeny[k] = NULL;
+ }
+}
+
+void space_rebuild_recycle_mapper(void *map_data, int num_elements,
+ void *extra_data) {
+
+ struct space *s = (struct space *)extra_data;
+ struct cell *cells = (struct cell *)map_data;
+
+ for (int k = 0; k < num_elements; k++) {
+ struct cell *c = &cells[k];
+ struct cell *cell_rec_begin = NULL, *cell_rec_end = NULL;
+ struct gravity_tensors *multipole_rec_begin = NULL,
+ *multipole_rec_end = NULL;
+ space_rebuild_recycle_rec(s, c, &cell_rec_begin, &cell_rec_end,
+ &multipole_rec_begin, &multipole_rec_end);
+ if (cell_rec_begin != NULL)
+ space_recycle_list(s, cell_rec_begin, cell_rec_end, multipole_rec_begin,
+ multipole_rec_end);
+ c->hydro.sorts = NULL;
+ c->stars.sorts = NULL;
+ c->nr_tasks = 0;
+ c->grav.nr_mm_tasks = 0;
+ c->hydro.density = NULL;
+ c->hydro.gradient = NULL;
+ c->hydro.force = NULL;
+ c->hydro.limiter = NULL;
+ c->grav.grav = NULL;
+ c->grav.mm = NULL;
+ c->hydro.dx_max_part = 0.0f;
+ c->hydro.dx_max_sort = 0.0f;
+ c->sinks.dx_max_part = 0.f;
+ c->stars.dx_max_part = 0.f;
+ c->stars.dx_max_sort = 0.f;
+ c->black_holes.dx_max_part = 0.f;
+ c->hydro.sorted = 0;
+ c->hydro.sort_allocated = 0;
+ c->stars.sorted = 0;
+ c->hydro.count = 0;
+ c->hydro.count_total = 0;
+ c->hydro.updated = 0;
+ c->grav.count = 0;
+ c->grav.count_total = 0;
+ c->grav.updated = 0;
+ c->sinks.count = 0;
+ c->stars.count = 0;
+ c->stars.count_total = 0;
+ c->stars.updated = 0;
+ c->black_holes.count = 0;
+ c->black_holes.count_total = 0;
+ c->black_holes.updated = 0;
+ c->grav.init = NULL;
+ c->grav.init_out = NULL;
+ c->hydro.extra_ghost = NULL;
+ c->hydro.ghost_in = NULL;
+ c->hydro.ghost_out = NULL;
+ c->hydro.ghost = NULL;
+ c->hydro.sink_formation = NULL;
+ c->hydro.star_formation = NULL;
+ c->hydro.stars_resort = NULL;
+ c->stars.ghost = NULL;
+ c->stars.density = NULL;
+ c->stars.feedback = NULL;
+ c->black_holes.density_ghost = NULL;
+ c->black_holes.swallow_ghost[0] = NULL;
+ c->black_holes.swallow_ghost[1] = NULL;
+ c->black_holes.swallow_ghost[2] = NULL;
+ c->black_holes.density = NULL;
+ c->black_holes.swallow = NULL;
+ c->black_holes.do_gas_swallow = NULL;
+ c->black_holes.do_bh_swallow = NULL;
+ c->black_holes.feedback = NULL;
+ c->kick1 = NULL;
+ c->kick2 = NULL;
+ c->timestep = NULL;
+ c->timestep_limiter = NULL;
+ c->timestep_sync = NULL;
+ c->hydro.end_force = NULL;
+ c->hydro.drift = NULL;
+ c->sinks.drift = NULL;
+ c->stars.drift = NULL;
+ c->stars.stars_in = NULL;
+ c->stars.stars_out = NULL;
+ c->black_holes.drift = NULL;
+ c->black_holes.black_holes_in = NULL;
+ c->black_holes.black_holes_out = NULL;
+ c->sinks.sink_in = NULL;
+ c->sinks.sink_out = NULL;
+ c->grav.drift = NULL;
+ c->grav.drift_out = NULL;
+ c->hydro.cooling_in = NULL;
+ c->hydro.cooling_out = NULL;
+ c->hydro.cooling = NULL;
+ c->grav.long_range = NULL;
+ c->grav.down_in = NULL;
+ c->grav.down = NULL;
+ c->grav.end_force = NULL;
+ c->top = c;
+ c->super = c;
+ c->hydro.super = c;
+ c->grav.super = c;
+ c->hydro.parts = NULL;
+ c->hydro.xparts = NULL;
+ c->grav.parts = NULL;
+ c->grav.parts_rebuild = NULL;
+ c->sinks.parts = NULL;
+ c->stars.parts = NULL;
+ c->stars.parts_rebuild = NULL;
+ c->black_holes.parts = NULL;
+ c->flags = 0;
+ c->hydro.ti_end_min = -1;
+ c->hydro.ti_end_max = -1;
+ c->grav.ti_end_min = -1;
+ c->grav.ti_end_max = -1;
+ c->sinks.ti_end_min = -1;
+ c->sinks.ti_end_max = -1;
+ c->stars.ti_end_min = -1;
+ c->stars.ti_end_max = -1;
+ c->black_holes.ti_end_min = -1;
+ c->black_holes.ti_end_max = -1;
+ c->hydro.rt_inject = NULL;
+ c->hydro.rt_in = NULL;
+ c->hydro.rt_out = NULL;
+ c->hydro.rt_ghost1 = NULL;
+ star_formation_logger_init(&c->stars.sfh);
+#if defined(SWIFT_DEBUG_CHECKS) || defined(SWIFT_CELL_GRAPH)
+ c->cellID = 0;
+#endif
+ if (s->with_self_gravity)
+ bzero(c->grav.multipole, sizeof(struct gravity_tensors));
+
+ cell_free_hydro_sorts(c);
+ cell_free_stars_sorts(c);
+#if WITH_MPI
+ c->mpi.tag = -1;
+ c->mpi.recv = NULL;
+ c->mpi.send = NULL;
+#endif
+ }
+}
+
+/**
+ * @brief Return a used cell to the buffer of unused sub-cells.
+ *
+ * @param s The #space.
+ * @param c The #cell.
+ */
+void space_recycle(struct space *s, struct cell *c) {
+
+ /* Clear the cell. */
+ if (lock_destroy(&c->hydro.lock) != 0 || lock_destroy(&c->grav.plock) != 0 ||
+ lock_destroy(&c->grav.mlock) != 0 || lock_destroy(&c->stars.lock) != 0 ||
+ lock_destroy(&c->sinks.lock) != 0 ||
+ lock_destroy(&c->sinks.sink_formation_lock) != 0 ||
+ lock_destroy(&c->black_holes.lock) != 0 ||
+ lock_destroy(&c->grav.star_formation_lock) != 0 ||
+ lock_destroy(&c->stars.star_formation_lock) != 0)
+ error("Failed to destroy spinlocks.");
+
+ /* Lock the space. */
+ lock_lock(&s->lock);
+
+ /* Hook the multipole back in the buffer */
+ if (s->with_self_gravity) {
+ c->grav.multipole->next = s->multipoles_sub;
+ s->multipoles_sub = c->grav.multipole;
+ }
+
+ /* Hook this cell into the buffer. */
+ c->next = s->cells_sub;
+ s->cells_sub = c;
+ s->tot_cells -= 1;
+
+ /* Unlock the space. */
+ lock_unlock_blind(&s->lock);
+}
+
+/**
+ * @brief Return a list of used cells to the buffer of unused sub-cells.
+ *
+ * @param s The #space.
+ * @param cell_list_begin Pointer to the first #cell in the linked list of
+ * cells joined by their @c next pointers.
+ * @param cell_list_end Pointer to the last #cell in the linked list of
+ * cells joined by their @c next pointers. It is assumed that this
+ * cell's @c next pointer is @c NULL.
+ * @param multipole_list_begin Pointer to the first #multipole in the linked
+ * list of
+ * multipoles joined by their @c next pointers.
+ * @param multipole_list_end Pointer to the last #multipole in the linked list
+ * of
+ * multipoles joined by their @c next pointers. It is assumed that this
+ * multipole's @c next pointer is @c NULL.
+ */
+void space_recycle_list(struct space *s, struct cell *cell_list_begin,
+ struct cell *cell_list_end,
+ struct gravity_tensors *multipole_list_begin,
+ struct gravity_tensors *multipole_list_end) {
+
+ int count = 0;
+
+ /* Clean up the list of cells. */
+ for (struct cell *c = cell_list_begin; c != NULL; c = c->next) {
+ /* Clear the cell. */
+ if (lock_destroy(&c->hydro.lock) != 0 ||
+ lock_destroy(&c->grav.plock) != 0 ||
+ lock_destroy(&c->grav.mlock) != 0 ||
+ lock_destroy(&c->stars.lock) != 0 ||
+ lock_destroy(&c->sinks.lock) != 0 ||
+ lock_destroy(&c->sinks.sink_formation_lock) != 0 ||
+ lock_destroy(&c->black_holes.lock) != 0 ||
+ lock_destroy(&c->stars.star_formation_lock) != 0 ||
+ lock_destroy(&c->grav.star_formation_lock) != 0)
+ error("Failed to destroy spinlocks.");
+
+ /* Count this cell. */
+ count += 1;
+ }
+
+ /* Lock the space. */
+ lock_lock(&s->lock);
+
+ /* Hook the cells into the buffer. */
+ cell_list_end->next = s->cells_sub;
+ s->cells_sub = cell_list_begin;
+ s->tot_cells -= count;
+
+ /* Hook the multipoles into the buffer. */
+ if (s->with_self_gravity) {
+ multipole_list_end->next = s->multipoles_sub;
+ s->multipoles_sub = multipole_list_begin;
+ }
+
+ /* Unlock the space. */
+ lock_unlock_blind(&s->lock);
+}
+
+/**
+ * @brief Free up any allocated cells.
+ *
+ * @param s The #space.
+ */
+void space_free_cells(struct space *s) {
+
+ ticks tic = getticks();
+
+ threadpool_map(&s->e->threadpool, space_rebuild_recycle_mapper, s->cells_top,
+ s->nr_cells, sizeof(struct cell), threadpool_auto_chunk_size,
+ s);
+ s->maxdepth = 0;
+
+ if (s->e->verbose)
+ message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+ clocks_getunit());
+}
diff --git a/src/space_regrid.c b/src/space_regrid.c
new file mode 100644
index 0000000000000000000000000000000000000000..b20a3b4894bf3ce6418cd288f74fe91cedc36ab7
--- /dev/null
+++ b/src/space_regrid.c
@@ -0,0 +1,397 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* This object's header. */
+#include "space.h"
+
+/* Local headers. */
+#include "cell.h"
+#include "engine.h"
+#include "scheduler.h"
+
+/*! Counter for cell IDs (when debugging) */
+#if defined(SWIFT_DEBUG_CHECKS) || defined(SWIFT_CELL_GRAPH)
+extern int last_cell_id;
+#endif
+
+/**
+ * @brief Re-build the top-level cell grid.
+ *
+ * @param s The #space.
+ * @param verbose Print messages to stdout or not.
+ */
+void space_regrid(struct space *s, int verbose) {
+
+ const size_t nr_parts = s->nr_parts;
+ const size_t nr_sparts = s->nr_sparts;
+ const size_t nr_bparts = s->nr_bparts;
+ const size_t nr_sinks = s->nr_sinks;
+ const ticks tic = getticks();
+ const integertime_t ti_current = (s->e != NULL) ? s->e->ti_current : 0;
+
+ /* Run through the cells and get the current h_max. */
+ // tic = getticks();
+ float h_max = s->cell_min / kernel_gamma / space_stretch;
+ if (nr_parts > 0) {
+
+ /* Can we use the list of local non-empty top-level cells? */
+ if (s->local_cells_with_particles_top != NULL) {
+ for (int k = 0; k < s->nr_local_cells_with_particles; ++k) {
+ const struct cell *c =
+ &s->cells_top[s->local_cells_with_particles_top[k]];
+ if (c->hydro.h_max > h_max) {
+ h_max = c->hydro.h_max;
+ }
+ if (c->stars.h_max > h_max) {
+ h_max = c->stars.h_max;
+ }
+ if (c->black_holes.h_max > h_max) {
+ h_max = c->black_holes.h_max;
+ }
+ if (c->sinks.r_cut_max > h_max) {
+ h_max = c->sinks.r_cut_max / kernel_gamma;
+ }
+ }
+
+ /* Can we instead use all the top-level cells? */
+ } else if (s->cells_top != NULL) {
+ for (int k = 0; k < s->nr_cells; k++) {
+ const struct cell *c = &s->cells_top[k];
+ if (c->nodeID == engine_rank && c->hydro.h_max > h_max) {
+ h_max = c->hydro.h_max;
+ }
+ if (c->nodeID == engine_rank && c->stars.h_max > h_max) {
+ h_max = c->stars.h_max;
+ }
+ if (c->nodeID == engine_rank && c->black_holes.h_max > h_max) {
+ h_max = c->black_holes.h_max;
+ }
+ if (c->nodeID == engine_rank && c->sinks.r_cut_max > h_max) {
+ h_max = c->sinks.r_cut_max / kernel_gamma;
+ }
+ }
+
+ /* Last option: run through the particles */
+ } else {
+ for (size_t k = 0; k < nr_parts; k++) {
+ if (s->parts[k].h > h_max) h_max = s->parts[k].h;
+ }
+ for (size_t k = 0; k < nr_sparts; k++) {
+ if (s->sparts[k].h > h_max) h_max = s->sparts[k].h;
+ }
+ for (size_t k = 0; k < nr_bparts; k++) {
+ if (s->bparts[k].h > h_max) h_max = s->bparts[k].h;
+ }
+ for (size_t k = 0; k < nr_sinks; k++) {
+ if (s->sinks[k].r_cut > h_max) h_max = s->sinks[k].r_cut / kernel_gamma;
+ }
+ }
+ }
+
+/* If we are running in parallel, make sure everybody agrees on
+ how large the largest cell should be. */
+#ifdef WITH_MPI
+ {
+ float buff;
+ if (MPI_Allreduce(&h_max, &buff, 1, MPI_FLOAT, MPI_MAX, MPI_COMM_WORLD) !=
+ MPI_SUCCESS)
+ error("Failed to aggregate the rebuild flag across nodes.");
+ h_max = buff;
+ }
+#endif
+ if (verbose) message("h_max is %.3e (cell_min=%.3e).", h_max, s->cell_min);
+
+ /* Get the new putative cell dimensions. */
+ const int cdim[3] = {
+ (int)floor(s->dim[0] /
+ fmax(h_max * kernel_gamma * space_stretch, s->cell_min)),
+ (int)floor(s->dim[1] /
+ fmax(h_max * kernel_gamma * space_stretch, s->cell_min)),
+ (int)floor(s->dim[2] /
+ fmax(h_max * kernel_gamma * space_stretch, s->cell_min))};
+
+ /* Check if we have enough cells for periodicity. */
+ if (s->periodic && (cdim[0] < 3 || cdim[1] < 3 || cdim[2] < 3))
+ error(
+ "Must have at least 3 cells in each spatial dimension when periodicity "
+ "is switched on.\nThis error is often caused by any of the "
+ "followings:\n"
+ " - too few particles to generate a sensible grid,\n"
+ " - the initial value of 'Scheduler:max_top_level_cells' is too "
+ "small,\n"
+ " - the (minimal) time-step is too large leading to particles with "
+ "predicted smoothing lengths too large for the box size,\n"
+ " - particles with velocities so large that they move by more than two "
+ "box sizes per time-step.\n");
+
+/* In MPI-Land, changing the top-level cell size requires that the
+ * global partition is recomputed and the particles redistributed.
+ * Be prepared to do that. */
+#ifdef WITH_MPI
+ double oldwidth[3];
+ double oldcdim[3];
+ int *oldnodeIDs = NULL;
+ if (cdim[0] < s->cdim[0] || cdim[1] < s->cdim[1] || cdim[2] < s->cdim[2]) {
+
+ /* Capture state of current space. */
+ oldcdim[0] = s->cdim[0];
+ oldcdim[1] = s->cdim[1];
+ oldcdim[2] = s->cdim[2];
+ oldwidth[0] = s->width[0];
+ oldwidth[1] = s->width[1];
+ oldwidth[2] = s->width[2];
+
+ if ((oldnodeIDs =
+ (int *)swift_malloc("nodeIDs", sizeof(int) * s->nr_cells)) == NULL)
+ error("Failed to allocate temporary nodeIDs.");
+
+ int cid = 0;
+ for (int i = 0; i < s->cdim[0]; i++) {
+ for (int j = 0; j < s->cdim[1]; j++) {
+ for (int k = 0; k < s->cdim[2]; k++) {
+ cid = cell_getid(oldcdim, i, j, k);
+ oldnodeIDs[cid] = s->cells_top[cid].nodeID;
+ }
+ }
+ }
+ }
+
+ /* Are we about to allocate new top level cells without a regrid?
+ * Can happen when restarting the application. */
+ const int no_regrid = (s->cells_top == NULL && oldnodeIDs == NULL);
+#endif
+
+ /* Do we need to re-build the upper-level cells? */
+ // tic = getticks();
+ if (s->cells_top == NULL || cdim[0] < s->cdim[0] || cdim[1] < s->cdim[1] ||
+ cdim[2] < s->cdim[2]) {
+
+/* Be verbose about this. */
+#ifdef SWIFT_DEBUG_CHECKS
+ message("(re)griding space cdim=(%d %d %d)", cdim[0], cdim[1], cdim[2]);
+ fflush(stdout);
+#endif
+
+ /* Free the old cells, if they were allocated. */
+ if (s->cells_top != NULL) {
+ space_free_cells(s);
+ swift_free("local_cells_with_tasks_top", s->local_cells_with_tasks_top);
+ swift_free("local_cells_top", s->local_cells_top);
+ swift_free("cells_with_particles_top", s->cells_with_particles_top);
+ swift_free("local_cells_with_particles_top",
+ s->local_cells_with_particles_top);
+ swift_free("cells_top", s->cells_top);
+ swift_free("multipoles_top", s->multipoles_top);
+ }
+
+ /* Also free the task arrays, these will be regenerated and we can use the
+ * memory while copying the particle arrays. */
+ if (s->e != NULL) scheduler_free_tasks(&s->e->sched);
+
+ /* Set the new cell dimensions only if smaller. */
+ for (int k = 0; k < 3; k++) {
+ s->cdim[k] = cdim[k];
+ s->width[k] = s->dim[k] / cdim[k];
+ s->iwidth[k] = 1.0 / s->width[k];
+ }
+ const float dmin = min3(s->width[0], s->width[1], s->width[2]);
+
+ /* Allocate the highest level of cells. */
+ s->tot_cells = s->nr_cells = cdim[0] * cdim[1] * cdim[2];
+
+ if (swift_memalign("cells_top", (void **)&s->cells_top, cell_align,
+ s->nr_cells * sizeof(struct cell)) != 0)
+ error("Failed to allocate top-level cells.");
+ bzero(s->cells_top, s->nr_cells * sizeof(struct cell));
+
+ /* Allocate the multipoles for the top-level cells. */
+ if (s->with_self_gravity) {
+ if (swift_memalign("multipoles_top", (void **)&s->multipoles_top,
+ multipole_align,
+ s->nr_cells * sizeof(struct gravity_tensors)) != 0)
+ error("Failed to allocate top-level multipoles.");
+ bzero(s->multipoles_top, s->nr_cells * sizeof(struct gravity_tensors));
+ }
+
+ /* Allocate the indices of local cells */
+ if (swift_memalign("local_cells_top", (void **)&s->local_cells_top,
+ SWIFT_STRUCT_ALIGNMENT, s->nr_cells * sizeof(int)) != 0)
+ error("Failed to allocate indices of local top-level cells.");
+ bzero(s->local_cells_top, s->nr_cells * sizeof(int));
+
+ /* Allocate the indices of local cells with tasks */
+ if (swift_memalign("local_cells_with_tasks_top",
+ (void **)&s->local_cells_with_tasks_top,
+ SWIFT_STRUCT_ALIGNMENT, s->nr_cells * sizeof(int)) != 0)
+ error("Failed to allocate indices of local top-level cells with tasks.");
+ bzero(s->local_cells_with_tasks_top, s->nr_cells * sizeof(int));
+
+ /* Allocate the indices of cells with particles */
+ if (swift_memalign("cells_with_particles_top",
+ (void **)&s->cells_with_particles_top,
+ SWIFT_STRUCT_ALIGNMENT, s->nr_cells * sizeof(int)) != 0)
+ error("Failed to allocate indices of top-level cells with particles.");
+ bzero(s->cells_with_particles_top, s->nr_cells * sizeof(int));
+
+ /* Allocate the indices of local cells with particles */
+ if (swift_memalign("local_cells_with_particles_top",
+ (void **)&s->local_cells_with_particles_top,
+ SWIFT_STRUCT_ALIGNMENT, s->nr_cells * sizeof(int)) != 0)
+ error(
+ "Failed to allocate indices of local top-level cells with "
+ "particles.");
+ bzero(s->local_cells_with_particles_top, s->nr_cells * sizeof(int));
+
+ /* Set the cells' locks */
+ for (int k = 0; k < s->nr_cells; k++) {
+ if (lock_init(&s->cells_top[k].hydro.lock) != 0)
+ error("Failed to init spinlock for hydro.");
+ if (lock_init(&s->cells_top[k].grav.plock) != 0)
+ error("Failed to init spinlock for gravity.");
+ if (lock_init(&s->cells_top[k].grav.mlock) != 0)
+ error("Failed to init spinlock for multipoles.");
+ if (lock_init(&s->cells_top[k].grav.star_formation_lock) != 0)
+ error("Failed to init spinlock for star formation (gpart).");
+ if (lock_init(&s->cells_top[k].stars.lock) != 0)
+ error("Failed to init spinlock for stars.");
+ if (lock_init(&s->cells_top[k].sinks.lock) != 0)
+ error("Failed to init spinlock for sinks.");
+ if (lock_init(&s->cells_top[k].sinks.sink_formation_lock) != 0)
+ error("Failed to init spinlock for sink formation.");
+ if (lock_init(&s->cells_top[k].black_holes.lock) != 0)
+ error("Failed to init spinlock for black holes.");
+ if (lock_init(&s->cells_top[k].stars.star_formation_lock) != 0)
+ error("Failed to init spinlock for star formation (spart).");
+ }
+
+ /* Set the cell location and sizes. */
+ for (int i = 0; i < cdim[0]; i++)
+ for (int j = 0; j < cdim[1]; j++)
+ for (int k = 0; k < cdim[2]; k++) {
+ const size_t cid = cell_getid(cdim, i, j, k);
+ struct cell *restrict c = &s->cells_top[cid];
+ c->loc[0] = i * s->width[0];
+ c->loc[1] = j * s->width[1];
+ c->loc[2] = k * s->width[2];
+ c->width[0] = s->width[0];
+ c->width[1] = s->width[1];
+ c->width[2] = s->width[2];
+ c->dmin = dmin;
+ c->depth = 0;
+ c->split = 0;
+ c->hydro.count = 0;
+ c->grav.count = 0;
+ c->stars.count = 0;
+ c->sinks.count = 0;
+ c->top = c;
+ c->super = c;
+ c->hydro.super = c;
+ c->grav.super = c;
+ c->hydro.ti_old_part = ti_current;
+ c->grav.ti_old_part = ti_current;
+ c->stars.ti_old_part = ti_current;
+ c->sinks.ti_old_part = ti_current;
+ c->black_holes.ti_old_part = ti_current;
+ c->grav.ti_old_multipole = ti_current;
+#ifdef WITH_MPI
+ c->mpi.tag = -1;
+ c->mpi.recv = NULL;
+ c->mpi.send = NULL;
+#endif // WITH_MPI
+ if (s->with_self_gravity) c->grav.multipole = &s->multipoles_top[cid];
+#if defined(SWIFT_DEBUG_CHECKS) || defined(SWIFT_CELL_GRAPH)
+ c->cellID = -last_cell_id;
+ last_cell_id++;
+#endif
+ }
+
+ /* Be verbose about the change. */
+ if (verbose)
+ message("set cell dimensions to [ %i %i %i ].", cdim[0], cdim[1],
+ cdim[2]);
+
+#ifdef WITH_MPI
+ if (oldnodeIDs != NULL) {
+ /* We have changed the top-level cell dimension, so need to redistribute
+ * cells around the nodes. We repartition using the old space node
+ * positions as a grid to resample. */
+ if (s->e->nodeID == 0)
+ message(
+ "basic cell dimensions have increased - recalculating the "
+ "global partition.");
+
+ if (!partition_space_to_space(oldwidth, oldcdim, oldnodeIDs, s)) {
+
+ /* Failed, try another technique that requires no settings. */
+ message("Failed to get a new partition, trying less optimal method");
+ struct partition initial_partition;
+#if defined(HAVE_PARMETIS) || defined(HAVE_METIS)
+ initial_partition.type = INITPART_METIS_NOWEIGHT;
+#else
+ initial_partition.type = INITPART_VECTORIZE;
+#endif
+ partition_initial_partition(&initial_partition, s->e->nodeID,
+ s->e->nr_nodes, s);
+ }
+
+ /* Re-distribute the particles to their new nodes. */
+ engine_redistribute(s->e);
+
+ /* Make the proxies. */
+ engine_makeproxies(s->e);
+
+ /* Finished with these. */
+ swift_free("nodeIDs", oldnodeIDs);
+
+ } else if (no_regrid && s->e != NULL) {
+ /* If we have created the top-levels cells and not done an initial
+ * partition (can happen when restarting), then the top-level cells
+ * are not assigned to a node, we must do that and then associate the
+ * particles with the cells. Note requires that
+ * partition_store_celllist() was called once before, or just before
+ * dumping the restart files.*/
+ partition_restore_celllist(s, s->e->reparttype);
+
+ /* Now re-distribute the particles, should just add to cells? */
+ engine_redistribute(s->e);
+
+ /* Make the proxies. */
+ engine_makeproxies(s->e);
+ }
+#endif /* WITH_MPI */
+
+ // message( "rebuilding upper-level cells took %.3f %s." ,
+ // clocks_from_ticks(double)(getticks() - tic), clocks_getunit());
+
+ } /* re-build upper-level cells? */
+ else { /* Otherwise, just clean up the cells. */
+
+ /* Free the old cells, if they were allocated. */
+ space_free_cells(s);
+ }
+
+ if (verbose)
+ message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+ clocks_getunit());
+}
diff --git a/src/space_sort.c b/src/space_sort.c
new file mode 100644
index 0000000000000000000000000000000000000000..562f706182f306e48127f94cf134f4e0389ea5f1
--- /dev/null
+++ b/src/space_sort.c
@@ -0,0 +1,353 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* This object's header. */
+#include "error.h"
+#include "memswap.h"
+#include "memuse.h"
+#include "space.h"
+
+/**
+ * @brief Sort the particles and condensed particles according to the given
+ * indices.
+ *
+ * @param parts The array of #part to sort.
+ * @param xparts The corresponding #xpart array to sort as well.
+ * @param ind The indices with respect to which the parts are sorted.
+ * @param counts Number of particles per index.
+ * @param num_bins Total number of bins (length of count).
+ * @param parts_offset Offset of the #part array from the global #part array.
+ */
+void space_parts_sort(struct part *parts, struct xpart *xparts,
+ int *restrict ind, int *restrict counts, int num_bins,
+ ptrdiff_t parts_offset) {
+ /* Create the offsets array. */
+ size_t *offsets = NULL;
+ if (swift_memalign("parts_offsets", (void **)&offsets, SWIFT_STRUCT_ALIGNMENT,
+ sizeof(size_t) * (num_bins + 1)) != 0)
+ error("Failed to allocate temporary cell offsets array.");
+
+ offsets[0] = 0;
+ for (int k = 1; k <= num_bins; k++) {
+ offsets[k] = offsets[k - 1] + counts[k - 1];
+ counts[k - 1] = 0;
+ }
+
+ /* Loop over local cells. */
+ for (int cid = 0; cid < num_bins; cid++) {
+ for (size_t k = offsets[cid] + counts[cid]; k < offsets[cid + 1]; k++) {
+ counts[cid]++;
+ int target_cid = ind[k];
+ if (target_cid == cid) {
+ continue;
+ }
+ struct part temp_part = parts[k];
+ struct xpart temp_xpart = xparts[k];
+ while (target_cid != cid) {
+ size_t j = offsets[target_cid] + counts[target_cid]++;
+ while (ind[j] == target_cid) {
+ j = offsets[target_cid] + counts[target_cid]++;
+ }
+ memswap(&parts[j], &temp_part, sizeof(struct part));
+ memswap(&xparts[j], &temp_xpart, sizeof(struct xpart));
+ memswap(&ind[j], &target_cid, sizeof(int));
+ if (parts[j].gpart)
+ parts[j].gpart->id_or_neg_offset = -(j + parts_offset);
+ }
+ parts[k] = temp_part;
+ xparts[k] = temp_xpart;
+ ind[k] = target_cid;
+ if (parts[k].gpart)
+ parts[k].gpart->id_or_neg_offset = -(k + parts_offset);
+ }
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ for (int k = 0; k < num_bins; k++)
+ if (offsets[k + 1] != offsets[k] + counts[k])
+ error("Bad offsets after shuffle.");
+#endif /* SWIFT_DEBUG_CHECKS */
+
+ swift_free("parts_offsets", offsets);
+}
+
+/**
+ * @brief Sort the s-particles according to the given indices.
+ *
+ * @param sparts The array of #spart to sort.
+ * @param ind The indices with respect to which the #spart are sorted.
+ * @param counts Number of particles per index.
+ * @param num_bins Total number of bins (length of counts).
+ * @param sparts_offset Offset of the #spart array from the global #spart.
+ * array.
+ */
+void space_sparts_sort(struct spart *sparts, int *restrict ind,
+ int *restrict counts, int num_bins,
+ ptrdiff_t sparts_offset) {
+ /* Create the offsets array. */
+ size_t *offsets = NULL;
+ if (swift_memalign("sparts_offsets", (void **)&offsets,
+ SWIFT_STRUCT_ALIGNMENT,
+ sizeof(size_t) * (num_bins + 1)) != 0)
+ error("Failed to allocate temporary cell offsets array.");
+
+ offsets[0] = 0;
+ for (int k = 1; k <= num_bins; k++) {
+ offsets[k] = offsets[k - 1] + counts[k - 1];
+ counts[k - 1] = 0;
+ }
+
+ /* Loop over local cells. */
+ for (int cid = 0; cid < num_bins; cid++) {
+ for (size_t k = offsets[cid] + counts[cid]; k < offsets[cid + 1]; k++) {
+ counts[cid]++;
+ int target_cid = ind[k];
+ if (target_cid == cid) {
+ continue;
+ }
+ struct spart temp_spart = sparts[k];
+ while (target_cid != cid) {
+ size_t j = offsets[target_cid] + counts[target_cid]++;
+ while (ind[j] == target_cid) {
+ j = offsets[target_cid] + counts[target_cid]++;
+ }
+ memswap(&sparts[j], &temp_spart, sizeof(struct spart));
+ memswap(&ind[j], &target_cid, sizeof(int));
+ if (sparts[j].gpart)
+ sparts[j].gpart->id_or_neg_offset = -(j + sparts_offset);
+ }
+ sparts[k] = temp_spart;
+ ind[k] = target_cid;
+ if (sparts[k].gpart)
+ sparts[k].gpart->id_or_neg_offset = -(k + sparts_offset);
+ }
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ for (int k = 0; k < num_bins; k++)
+ if (offsets[k + 1] != offsets[k] + counts[k])
+ error("Bad offsets after shuffle.");
+#endif /* SWIFT_DEBUG_CHECKS */
+
+ swift_free("sparts_offsets", offsets);
+}
+
+/**
+ * @brief Sort the b-particles according to the given indices.
+ *
+ * @param bparts The array of #bpart to sort.
+ * @param ind The indices with respect to which the #bpart are sorted.
+ * @param counts Number of particles per index.
+ * @param num_bins Total number of bins (length of counts).
+ * @param bparts_offset Offset of the #bpart array from the global #bpart.
+ * array.
+ */
+void space_bparts_sort(struct bpart *bparts, int *restrict ind,
+ int *restrict counts, int num_bins,
+ ptrdiff_t bparts_offset) {
+ /* Create the offsets array. */
+ size_t *offsets = NULL;
+ if (swift_memalign("bparts_offsets", (void **)&offsets,
+ SWIFT_STRUCT_ALIGNMENT,
+ sizeof(size_t) * (num_bins + 1)) != 0)
+ error("Failed to allocate temporary cell offsets array.");
+
+ offsets[0] = 0;
+ for (int k = 1; k <= num_bins; k++) {
+ offsets[k] = offsets[k - 1] + counts[k - 1];
+ counts[k - 1] = 0;
+ }
+
+ /* Loop over local cells. */
+ for (int cid = 0; cid < num_bins; cid++) {
+ for (size_t k = offsets[cid] + counts[cid]; k < offsets[cid + 1]; k++) {
+ counts[cid]++;
+ int target_cid = ind[k];
+ if (target_cid == cid) {
+ continue;
+ }
+ struct bpart temp_bpart = bparts[k];
+ while (target_cid != cid) {
+ size_t j = offsets[target_cid] + counts[target_cid]++;
+ while (ind[j] == target_cid) {
+ j = offsets[target_cid] + counts[target_cid]++;
+ }
+ memswap(&bparts[j], &temp_bpart, sizeof(struct bpart));
+ memswap(&ind[j], &target_cid, sizeof(int));
+ if (bparts[j].gpart)
+ bparts[j].gpart->id_or_neg_offset = -(j + bparts_offset);
+ }
+ bparts[k] = temp_bpart;
+ ind[k] = target_cid;
+ if (bparts[k].gpart)
+ bparts[k].gpart->id_or_neg_offset = -(k + bparts_offset);
+ }
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ for (int k = 0; k < num_bins; k++)
+ if (offsets[k + 1] != offsets[k] + counts[k])
+ error("Bad offsets after shuffle.");
+#endif /* SWIFT_DEBUG_CHECKS */
+
+ swift_free("bparts_offsets", offsets);
+}
+
+/**
+ * @brief Sort the sink-particles according to the given indices.
+ *
+ * @param sinks The array of #sink to sort.
+ * @param ind The indices with respect to which the #sink are sorted.
+ * @param counts Number of particles per index.
+ * @param num_bins Total number of bins (length of counts).
+ * @param sinks_offset Offset of the #sink array from the global #sink.
+ * array.
+ */
+void space_sinks_sort(struct sink *sinks, int *restrict ind,
+ int *restrict counts, int num_bins,
+ ptrdiff_t sinks_offset) {
+ /* Create the offsets array. */
+ size_t *offsets = NULL;
+ if (swift_memalign("sinks_offsets", (void **)&offsets, SWIFT_STRUCT_ALIGNMENT,
+ sizeof(size_t) * (num_bins + 1)) != 0)
+ error("Failed to allocate temporary cell offsets array.");
+
+ offsets[0] = 0;
+ for (int k = 1; k <= num_bins; k++) {
+ offsets[k] = offsets[k - 1] + counts[k - 1];
+ counts[k - 1] = 0;
+ }
+
+ /* Loop over local cells. */
+ for (int cid = 0; cid < num_bins; cid++) {
+ for (size_t k = offsets[cid] + counts[cid]; k < offsets[cid + 1]; k++) {
+ counts[cid]++;
+ int target_cid = ind[k];
+ if (target_cid == cid) {
+ continue;
+ }
+ struct sink temp_sink = sinks[k];
+ while (target_cid != cid) {
+ size_t j = offsets[target_cid] + counts[target_cid]++;
+ while (ind[j] == target_cid) {
+ j = offsets[target_cid] + counts[target_cid]++;
+ }
+ memswap(&sinks[j], &temp_sink, sizeof(struct sink));
+ memswap(&ind[j], &target_cid, sizeof(int));
+ if (sinks[j].gpart)
+ sinks[j].gpart->id_or_neg_offset = -(j + sinks_offset);
+ }
+ sinks[k] = temp_sink;
+ ind[k] = target_cid;
+ if (sinks[k].gpart)
+ sinks[k].gpart->id_or_neg_offset = -(k + sinks_offset);
+ }
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ for (int k = 0; k < num_bins; k++)
+ if (offsets[k + 1] != offsets[k] + counts[k])
+ error("Bad offsets after shuffle.");
+#endif /* SWIFT_DEBUG_CHECKS */
+
+ swift_free("sinks_offsets", offsets);
+}
+
+/**
+ * @brief Sort the g-particles according to the given indices.
+ *
+ * @param gparts The array of #gpart to sort.
+ * @param parts Global #part array for re-linking.
+ * @param sinks Global #sink array for re-linking.
+ * @param sparts Global #spart array for re-linking.
+ * @param bparts Global #bpart array for re-linking.
+ * @param ind The indices with respect to which the gparts are sorted.
+ * @param counts Number of particles per index.
+ * @param num_bins Total number of bins (length of counts).
+ */
+void space_gparts_sort(struct gpart *gparts, struct part *parts,
+ struct sink *sinks, struct spart *sparts,
+ struct bpart *bparts, int *restrict ind,
+ int *restrict counts, int num_bins) {
+ /* Create the offsets array. */
+ size_t *offsets = NULL;
+ if (swift_memalign("gparts_offsets", (void **)&offsets,
+ SWIFT_STRUCT_ALIGNMENT,
+ sizeof(size_t) * (num_bins + 1)) != 0)
+ error("Failed to allocate temporary cell offsets array.");
+
+ offsets[0] = 0;
+ for (int k = 1; k <= num_bins; k++) {
+ offsets[k] = offsets[k - 1] + counts[k - 1];
+ counts[k - 1] = 0;
+ }
+
+ /* Loop over local cells. */
+ for (int cid = 0; cid < num_bins; cid++) {
+ for (size_t k = offsets[cid] + counts[cid]; k < offsets[cid + 1]; k++) {
+ counts[cid]++;
+ int target_cid = ind[k];
+ if (target_cid == cid) {
+ continue;
+ }
+ struct gpart temp_gpart = gparts[k];
+ while (target_cid != cid) {
+ size_t j = offsets[target_cid] + counts[target_cid]++;
+ while (ind[j] == target_cid) {
+ j = offsets[target_cid] + counts[target_cid]++;
+ }
+ memswap_unaligned(&gparts[j], &temp_gpart, sizeof(struct gpart));
+ memswap(&ind[j], &target_cid, sizeof(int));
+ if (gparts[j].type == swift_type_gas) {
+ parts[-gparts[j].id_or_neg_offset].gpart = &gparts[j];
+ } else if (gparts[j].type == swift_type_stars) {
+ sparts[-gparts[j].id_or_neg_offset].gpart = &gparts[j];
+ } else if (gparts[j].type == swift_type_black_hole) {
+ bparts[-gparts[j].id_or_neg_offset].gpart = &gparts[j];
+ } else if (gparts[j].type == swift_type_sink) {
+ sinks[-gparts[j].id_or_neg_offset].gpart = &gparts[j];
+ }
+ }
+ gparts[k] = temp_gpart;
+ ind[k] = target_cid;
+ if (gparts[k].type == swift_type_gas) {
+ parts[-gparts[k].id_or_neg_offset].gpart = &gparts[k];
+ } else if (gparts[k].type == swift_type_stars) {
+ sparts[-gparts[k].id_or_neg_offset].gpart = &gparts[k];
+ } else if (gparts[k].type == swift_type_black_hole) {
+ bparts[-gparts[k].id_or_neg_offset].gpart = &gparts[k];
+ } else if (gparts[k].type == swift_type_sink) {
+ sinks[-gparts[k].id_or_neg_offset].gpart = &gparts[k];
+ }
+ }
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ for (int k = 0; k < num_bins; k++)
+ if (offsets[k + 1] != offsets[k] + counts[k])
+ error("Bad offsets after shuffle.");
+#endif /* SWIFT_DEBUG_CHECKS */
+
+ swift_free("gparts_offsets", offsets);
+}
diff --git a/src/space_split.c b/src/space_split.c
new file mode 100644
index 0000000000000000000000000000000000000000..728c6f8984b3476bfc0729546059477c3d1f38cf
--- /dev/null
+++ b/src/space_split.c
@@ -0,0 +1,723 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
+ * Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ * 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* This object's header. */
+#include "space.h"
+
+/* Local headers. */
+#include "cell.h"
+#include "debug.h"
+#include "engine.h"
+#include "multipole.h"
+#include "star_formation_logger.h"
+#include "threadpool.h"
+
+/*! Counter for cell IDs (when debugging) */
+#if defined(SWIFT_DEBUG_CHECKS) || defined(SWIFT_CELL_GRAPH)
+extern int last_cell_id;
+#endif
+
+/**
+ * @brief Recursively split a cell.
+ *
+ * @param s The #space in which the cell lives.
+ * @param c The #cell to split recursively.
+ * @param buff A buffer for particle sorting, should be of size at least
+ * c->hydro.count or @c NULL.
+ * @param sbuff A buffer for particle sorting, should be of size at least
+ * c->stars.count or @c NULL.
+ * @param bbuff A buffer for particle sorting, should be of size at least
+ * c->black_holes.count or @c NULL.
+ * @param gbuff A buffer for particle sorting, should be of size at least
+ * c->grav.count or @c NULL.
+ * @param sink_buff A buffer for particle sorting, should be of size at least
+ * c->sinks.count or @c NULL.
+ */
+void space_split_recursive(struct space *s, struct cell *c,
+ struct cell_buff *restrict buff,
+ struct cell_buff *restrict sbuff,
+ struct cell_buff *restrict bbuff,
+ struct cell_buff *restrict gbuff,
+ struct cell_buff *restrict sink_buff) {
+
+ const int count = c->hydro.count;
+ const int gcount = c->grav.count;
+ const int scount = c->stars.count;
+ const int bcount = c->black_holes.count;
+ const int sink_count = c->sinks.count;
+ const int with_self_gravity = s->with_self_gravity;
+ const int depth = c->depth;
+ int maxdepth = 0;
+ float h_max = 0.0f;
+ float sinks_h_max = 0.f;
+ float stars_h_max = 0.f;
+ float black_holes_h_max = 0.f;
+ integertime_t ti_hydro_end_min = max_nr_timesteps, ti_hydro_end_max = 0,
+ ti_hydro_beg_max = 0;
+ integertime_t ti_gravity_end_min = max_nr_timesteps, ti_gravity_end_max = 0,
+ ti_gravity_beg_max = 0;
+ integertime_t ti_stars_end_min = max_nr_timesteps, ti_stars_end_max = 0,
+ ti_stars_beg_max = 0;
+ integertime_t ti_sinks_end_min = max_nr_timesteps, ti_sinks_end_max = 0,
+ ti_sinks_beg_max = 0;
+ integertime_t ti_black_holes_end_min = max_nr_timesteps,
+ ti_black_holes_end_max = 0, ti_black_holes_beg_max = 0;
+ struct part *parts = c->hydro.parts;
+ struct gpart *gparts = c->grav.parts;
+ struct spart *sparts = c->stars.parts;
+ struct bpart *bparts = c->black_holes.parts;
+ struct xpart *xparts = c->hydro.xparts;
+ struct sink *sinks = c->sinks.parts;
+ struct engine *e = s->e;
+ const integertime_t ti_current = e->ti_current;
+
+ /* If the buff is NULL, allocate it, and remember to free it. */
+ const int allocate_buffer = (buff == NULL && gbuff == NULL && sbuff == NULL &&
+ bbuff == NULL && sink_buff == NULL);
+ if (allocate_buffer) {
+ if (count > 0) {
+ if (swift_memalign("tempbuff", (void **)&buff, SWIFT_STRUCT_ALIGNMENT,
+ sizeof(struct cell_buff) * count) != 0)
+ error("Failed to allocate temporary indices.");
+ for (int k = 0; k < count; k++) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (parts[k].time_bin == time_bin_inhibited)
+ error("Inhibited particle present in space_split()");
+ if (parts[k].time_bin == time_bin_not_created)
+ error("Extra particle present in space_split()");
+#endif
+ buff[k].x[0] = parts[k].x[0];
+ buff[k].x[1] = parts[k].x[1];
+ buff[k].x[2] = parts[k].x[2];
+ }
+ }
+ if (gcount > 0) {
+ if (swift_memalign("tempgbuff", (void **)&gbuff, SWIFT_STRUCT_ALIGNMENT,
+ sizeof(struct cell_buff) * gcount) != 0)
+ error("Failed to allocate temporary indices.");
+ for (int k = 0; k < gcount; k++) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (gparts[k].time_bin == time_bin_inhibited)
+ error("Inhibited particle present in space_split()");
+ if (gparts[k].time_bin == time_bin_not_created)
+ error("Extra particle present in space_split()");
+#endif
+ gbuff[k].x[0] = gparts[k].x[0];
+ gbuff[k].x[1] = gparts[k].x[1];
+ gbuff[k].x[2] = gparts[k].x[2];
+ }
+ }
+ if (scount > 0) {
+ if (swift_memalign("tempsbuff", (void **)&sbuff, SWIFT_STRUCT_ALIGNMENT,
+ sizeof(struct cell_buff) * scount) != 0)
+ error("Failed to allocate temporary indices.");
+ for (int k = 0; k < scount; k++) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (sparts[k].time_bin == time_bin_inhibited)
+ error("Inhibited particle present in space_split()");
+ if (sparts[k].time_bin == time_bin_not_created)
+ error("Extra particle present in space_split()");
+#endif
+ sbuff[k].x[0] = sparts[k].x[0];
+ sbuff[k].x[1] = sparts[k].x[1];
+ sbuff[k].x[2] = sparts[k].x[2];
+ }
+ }
+ if (bcount > 0) {
+ if (swift_memalign("tempbbuff", (void **)&bbuff, SWIFT_STRUCT_ALIGNMENT,
+ sizeof(struct cell_buff) * bcount) != 0)
+ error("Failed to allocate temporary indices.");
+ for (int k = 0; k < bcount; k++) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (bparts[k].time_bin == time_bin_inhibited)
+ error("Inhibited particle present in space_split()");
+ if (bparts[k].time_bin == time_bin_not_created)
+ error("Extra particle present in space_split()");
+#endif
+ bbuff[k].x[0] = bparts[k].x[0];
+ bbuff[k].x[1] = bparts[k].x[1];
+ bbuff[k].x[2] = bparts[k].x[2];
+ }
+ }
+ if (sink_count > 0) {
+ if (swift_memalign("temp_sink_buff", (void **)&sink_buff,
+ SWIFT_STRUCT_ALIGNMENT,
+ sizeof(struct cell_buff) * sink_count) != 0)
+ error("Failed to allocate temporary indices.");
+ for (int k = 0; k < sink_count; k++) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (sinks[k].time_bin == time_bin_inhibited)
+ error("Inhibited particle present in space_split()");
+ if (sinks[k].time_bin == time_bin_not_created)
+ error("Extra particle present in space_split()");
+#endif
+ sink_buff[k].x[0] = sinks[k].x[0];
+ sink_buff[k].x[1] = sinks[k].x[1];
+ sink_buff[k].x[2] = sinks[k].x[2];
+ }
+ }
+ }
+
+ /* If the depth is too large, we have a problem and should stop. */
+ if (depth > space_cell_maxdepth) {
+ error(
+ "Exceeded maximum depth (%d) when splitting cells, aborting. This is "
+ "most likely due to having too many particles at the exact same "
+ "position, making the construction of a tree impossible.",
+ space_cell_maxdepth);
+ }
+
+ /* Split or let it be? */
+ if ((with_self_gravity && gcount > space_splitsize) ||
+ (!with_self_gravity &&
+ (count > space_splitsize || scount > space_splitsize))) {
+
+ /* No longer just a leaf. */
+ c->split = 1;
+
+ /* Create the cell's progeny. */
+ space_getcells(s, 8, c->progeny);
+ for (int k = 0; k < 8; k++) {
+ struct cell *cp = c->progeny[k];
+ cp->hydro.count = 0;
+ cp->grav.count = 0;
+ cp->stars.count = 0;
+ cp->sinks.count = 0;
+ cp->black_holes.count = 0;
+ cp->hydro.count_total = 0;
+ cp->grav.count_total = 0;
+ cp->sinks.count_total = 0;
+ cp->stars.count_total = 0;
+ cp->black_holes.count_total = 0;
+ cp->hydro.ti_old_part = c->hydro.ti_old_part;
+ cp->grav.ti_old_part = c->grav.ti_old_part;
+ cp->grav.ti_old_multipole = c->grav.ti_old_multipole;
+ cp->stars.ti_old_part = c->stars.ti_old_part;
+ cp->sinks.ti_old_part = c->sinks.ti_old_part;
+ cp->black_holes.ti_old_part = c->black_holes.ti_old_part;
+ cp->loc[0] = c->loc[0];
+ cp->loc[1] = c->loc[1];
+ cp->loc[2] = c->loc[2];
+ cp->width[0] = c->width[0] / 2;
+ cp->width[1] = c->width[1] / 2;
+ cp->width[2] = c->width[2] / 2;
+ cp->dmin = c->dmin / 2;
+ if (k & 4) cp->loc[0] += cp->width[0];
+ if (k & 2) cp->loc[1] += cp->width[1];
+ if (k & 1) cp->loc[2] += cp->width[2];
+ cp->depth = c->depth + 1;
+ cp->split = 0;
+ cp->hydro.h_max = 0.f;
+ cp->hydro.dx_max_part = 0.f;
+ cp->hydro.dx_max_sort = 0.f;
+ cp->stars.h_max = 0.f;
+ cp->stars.dx_max_part = 0.f;
+ cp->stars.dx_max_sort = 0.f;
+ cp->sinks.r_cut_max = 0.f;
+ cp->sinks.dx_max_part = 0.f;
+ cp->black_holes.h_max = 0.f;
+ cp->black_holes.dx_max_part = 0.f;
+ cp->nodeID = c->nodeID;
+ cp->parent = c;
+ cp->top = c->top;
+ cp->super = NULL;
+ cp->hydro.super = NULL;
+ cp->grav.super = NULL;
+ cp->flags = 0;
+ star_formation_logger_init(&cp->stars.sfh);
+#ifdef WITH_MPI
+ cp->mpi.tag = -1;
+#endif // WITH_MPI
+#if defined(SWIFT_DEBUG_CHECKS) || defined(SWIFT_CELL_GRAPH)
+ cp->cellID = last_cell_id++;
+#endif
+ }
+
+ /* Split the cell's particle data. */
+ cell_split(c, c->hydro.parts - s->parts, c->stars.parts - s->sparts,
+ c->black_holes.parts - s->bparts, c->sinks.parts - s->sinks,
+ buff, sbuff, bbuff, gbuff, sink_buff);
+
+ /* Buffers for the progenitors */
+ struct cell_buff *progeny_buff = buff, *progeny_gbuff = gbuff,
+ *progeny_sbuff = sbuff, *progeny_bbuff = bbuff,
+ *progeny_sink_buff = sink_buff;
+
+ for (int k = 0; k < 8; k++) {
+
+ /* Get the progenitor */
+ struct cell *cp = c->progeny[k];
+
+ /* Remove any progeny with zero particles. */
+ if (cp->hydro.count == 0 && cp->grav.count == 0 && cp->stars.count == 0 &&
+ cp->black_holes.count == 0 && cp->sinks.count == 0) {
+
+ space_recycle(s, cp);
+ c->progeny[k] = NULL;
+
+ } else {
+
+ /* Recurse */
+ space_split_recursive(s, cp, progeny_buff, progeny_sbuff, progeny_bbuff,
+ progeny_gbuff, progeny_sink_buff);
+
+ /* Update the pointers in the buffers */
+ progeny_buff += cp->hydro.count;
+ progeny_gbuff += cp->grav.count;
+ progeny_sbuff += cp->stars.count;
+ progeny_bbuff += cp->black_holes.count;
+ progeny_sink_buff += cp->sinks.count;
+
+ /* Update the cell-wide properties */
+ h_max = max(h_max, cp->hydro.h_max);
+ stars_h_max = max(stars_h_max, cp->stars.h_max);
+ black_holes_h_max = max(black_holes_h_max, cp->black_holes.h_max);
+ sinks_h_max = max(sinks_h_max, cp->sinks.r_cut_max);
+
+ ti_hydro_end_min = min(ti_hydro_end_min, cp->hydro.ti_end_min);
+ ti_hydro_end_max = max(ti_hydro_end_max, cp->hydro.ti_end_max);
+ ti_hydro_beg_max = max(ti_hydro_beg_max, cp->hydro.ti_beg_max);
+ ti_gravity_end_min = min(ti_gravity_end_min, cp->grav.ti_end_min);
+ ti_gravity_end_max = max(ti_gravity_end_max, cp->grav.ti_end_max);
+ ti_gravity_beg_max = max(ti_gravity_beg_max, cp->grav.ti_beg_max);
+ ti_stars_end_min = min(ti_stars_end_min, cp->stars.ti_end_min);
+ ti_stars_end_max = max(ti_stars_end_max, cp->stars.ti_end_max);
+ ti_stars_beg_max = max(ti_stars_beg_max, cp->stars.ti_beg_max);
+ ti_sinks_end_min = min(ti_sinks_end_min, cp->sinks.ti_end_min);
+ ti_sinks_end_max = max(ti_sinks_end_max, cp->sinks.ti_end_max);
+ ti_sinks_beg_max = max(ti_sinks_beg_max, cp->sinks.ti_beg_max);
+ ti_black_holes_end_min =
+ min(ti_black_holes_end_min, cp->black_holes.ti_end_min);
+ ti_black_holes_end_max =
+ max(ti_black_holes_end_max, cp->black_holes.ti_end_max);
+ ti_black_holes_beg_max =
+ max(ti_black_holes_beg_max, cp->black_holes.ti_beg_max);
+
+ star_formation_logger_add(&c->stars.sfh, &cp->stars.sfh);
+
+ /* Increase the depth */
+ maxdepth = max(maxdepth, cp->maxdepth);
+ }
+ }
+
+ /* Deal with the multipole */
+ if (s->with_self_gravity) {
+
+ /* Reset everything */
+ gravity_reset(c->grav.multipole);
+
+ /* Compute CoM and bulk velocity from all progenies */
+ double CoM[3] = {0., 0., 0.};
+ double vel[3] = {0., 0., 0.};
+ float max_delta_vel[3] = {0.f, 0.f, 0.f};
+ float min_delta_vel[3] = {0.f, 0.f, 0.f};
+ double mass = 0.;
+
+ for (int k = 0; k < 8; ++k) {
+ if (c->progeny[k] != NULL) {
+ const struct gravity_tensors *m = c->progeny[k]->grav.multipole;
+
+ mass += m->m_pole.M_000;
+
+ CoM[0] += m->CoM[0] * m->m_pole.M_000;
+ CoM[1] += m->CoM[1] * m->m_pole.M_000;
+ CoM[2] += m->CoM[2] * m->m_pole.M_000;
+
+ vel[0] += m->m_pole.vel[0] * m->m_pole.M_000;
+ vel[1] += m->m_pole.vel[1] * m->m_pole.M_000;
+ vel[2] += m->m_pole.vel[2] * m->m_pole.M_000;
+
+ max_delta_vel[0] = max(m->m_pole.max_delta_vel[0], max_delta_vel[0]);
+ max_delta_vel[1] = max(m->m_pole.max_delta_vel[1], max_delta_vel[1]);
+ max_delta_vel[2] = max(m->m_pole.max_delta_vel[2], max_delta_vel[2]);
+
+ min_delta_vel[0] = min(m->m_pole.min_delta_vel[0], min_delta_vel[0]);
+ min_delta_vel[1] = min(m->m_pole.min_delta_vel[1], min_delta_vel[1]);
+ min_delta_vel[2] = min(m->m_pole.min_delta_vel[2], min_delta_vel[2]);
+ }
+ }
+
+ /* Final operation on the CoM and bulk velocity */
+ const double inv_mass = 1. / mass;
+ c->grav.multipole->CoM[0] = CoM[0] * inv_mass;
+ c->grav.multipole->CoM[1] = CoM[1] * inv_mass;
+ c->grav.multipole->CoM[2] = CoM[2] * inv_mass;
+ c->grav.multipole->m_pole.vel[0] = vel[0] * inv_mass;
+ c->grav.multipole->m_pole.vel[1] = vel[1] * inv_mass;
+ c->grav.multipole->m_pole.vel[2] = vel[2] * inv_mass;
+
+ /* Min max velocity along each axis */
+ c->grav.multipole->m_pole.max_delta_vel[0] = max_delta_vel[0];
+ c->grav.multipole->m_pole.max_delta_vel[1] = max_delta_vel[1];
+ c->grav.multipole->m_pole.max_delta_vel[2] = max_delta_vel[2];
+ c->grav.multipole->m_pole.min_delta_vel[0] = min_delta_vel[0];
+ c->grav.multipole->m_pole.min_delta_vel[1] = min_delta_vel[1];
+ c->grav.multipole->m_pole.min_delta_vel[2] = min_delta_vel[2];
+
+ /* Now shift progeny multipoles and add them up */
+ struct multipole temp;
+ double r_max = 0.;
+ for (int k = 0; k < 8; ++k) {
+ if (c->progeny[k] != NULL) {
+ const struct cell *cp = c->progeny[k];
+ const struct multipole *m = &cp->grav.multipole->m_pole;
+
+ /* Contribution to multipole */
+ gravity_M2M(&temp, m, c->grav.multipole->CoM,
+ cp->grav.multipole->CoM);
+ gravity_multipole_add(&c->grav.multipole->m_pole, &temp);
+
+ /* Upper limit of max CoM<->gpart distance */
+ const double dx =
+ c->grav.multipole->CoM[0] - cp->grav.multipole->CoM[0];
+ const double dy =
+ c->grav.multipole->CoM[1] - cp->grav.multipole->CoM[1];
+ const double dz =
+ c->grav.multipole->CoM[2] - cp->grav.multipole->CoM[2];
+ const double r2 = dx * dx + dy * dy + dz * dz;
+ r_max = max(r_max, cp->grav.multipole->r_max + sqrt(r2));
+ }
+ }
+
+ /* Alternative upper limit of max CoM<->gpart distance */
+ const double dx =
+ c->grav.multipole->CoM[0] > c->loc[0] + c->width[0] / 2.
+ ? c->grav.multipole->CoM[0] - c->loc[0]
+ : c->loc[0] + c->width[0] - c->grav.multipole->CoM[0];
+ const double dy =
+ c->grav.multipole->CoM[1] > c->loc[1] + c->width[1] / 2.
+ ? c->grav.multipole->CoM[1] - c->loc[1]
+ : c->loc[1] + c->width[1] - c->grav.multipole->CoM[1];
+ const double dz =
+ c->grav.multipole->CoM[2] > c->loc[2] + c->width[2] / 2.
+ ? c->grav.multipole->CoM[2] - c->loc[2]
+ : c->loc[2] + c->width[2] - c->grav.multipole->CoM[2];
+
+ /* Take minimum of both limits */
+ c->grav.multipole->r_max = min(r_max, sqrt(dx * dx + dy * dy + dz * dz));
+
+ /* Store the value at rebuild time */
+ c->grav.multipole->r_max_rebuild = c->grav.multipole->r_max;
+ c->grav.multipole->CoM_rebuild[0] = c->grav.multipole->CoM[0];
+ c->grav.multipole->CoM_rebuild[1] = c->grav.multipole->CoM[1];
+ c->grav.multipole->CoM_rebuild[2] = c->grav.multipole->CoM[2];
+
+ /* Compute the multipole power */
+ gravity_multipole_compute_power(&c->grav.multipole->m_pole);
+
+ } /* Deal with gravity */
+ } /* Split or let it be? */
+
+ /* Otherwise, collect the data from the particles this cell. */
+ else {
+
+ /* Clear the progeny. */
+ bzero(c->progeny, sizeof(struct cell *) * 8);
+ c->split = 0;
+ maxdepth = c->depth;
+
+ ti_hydro_end_min = max_nr_timesteps;
+ ti_hydro_end_max = 0;
+ ti_hydro_beg_max = 0;
+
+ ti_gravity_end_min = max_nr_timesteps;
+ ti_gravity_end_max = 0;
+ ti_gravity_beg_max = 0;
+
+ ti_stars_end_min = max_nr_timesteps;
+ ti_stars_end_max = 0;
+ ti_stars_beg_max = 0;
+
+ ti_black_holes_end_min = max_nr_timesteps;
+ ti_black_holes_end_max = 0;
+ ti_black_holes_beg_max = 0;
+
+ /* parts: Get dt_min/dt_max and h_max. */
+ for (int k = 0; k < count; k++) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (parts[k].time_bin == time_bin_not_created)
+ error("Extra particle present in space_split()");
+ if (parts[k].time_bin == time_bin_inhibited)
+ error("Inhibited particle present in space_split()");
+#endif
+
+ /* When does this particle's time-step start and end? */
+ const timebin_t time_bin = parts[k].time_bin;
+ const integertime_t ti_end = get_integer_time_end(ti_current, time_bin);
+ const integertime_t ti_beg = get_integer_time_begin(ti_current, time_bin);
+
+ ti_hydro_end_min = min(ti_hydro_end_min, ti_end);
+ ti_hydro_end_max = max(ti_hydro_end_max, ti_end);
+ ti_hydro_beg_max = max(ti_hydro_beg_max, ti_beg);
+
+ h_max = max(h_max, parts[k].h);
+
+ /* Collect SFR from the particles after rebuilt */
+ star_formation_logger_log_inactive_part(&parts[k], &xparts[k],
+ &c->stars.sfh);
+ }
+
+ /* xparts: Reset x_diff */
+ for (int k = 0; k < count; k++) {
+ xparts[k].x_diff[0] = 0.f;
+ xparts[k].x_diff[1] = 0.f;
+ xparts[k].x_diff[2] = 0.f;
+ }
+
+ /* gparts: Get dt_min/dt_max. */
+ for (int k = 0; k < gcount; k++) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (gparts[k].time_bin == time_bin_not_created)
+ error("Extra g-particle present in space_split()");
+ if (gparts[k].time_bin == time_bin_inhibited)
+ error("Inhibited g-particle present in space_split()");
+#endif
+
+ /* When does this particle's time-step start and end? */
+ const timebin_t time_bin = gparts[k].time_bin;
+ const integertime_t ti_end = get_integer_time_end(ti_current, time_bin);
+ const integertime_t ti_beg = get_integer_time_begin(ti_current, time_bin);
+
+ ti_gravity_end_min = min(ti_gravity_end_min, ti_end);
+ ti_gravity_end_max = max(ti_gravity_end_max, ti_end);
+ ti_gravity_beg_max = max(ti_gravity_beg_max, ti_beg);
+ }
+
+ /* sparts: Get dt_min/dt_max */
+ for (int k = 0; k < scount; k++) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (sparts[k].time_bin == time_bin_not_created)
+ error("Extra s-particle present in space_split()");
+ if (sparts[k].time_bin == time_bin_inhibited)
+ error("Inhibited s-particle present in space_split()");
+#endif
+
+ /* When does this particle's time-step start and end? */
+ const timebin_t time_bin = sparts[k].time_bin;
+ const integertime_t ti_end = get_integer_time_end(ti_current, time_bin);
+ const integertime_t ti_beg = get_integer_time_begin(ti_current, time_bin);
+
+ ti_stars_end_min = min(ti_stars_end_min, ti_end);
+ ti_stars_end_max = max(ti_stars_end_max, ti_end);
+ ti_stars_beg_max = max(ti_stars_beg_max, ti_beg);
+
+ stars_h_max = max(stars_h_max, sparts[k].h);
+
+ /* Reset x_diff */
+ sparts[k].x_diff[0] = 0.f;
+ sparts[k].x_diff[1] = 0.f;
+ sparts[k].x_diff[2] = 0.f;
+ }
+
+ /* sinks: Get dt_min/dt_max */
+ for (int k = 0; k < sink_count; k++) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (sinks[k].time_bin == time_bin_not_created)
+ error("Extra sink-particle present in space_split()");
+ if (sinks[k].time_bin == time_bin_inhibited)
+ error("Inhibited sink-particle present in space_split()");
+#endif
+
+ /* When does this particle's time-step start and end? */
+ const timebin_t time_bin = sinks[k].time_bin;
+ const integertime_t ti_end = get_integer_time_end(ti_current, time_bin);
+ const integertime_t ti_beg = get_integer_time_begin(ti_current, time_bin);
+
+ ti_sinks_end_min = min(ti_sinks_end_min, ti_end);
+ ti_sinks_end_max = max(ti_sinks_end_max, ti_end);
+ ti_sinks_beg_max = max(ti_sinks_beg_max, ti_beg);
+
+ sinks_h_max = max(sinks_h_max, sinks[k].r_cut);
+
+ /* Reset x_diff */
+ sinks[k].x_diff[0] = 0.f;
+ sinks[k].x_diff[1] = 0.f;
+ sinks[k].x_diff[2] = 0.f;
+ }
+
+ /* bparts: Get dt_min/dt_max */
+ for (int k = 0; k < bcount; k++) {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (bparts[k].time_bin == time_bin_not_created)
+ error("Extra b-particle present in space_split()");
+ if (bparts[k].time_bin == time_bin_inhibited)
+ error("Inhibited b-particle present in space_split()");
+#endif
+
+ /* When does this particle's time-step start and end? */
+ const timebin_t time_bin = bparts[k].time_bin;
+ const integertime_t ti_end = get_integer_time_end(ti_current, time_bin);
+ const integertime_t ti_beg = get_integer_time_begin(ti_current, time_bin);
+
+ ti_black_holes_end_min = min(ti_black_holes_end_min, ti_end);
+ ti_black_holes_end_max = max(ti_black_holes_end_max, ti_end);
+ ti_black_holes_beg_max = max(ti_black_holes_beg_max, ti_beg);
+
+ black_holes_h_max = max(black_holes_h_max, bparts[k].h);
+
+ /* Reset x_diff */
+ bparts[k].x_diff[0] = 0.f;
+ bparts[k].x_diff[1] = 0.f;
+ bparts[k].x_diff[2] = 0.f;
+ }
+
+ /* Construct the multipole and the centre of mass*/
+ if (s->with_self_gravity) {
+ if (gcount > 0) {
+
+ gravity_P2M(c->grav.multipole, c->grav.parts, c->grav.count,
+ e->gravity_properties);
+
+ /* Compute the multipole power */
+ gravity_multipole_compute_power(&c->grav.multipole->m_pole);
+
+ } else {
+
+ /* No gparts in that leaf cell */
+
+ /* Set the values to something sensible */
+ gravity_multipole_init(&c->grav.multipole->m_pole);
+ if (c->nodeID == engine_rank) {
+ c->grav.multipole->CoM[0] = c->loc[0] + c->width[0] / 2.;
+ c->grav.multipole->CoM[1] = c->loc[1] + c->width[1] / 2.;
+ c->grav.multipole->CoM[2] = c->loc[2] + c->width[2] / 2.;
+ c->grav.multipole->r_max = 0.;
+ }
+ }
+
+ /* Store the value at rebuild time */
+ c->grav.multipole->r_max_rebuild = c->grav.multipole->r_max;
+ c->grav.multipole->CoM_rebuild[0] = c->grav.multipole->CoM[0];
+ c->grav.multipole->CoM_rebuild[1] = c->grav.multipole->CoM[1];
+ c->grav.multipole->CoM_rebuild[2] = c->grav.multipole->CoM[2];
+ }
+ }
+
+ /* Set the values for this cell. */
+ c->hydro.h_max = h_max;
+ c->hydro.ti_end_min = ti_hydro_end_min;
+ c->hydro.ti_end_max = ti_hydro_end_max;
+ c->hydro.ti_beg_max = ti_hydro_beg_max;
+ c->grav.ti_end_min = ti_gravity_end_min;
+ c->grav.ti_end_max = ti_gravity_end_max;
+ c->grav.ti_beg_max = ti_gravity_beg_max;
+ c->stars.ti_end_min = ti_stars_end_min;
+ c->stars.ti_end_max = ti_stars_end_max;
+ c->stars.ti_beg_max = ti_stars_beg_max;
+ c->stars.h_max = stars_h_max;
+ c->sinks.ti_end_min = ti_sinks_end_min;
+ c->sinks.ti_end_max = ti_sinks_end_max;
+ c->sinks.ti_beg_max = ti_sinks_beg_max;
+ c->sinks.r_cut_max = sinks_h_max;
+ c->black_holes.ti_end_min = ti_black_holes_end_min;
+ c->black_holes.ti_end_max = ti_black_holes_end_max;
+ c->black_holes.ti_beg_max = ti_black_holes_beg_max;
+ c->black_holes.h_max = black_holes_h_max;
+ c->maxdepth = maxdepth;
+
+ /* Set ownership according to the start of the parts array. */
+ if (s->nr_parts > 0)
+ c->owner = ((c->hydro.parts - s->parts) % s->nr_parts) * s->nr_queues /
+ s->nr_parts;
+ else if (s->nr_sinks > 0)
+ c->owner = ((c->sinks.parts - s->sinks) % s->nr_sinks) * s->nr_queues /
+ s->nr_sinks;
+ else if (s->nr_sparts > 0)
+ c->owner = ((c->stars.parts - s->sparts) % s->nr_sparts) * s->nr_queues /
+ s->nr_sparts;
+ else if (s->nr_bparts > 0)
+ c->owner = ((c->black_holes.parts - s->bparts) % s->nr_bparts) *
+ s->nr_queues / s->nr_bparts;
+ else if (s->nr_gparts > 0)
+ c->owner = ((c->grav.parts - s->gparts) % s->nr_gparts) * s->nr_queues /
+ s->nr_gparts;
+ else
+ c->owner = 0; /* Ok, there is really nothing on this rank... */
+
+ /* Store the global max depth */
+ if (c->depth == 0) atomic_max(&s->maxdepth, maxdepth);
+
+ /* Clean up. */
+ if (allocate_buffer) {
+ if (buff != NULL) swift_free("tempbuff", buff);
+ if (gbuff != NULL) swift_free("tempgbuff", gbuff);
+ if (sbuff != NULL) swift_free("tempsbuff", sbuff);
+ if (bbuff != NULL) swift_free("tempbbuff", bbuff);
+ if (sink_buff != NULL) swift_free("temp_sink_buff", sink_buff);
+ }
+}
+
+/**
+ * @brief #threadpool mapper function to split cells if they contain
+ * too many particles.
+ *
+ * @param map_data Pointer towards the top-cells.
+ * @param num_cells The number of cells to treat.
+ * @param extra_data Pointers to the #space.
+ */
+void space_split_mapper(void *map_data, int num_cells, void *extra_data) {
+
+ /* Unpack the inputs. */
+ struct space *s = (struct space *)extra_data;
+ struct cell *cells_top = s->cells_top;
+ int *local_cells_with_particles = (int *)map_data;
+
+ /* Loop over the non-empty cells */
+ for (int ind = 0; ind < num_cells; ind++) {
+ struct cell *c = &cells_top[local_cells_with_particles[ind]];
+ space_split_recursive(s, c, NULL, NULL, NULL, NULL, NULL);
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* All cells and particles should have consistent h_max values. */
+ for (int ind = 0; ind < num_cells; ind++) {
+ int depth = 0;
+ const struct cell *c = &cells_top[local_cells_with_particles[ind]];
+ if (!checkCellhdxmax(c, &depth)) message(" at cell depth %d", depth);
+ }
+#endif
+}
+
+/**
+ * @brief Split particles between cells of a hierarchy.
+ *
+ * This is done in parallel using threads in the #threadpool.
+ * Only do this for the local non-empty top-level cells.
+ *
+ * @param s The #space.
+ * @param verbose Are we talkative ?
+ */
+void space_split(struct space *s, int verbose) {
+
+ const ticks tic = getticks();
+
+ threadpool_map(&s->e->threadpool, space_split_mapper,
+ s->local_cells_with_particles_top,
+ s->nr_local_cells_with_particles, sizeof(int),
+ threadpool_auto_chunk_size, s);
+
+ if (verbose)
+ message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+ clocks_getunit());
+}