diff --git a/src/engine.c b/src/engine.c
index d0acea245941462f1e747bf28de2f2fd7ae95322..454e3dd76608b70cdef4e9b032e7885fc40c485f 100644
--- a/src/engine.c
+++ b/src/engine.c
@@ -3930,6 +3930,9 @@ void engine_rebuild(struct engine *e, int clean_smoothing_length_values) {
/* Re-build the space. */
space_rebuild(e->s, e->verbose);
+ /* Construct the list of purely local cells */
+ space_list_local_cells(e->s);
+
/* Re-compute the mesh forces */
if ((e->policy & engine_policy_self_gravity) && e->s->periodic)
pm_mesh_compute_potential(e->mesh, e->s, e->verbose);
@@ -4217,7 +4220,7 @@ void engine_collect_end_of_step(struct engine *e, int apply) {
/* Collect information from the local top-level cells */
threadpool_map(&e->threadpool, engine_collect_end_of_step_mapper,
- s->local_cells_top, s->nr_local_cells, sizeof(int), 0, &data);
+ s->local_cells_with_tasks_top, s->nr_local_cells_with_tasks, sizeof(int), 0, &data);
/* Store these in the temporary collection group. */
collectgroup1_init(&e->collect_group1, data.updates, data.g_updates,
@@ -5085,14 +5088,15 @@ void engine_unskip(struct engine *e) {
#endif // WITH_PROFILER
/* Move the active local cells to the top of the list. */
- int *local_cells = e->s->local_cells_top;
+ int *local_cells = e->s->local_cells_with_tasks_top;
int num_active_cells = 0;
- for (int k = 0; k < s->nr_local_cells; k++) {
+ for (int k = 0; k < s->nr_local_cells_with_tasks; k++) {
struct cell *c = &s->cells_top[local_cells[k]];
+
if ((e->policy & engine_policy_hydro && cell_is_active_hydro(c, e)) ||
- (e->policy &
- (engine_policy_self_gravity | engine_policy_external_gravity) &&
- cell_is_active_gravity(c, e))) {
+ (e->policy & engine_policy_self_gravity && cell_is_active_gravity(c, e)) ||
+ (e->policy & engine_policy_external_gravity && cell_is_active_gravity(c, e))) {
+
if (num_active_cells != k)
memswap(&local_cells[k], &local_cells[num_active_cells], sizeof(int));
num_active_cells += 1;
diff --git a/src/space.c b/src/space.c
index 9a9dd05c237a69d034bf618490c834ade5ee33f7..1580c1df64e35a919b4b80f79eeea9bf74d1ce75 100644
--- a/src/space.c
+++ b/src/space.c
@@ -360,6 +360,7 @@ void space_regrid(struct space *s, int verbose) {
/* Free the old cells, if they were allocated. */
if (s->cells_top != NULL) {
space_free_cells(s);
+ free(s->local_cells_with_tasks_top);
free(s->local_cells_top);
free(s->cells_top);
free(s->multipoles_top);
@@ -398,6 +399,12 @@ void space_regrid(struct space *s, int verbose) {
error("Failed to allocate indices of local top-level cells.");
bzero(s->local_cells_top, s->nr_cells * sizeof(int));
+ /* Allocate the indices of local cells with tasks */
+ if (posix_memalign((void **)&s->local_cells_with_tasks_top, SWIFT_STRUCT_ALIGNMENT,
+ s->nr_cells * sizeof(int)) != 0)
+ error("Failed to allocate indices of local top-level cells.");
+ bzero(s->local_cells_with_tasks_top, s->nr_cells * sizeof(int));
+
/* Set the cells' locks */
for (int k = 0; k < s->nr_cells; k++) {
if (lock_init(&s->cells_top[k].lock) != 0)
@@ -2337,7 +2344,7 @@ void space_free_buff_sort_indices(struct space *s) {
/**
* @brief Construct the list of top-level cells that have any tasks in
- * their hierarchy.
+ * their hierarchy on this MPI rank.
*
* This assumes the list has been pre-allocated at a regrid.
*
@@ -2345,15 +2352,36 @@ void space_free_buff_sort_indices(struct space *s) {
*/
void space_list_cells_with_tasks(struct space *s) {
- /* Let's rebuild the list of local top-level cells */
- s->nr_local_cells = 0;
+ s->nr_local_cells_with_tasks = 0;
+
for (int i = 0; i < s->nr_cells; ++i)
if (cell_has_tasks(&s->cells_top[i])) {
+ s->local_cells_with_tasks_top[s->nr_local_cells_with_tasks] = i;
+ s->nr_local_cells_with_tasks++;
+ }
+ if (s->e->verbose)
+ message("Have %d local top-level cells with tasks (total=%d)", s->nr_local_cells_with_tasks, s->nr_cells);
+}
+
+/**
+ * @brief Construct the list of local top-level cells.
+ *
+ * This assumes the list has been pre-allocated at a regrid.
+ *
+ * @param s The #space.
+ */
+void space_list_local_cells(struct space *s) {
+
+ s->nr_local_cells = 0;
+
+ for (int i = 0; i < s->nr_cells; ++i)
+ if(s->cells_top[i].nodeID == engine_rank) {
s->local_cells_top[s->nr_local_cells] = i;
s->nr_local_cells++;
}
+
if (s->e->verbose)
- message("Have %d local cells (total=%d)", s->nr_local_cells, s->nr_cells);
+ message("Have %d local top-level cells (total=%d)", s->nr_local_cells, s->nr_cells);
}
void space_synchronize_particle_positions_mapper(void *map_data, int nr_gparts,
@@ -3287,6 +3315,7 @@ void space_clean(struct space *s) {
free(s->cells_top);
free(s->multipoles_top);
free(s->local_cells_top);
+ free(s->local_cells_with_tasks_top);
free(s->parts);
free(s->xparts);
free(s->gparts);
@@ -3338,6 +3367,7 @@ void space_struct_restore(struct space *s, FILE *stream) {
s->multipoles_top = NULL;
s->multipoles_sub = NULL;
s->local_cells_top = NULL;
+ s->local_cells_with_tasks_top = NULL;
s->grav_top_level = NULL;
#ifdef WITH_MPI
s->parts_foreign = NULL;
diff --git a/src/space.h b/src/space.h
index e3173ece1e2749a3afb8072b179150587a100a82..c55282529b9bcf785e8f49633714b43383a19c73 100644
--- a/src/space.h
+++ b/src/space.h
@@ -106,9 +106,12 @@ struct space {
/*! Total number of cells (top- and sub-) */
int tot_cells;
- /*! Number of *local* top-level cells with tasks */
+ /*! Number of *local* top-level cells */
int nr_local_cells;
+ /*! Number of *local* top-level cells with tasks */
+ int nr_local_cells_with_tasks;
+
/*! The (level 0) cells themselves. */
struct cell *cells_top;
@@ -121,9 +124,12 @@ struct space {
/*! Buffer of unused multipoles for the sub-cells. */
struct gravity_tensors *multipoles_sub;
- /*! The indices of the *local* top-level cells with tasks */
+ /*! The indices of the *local* top-level cells */
int *local_cells_top;
+ /*! The indices of the *local* top-level cells with tasks */
+ int *local_cells_with_tasks_top;
+
/*! The total number of parts in the space. */
size_t nr_parts, size_parts;
@@ -228,6 +234,7 @@ void space_recycle_list(struct space *s, struct cell *cell_list_begin,
void space_split(struct space *s, struct cell *cells, int nr_cells,
int verbose);
void space_split_mapper(void *map_data, int num_elements, void *extra_data);
+void space_list_local_cells(struct space *s);
void space_list_cells_with_tasks(struct space *s);
void space_parts_get_cell_index(struct space *s, int *ind, int *cell_counts,
struct cell *cells, int verbose);