diff --git a/src/engine.c b/src/engine.c
index a56adda16321cf36c14179dc9a9314de5420a305..6e0d788261599f02890db439d6953983e585944e 100644
--- a/src/engine.c
+++ b/src/engine.c
@@ -1383,6 +1383,11 @@ void engine_repartition_trigger(struct engine *e) {
#ifdef WITH_MPI
+ if (e->step > 2 && e->step % 5 == 0) {
+ e->forcerepart = 1;
+ return;
+ }
+
/* Do nothing if there have not been enough steps since the last
* repartition, don't want to repeat this too often or immediately after
* a repartition step. Also nothing to do when requested. */
@@ -4483,10 +4488,12 @@ int engine_estimate_nr_tasks(struct engine *e) {
* @brief Rebuild the space and tasks.
*
* @param e The #engine.
+ * @param repartitioned Did we just redistribute?
* @param clean_smoothing_length_values Are we cleaning up the values of
* the smoothing lengths before building the tasks ?
*/
-void engine_rebuild(struct engine *e, int clean_smoothing_length_values) {
+void engine_rebuild(struct engine *e, int repartitioned,
+ int clean_smoothing_length_values) {
const ticks tic = getticks();
@@ -4495,7 +4502,7 @@ void engine_rebuild(struct engine *e, int clean_smoothing_length_values) {
e->restarting = 0;
/* Re-build the space. */
- space_rebuild(e->s, e->verbose);
+ space_rebuild(e->s, repartitioned, e->verbose);
/* Construct the list of purely local cells */
space_list_local_cells(e->s);
@@ -4605,6 +4612,7 @@ void engine_prepare(struct engine *e) {
const ticks tic = getticks();
int drifted_all = 0;
+ int repartitioned = 0;
/* Unskip active tasks and check for rebuild */
if (!e->forcerebuild && !e->forcerepart && !e->restarting) engine_unskip(e);
@@ -4629,6 +4637,7 @@ void engine_prepare(struct engine *e) {
/* And repartition */
engine_repartition(e);
+ repartitioned = 1;
}
/* Do we need rebuilding ? */
@@ -4638,7 +4647,7 @@ void engine_prepare(struct engine *e) {
if (!e->restarting && !drifted_all) engine_drift_all(e);
/* And rebuild */
- engine_rebuild(e, 0);
+ engine_rebuild(e, repartitioned, 0);
}
#ifdef SWIFT_DEBUG_CHECKS
@@ -5168,7 +5177,7 @@ void engine_init_particles(struct engine *e, int flag_entropy_ICs,
if (e->nodeID == 0) message("Computing initial gas densities.");
/* Construct all cells and tasks to start everything */
- engine_rebuild(e, clean_h_values);
+ engine_rebuild(e, 0, clean_h_values);
/* No time integration. We just want the density and ghosts */
engine_skip_force_and_kick(e);
@@ -5391,7 +5400,7 @@ void engine_step(struct engine *e) {
}
/* We need some cells to exist but not the whole task stuff. */
- if (e->restarting) space_rebuild(e->s, e->verbose);
+ if (e->restarting) space_rebuild(e->s, 0, e->verbose);
/* Move forward in time */
e->ti_old = e->ti_current;
diff --git a/src/engine.h b/src/engine.h
index 5c31fcc51e11351dade2075deaa40aa35848737d..7505a643cc6e911457192d734abb5db7f08512b3 100644
--- a/src/engine.h
+++ b/src/engine.h
@@ -427,7 +427,7 @@ void engine_exchange_strays(struct engine *e, const size_t offset_parts,
const size_t offset_gparts, const int *ind_gpart,
size_t *Ngpart, const size_t offset_sparts,
const int *ind_spart, size_t *Nspart);
-void engine_rebuild(struct engine *e, int clean_h_values);
+void engine_rebuild(struct engine *e, int redistributed, int clean_h_values);
void engine_repartition(struct engine *e);
void engine_repartition_trigger(struct engine *e);
void engine_makeproxies(struct engine *e);
diff --git a/src/space.c b/src/space.c
index 5ee6e4c3c1934e6c3c6c4995e59ad79140c4139d..323dfdd7228c524ad446a64f9b3967c59d6f4552 100644
--- a/src/space.c
+++ b/src/space.c
@@ -563,10 +563,10 @@ void space_regrid(struct space *s, int verbose) {
* @brief Re-build the cells as well as the tasks.
*
* @param s The #space in which to update the cells.
+ * @param repartitioned Did we just repartition?
* @param verbose Print messages to stdout or not
- *
*/
-void space_rebuild(struct space *s, int verbose) {
+void space_rebuild(struct space *s, int repartitioned, int verbose) {
const ticks tic = getticks();
@@ -623,10 +623,19 @@ void space_rebuild(struct space *s, int verbose) {
space_sparts_get_cell_index(s, sind, cell_spart_counts,
&count_inhibited_sparts, verbose);
+#ifdef SWIFT_DEBUG_CHECKS
+ if (repartitioned && count_inhibited_parts)
+ error("We just repartitioned but still found inhibited parts.");
+ if (repartitioned && count_inhibited_sparts)
+ error("We just repartitioned but still found inhibited sparts.");
+ if (repartitioned && count_inhibited_gparts)
+ error("We just repartitioned but still found inhibited gparts.");
+#endif
+
const int local_nodeID = s->e->nodeID;
/* Move non-local parts and inhibited parts to the end of the list. */
- if (s->e->nr_nodes > 1 || count_inhibited_parts > 0) {
+ if (!repartitioned && (s->e->nr_nodes > 1 || count_inhibited_parts > 0)) {
for (size_t k = 0; k < nr_parts; /* void */) {
/* Inhibited particle or foreign particle */
@@ -677,7 +686,7 @@ void space_rebuild(struct space *s, int verbose) {
#endif /* SWIFT_DEBUG_CHECKS */
/* Move non-local sparts and inhibited sparts to the end of the list. */
- if (s->e->nr_nodes > 1 || count_inhibited_sparts > 0) {
+ if (!repartitioned && (s->e->nr_nodes > 1 || count_inhibited_sparts > 0)) {
for (size_t k = 0; k < nr_sparts; /* void */) {
/* Inhibited particle or foreign particle */
@@ -726,7 +735,7 @@ void space_rebuild(struct space *s, int verbose) {
#endif /* SWIFT_DEBUG_CHECKS */
/* Move non-local gparts and inhibited parts to the end of the list. */
- if (s->e->nr_nodes > 1 || count_inhibited_gparts > 0) {
+ if (!repartitioned && (s->e->nr_nodes > 1 || count_inhibited_gparts > 0)) {
for (size_t k = 0; k < nr_gparts; /* void */) {
/* Inhibited particle or foreign particle */
@@ -782,18 +791,31 @@ void space_rebuild(struct space *s, int verbose) {
#ifdef WITH_MPI
/* Exchange the strays, note that this potentially re-allocates
- the parts arrays. */
- size_t nr_parts_exchanged = s->nr_parts - nr_parts;
- size_t nr_gparts_exchanged = s->nr_gparts - nr_gparts;
- size_t nr_sparts_exchanged = s->nr_sparts - nr_sparts;
- engine_exchange_strays(s->e, nr_parts, &ind[nr_parts], &nr_parts_exchanged,
- nr_gparts, &gind[nr_gparts], &nr_gparts_exchanged,
- nr_sparts, &sind[nr_sparts], &nr_sparts_exchanged);
-
- /* Set the new particle counts. */
- s->nr_parts = nr_parts + nr_parts_exchanged;
- s->nr_gparts = nr_gparts + nr_gparts_exchanged;
- s->nr_sparts = nr_sparts + nr_sparts_exchanged;
+ the parts arrays. This can be skipped if we just repartitioned aspace
+ there should be no strays */
+ if (!repartitioned) {
+
+ size_t nr_parts_exchanged = s->nr_parts - nr_parts;
+ size_t nr_gparts_exchanged = s->nr_gparts - nr_gparts;
+ size_t nr_sparts_exchanged = s->nr_sparts - nr_sparts;
+ engine_exchange_strays(s->e, nr_parts, &ind[nr_parts], &nr_parts_exchanged,
+ nr_gparts, &gind[nr_gparts], &nr_gparts_exchanged,
+ nr_sparts, &sind[nr_sparts], &nr_sparts_exchanged);
+
+ /* Set the new particle counts. */
+ s->nr_parts = nr_parts + nr_parts_exchanged;
+ s->nr_gparts = nr_gparts + nr_gparts_exchanged;
+ s->nr_sparts = nr_sparts + nr_sparts_exchanged;
+ } else {
+#ifdef SWIFT_DEBUG_CHECKS
+ if (s->nr_parts != nr_parts)
+ error("Number of parts changing after repartition");
+ if (s->nr_sparts != nr_sparts)
+ error("Number of sparts changing after repartition");
+ if (s->nr_gparts != nr_gparts)
+ error("Number of gparts changing after repartition");
+#endif
+ }
/* Clear non-local cell counts. */
for (int k = 0; k < s->nr_cells; k++) {
diff --git a/src/space.h b/src/space.h
index 5b5be99fba347adcb7857bf567b7a9663d1a588f..7fb9cf164095d8618bf8b42d950f41a02f82858d 100644
--- a/src/space.h
+++ b/src/space.h
@@ -232,7 +232,7 @@ void space_map_parts_xparts(struct space *s,
struct cell *c));
void space_map_cells_post(struct space *s, int full,
void (*fun)(struct cell *c, void *data), void *data);
-void space_rebuild(struct space *s, int verbose);
+void space_rebuild(struct space *s, int repartitioned, int verbose);
void space_recycle(struct space *s, struct cell *c);
void space_recycle_list(struct space *s, struct cell *cell_list_begin,
struct cell *cell_list_end,