Commit bd4f4927 authored by Matthieu Schaller's avatar Matthieu Schaller
Browse files

When rebuilding just after a redistribute, skip the search for foreign particles

as there shouldn't be any.
parent f7d39bf9
......@@ -1383,6 +1383,11 @@ void engine_repartition_trigger(struct engine *e) {
#ifdef WITH_MPI
if (e->step > 2 && e->step % 5 == 0) {
e->forcerepart = 1;
return;
}
/* Do nothing if there have not been enough steps since the last
* repartition, don't want to repeat this too often or immediately after
* a repartition step. Also nothing to do when requested. */
......@@ -4483,10 +4488,12 @@ int engine_estimate_nr_tasks(struct engine *e) {
* @brief Rebuild the space and tasks.
*
* @param e The #engine.
* @param repartitioned Did we just redistribute?
* @param clean_smoothing_length_values Are we cleaning up the values of
* the smoothing lengths before building the tasks ?
*/
void engine_rebuild(struct engine *e, int clean_smoothing_length_values) {
void engine_rebuild(struct engine *e, int repartitioned,
int clean_smoothing_length_values) {
const ticks tic = getticks();
......@@ -4495,7 +4502,7 @@ void engine_rebuild(struct engine *e, int clean_smoothing_length_values) {
e->restarting = 0;
/* Re-build the space. */
space_rebuild(e->s, e->verbose);
space_rebuild(e->s, repartitioned, e->verbose);
/* Construct the list of purely local cells */
space_list_local_cells(e->s);
......@@ -4605,6 +4612,7 @@ void engine_prepare(struct engine *e) {
const ticks tic = getticks();
int drifted_all = 0;
int repartitioned = 0;
/* Unskip active tasks and check for rebuild */
if (!e->forcerebuild && !e->forcerepart && !e->restarting) engine_unskip(e);
......@@ -4629,6 +4637,7 @@ void engine_prepare(struct engine *e) {
/* And repartition */
engine_repartition(e);
repartitioned = 1;
}
/* Do we need rebuilding ? */
......@@ -4638,7 +4647,7 @@ void engine_prepare(struct engine *e) {
if (!e->restarting && !drifted_all) engine_drift_all(e);
/* And rebuild */
engine_rebuild(e, 0);
engine_rebuild(e, repartitioned, 0);
}
#ifdef SWIFT_DEBUG_CHECKS
......@@ -5168,7 +5177,7 @@ void engine_init_particles(struct engine *e, int flag_entropy_ICs,
if (e->nodeID == 0) message("Computing initial gas densities.");
/* Construct all cells and tasks to start everything */
engine_rebuild(e, clean_h_values);
engine_rebuild(e, 0, clean_h_values);
/* No time integration. We just want the density and ghosts */
engine_skip_force_and_kick(e);
......@@ -5391,7 +5400,7 @@ void engine_step(struct engine *e) {
}
/* We need some cells to exist but not the whole task stuff. */
if (e->restarting) space_rebuild(e->s, e->verbose);
if (e->restarting) space_rebuild(e->s, 0, e->verbose);
/* Move forward in time */
e->ti_old = e->ti_current;
......
......@@ -427,7 +427,7 @@ void engine_exchange_strays(struct engine *e, const size_t offset_parts,
const size_t offset_gparts, const int *ind_gpart,
size_t *Ngpart, const size_t offset_sparts,
const int *ind_spart, size_t *Nspart);
void engine_rebuild(struct engine *e, int clean_h_values);
void engine_rebuild(struct engine *e, int redistributed, int clean_h_values);
void engine_repartition(struct engine *e);
void engine_repartition_trigger(struct engine *e);
void engine_makeproxies(struct engine *e);
......
......@@ -563,10 +563,10 @@ void space_regrid(struct space *s, int verbose) {
* @brief Re-build the cells as well as the tasks.
*
* @param s The #space in which to update the cells.
* @param repartitioned Did we just repartition?
* @param verbose Print messages to stdout or not
*
*/
void space_rebuild(struct space *s, int verbose) {
void space_rebuild(struct space *s, int repartitioned, int verbose) {
const ticks tic = getticks();
......@@ -623,10 +623,19 @@ void space_rebuild(struct space *s, int verbose) {
space_sparts_get_cell_index(s, sind, cell_spart_counts,
&count_inhibited_sparts, verbose);
#ifdef SWIFT_DEBUG_CHECKS
if (repartitioned && count_inhibited_parts)
error("We just repartitioned but still found inhibited parts.");
if (repartitioned && count_inhibited_sparts)
error("We just repartitioned but still found inhibited sparts.");
if (repartitioned && count_inhibited_gparts)
error("We just repartitioned but still found inhibited gparts.");
#endif
const int local_nodeID = s->e->nodeID;
/* Move non-local parts and inhibited parts to the end of the list. */
if (s->e->nr_nodes > 1 || count_inhibited_parts > 0) {
if (!repartitioned && (s->e->nr_nodes > 1 || count_inhibited_parts > 0)) {
for (size_t k = 0; k < nr_parts; /* void */) {
/* Inhibited particle or foreign particle */
......@@ -677,7 +686,7 @@ void space_rebuild(struct space *s, int verbose) {
#endif /* SWIFT_DEBUG_CHECKS */
/* Move non-local sparts and inhibited sparts to the end of the list. */
if (s->e->nr_nodes > 1 || count_inhibited_sparts > 0) {
if (!repartitioned && (s->e->nr_nodes > 1 || count_inhibited_sparts > 0)) {
for (size_t k = 0; k < nr_sparts; /* void */) {
/* Inhibited particle or foreign particle */
......@@ -726,7 +735,7 @@ void space_rebuild(struct space *s, int verbose) {
#endif /* SWIFT_DEBUG_CHECKS */
/* Move non-local gparts and inhibited parts to the end of the list. */
if (s->e->nr_nodes > 1 || count_inhibited_gparts > 0) {
if (!repartitioned && (s->e->nr_nodes > 1 || count_inhibited_gparts > 0)) {
for (size_t k = 0; k < nr_gparts; /* void */) {
/* Inhibited particle or foreign particle */
......@@ -782,18 +791,31 @@ void space_rebuild(struct space *s, int verbose) {
#ifdef WITH_MPI
/* Exchange the strays, note that this potentially re-allocates
the parts arrays. */
size_t nr_parts_exchanged = s->nr_parts - nr_parts;
size_t nr_gparts_exchanged = s->nr_gparts - nr_gparts;
size_t nr_sparts_exchanged = s->nr_sparts - nr_sparts;
engine_exchange_strays(s->e, nr_parts, &ind[nr_parts], &nr_parts_exchanged,
nr_gparts, &gind[nr_gparts], &nr_gparts_exchanged,
nr_sparts, &sind[nr_sparts], &nr_sparts_exchanged);
/* Set the new particle counts. */
s->nr_parts = nr_parts + nr_parts_exchanged;
s->nr_gparts = nr_gparts + nr_gparts_exchanged;
s->nr_sparts = nr_sparts + nr_sparts_exchanged;
the parts arrays. This can be skipped if we just repartitioned aspace
there should be no strays */
if (!repartitioned) {
size_t nr_parts_exchanged = s->nr_parts - nr_parts;
size_t nr_gparts_exchanged = s->nr_gparts - nr_gparts;
size_t nr_sparts_exchanged = s->nr_sparts - nr_sparts;
engine_exchange_strays(s->e, nr_parts, &ind[nr_parts], &nr_parts_exchanged,
nr_gparts, &gind[nr_gparts], &nr_gparts_exchanged,
nr_sparts, &sind[nr_sparts], &nr_sparts_exchanged);
/* Set the new particle counts. */
s->nr_parts = nr_parts + nr_parts_exchanged;
s->nr_gparts = nr_gparts + nr_gparts_exchanged;
s->nr_sparts = nr_sparts + nr_sparts_exchanged;
} else {
#ifdef SWIFT_DEBUG_CHECKS
if (s->nr_parts != nr_parts)
error("Number of parts changing after repartition");
if (s->nr_sparts != nr_sparts)
error("Number of sparts changing after repartition");
if (s->nr_gparts != nr_gparts)
error("Number of gparts changing after repartition");
#endif
}
/* Clear non-local cell counts. */
for (int k = 0; k < s->nr_cells; k++) {
......
......@@ -232,7 +232,7 @@ void space_map_parts_xparts(struct space *s,
struct cell *c));
void space_map_cells_post(struct space *s, int full,
void (*fun)(struct cell *c, void *data), void *data);
void space_rebuild(struct space *s, int verbose);
void space_rebuild(struct space *s, int repartitioned, int verbose);
void space_recycle(struct space *s, struct cell *c);
void space_recycle_list(struct space *s, struct cell *cell_list_begin,
struct cell *cell_list_end,
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment