diff --git a/src/space.c b/src/space.c index a4f905ff575811a7010688daae88c40facf0caca..f5674b969fb0ec427320b647b8cae80f3b019b44 100644 --- a/src/space.c +++ b/src/space.c @@ -43,6 +43,7 @@ #include "lock.h" #include "minmax.h" #include "runner.h" +#include "tools.h" /* Shared sort structure. */ struct parallel_sort space_sort_struct; @@ -200,10 +201,13 @@ void space_regrid(struct space *s, double cell_max, int verbose) { "Must have at least 3 cells in each spatial dimension when periodicity " "is switched on."); -/* In MPI-Land, we're not allowed to change the top-level cell size. */ + /* In MPI-Land, changing the top-level cell size requires that the + * global partition is recomputed and the particles redistributed. + * Be prepared to do that. */ #ifdef WITH_MPI + int partition = 0; if (cdim[0] < s->cdim[0] || cdim[1] < s->cdim[1] || cdim[2] < s->cdim[2]) - error("Root-level change of cell size not allowed."); + partition = 1; #endif /* Do we need to re-build the upper-level cells? */ @@ -263,6 +267,32 @@ void space_regrid(struct space *s, double cell_max, int verbose) { cdim[2]); fflush(stdout); +#ifdef WITH_MPI + /* XXX create an engine_resplit() function to use the same method as the + * initial partition. Fake for now. */ + if (partition) { + if (s->e->nodeID == 0) + message("cell dimensions have decreased. Recalculating the " + "global partition."); + + /* Change the global partitioning. */ + int grid[3]; + factor(s->e->nr_nodes, &grid[0], &grid[1]); + factor(s->e->nr_nodes / grid[1], &grid[0], &grid[2]); + factor(grid[0] * grid[1], &grid[1], &grid[0]); + + /* Run through the cells and set their nodeID. */ + int ind[3]; + for (int k = 0; k < s->nr_cells; k++) { + c = &s->cells[k]; + for (int j = 0; j < 3; j++) ind[j] = c->loc[j] / s->dim[j] * grid[j]; + c->nodeID = ind[0] + grid[0] * (ind[1] + grid[1] * ind[2]); + } + + /* Make the proxies. */ + engine_makeproxies(s->e); + } +#endif } /* re-build upper-level cells? */ // message( "rebuilding upper-level cells took %.3f ms." , (double)(getticks() // - tic) / CPU_TPS * 1000 ); @@ -304,7 +334,7 @@ void space_regrid(struct space *s, double cell_max, int verbose) { void space_rebuild(struct space *s, double cell_max, int verbose) { - int j, k, cdim[3], nr_parts = s->nr_parts, nr_gparts = s->nr_gparts; + int j, k, cdim[3], nr_parts, nr_gparts; struct cell *restrict c, *restrict cells; struct part *restrict p; int *ind; @@ -316,7 +346,11 @@ void space_rebuild(struct space *s, double cell_max, int verbose) { /* Re-grid if necessary, or just re-set the cell data. */ space_regrid(s, cell_max, verbose); + + /* Now set local space variables. */ + nr_parts = s->nr_parts; cells = s->cells; + nr_gparts = s->nr_gparts; /* Run through the particles and get their cell index. */ // tic = getticks();