Commit f6afa976 authored by Matthieu Schaller's avatar Matthieu Schaller
Browse files

Construct a list of local top-level cells that are then used in the unskipping

parent d408db13
...@@ -544,8 +544,8 @@ int main(int argc, char *argv[]) { ...@@ -544,8 +544,8 @@ int main(int argc, char *argv[]) {
/* Initialize the space with these data. */ /* Initialize the space with these data. */
if (myrank == 0) clocks_gettime(&tic); if (myrank == 0) clocks_gettime(&tic);
struct space s; struct space s;
space_init(&s, params, dim, parts, gparts, sparts, Ngas, Ngpart, Nspart, space_init(&s, myrank, params, dim, parts, gparts, sparts, Ngas, Ngpart,
periodic, replicate, with_self_gravity, talking, dry_run); Nspart, periodic, replicate, with_self_gravity, talking, dry_run);
if (myrank == 0) { if (myrank == 0) {
clocks_gettime(&toc); clocks_gettime(&toc);
message("space_init took %.3f %s.", clocks_diff(&tic, &toc), message("space_init took %.3f %s.", clocks_diff(&tic, &toc),
......
...@@ -3807,8 +3807,8 @@ void engine_unskip(struct engine *e) { ...@@ -3807,8 +3807,8 @@ void engine_unskip(struct engine *e) {
const ticks tic = getticks(); const ticks tic = getticks();
/* Activate all the regular tasks */ /* Activate all the regular tasks */
threadpool_map(&e->threadpool, runner_do_unskip_mapper, e->s->cells_top, threadpool_map(&e->threadpool, runner_do_unskip_mapper, e->s->local_cells_top,
e->s->nr_cells, sizeof(struct cell), 1, e); e->s->nr_local_cells, sizeof(int), 1, e);
/* And the top level gravity FFT one */ /* And the top level gravity FFT one */
if (e->s->periodic && (e->policy & engine_policy_self_gravity)) if (e->s->periodic && (e->policy & engine_policy_self_gravity))
......
...@@ -868,10 +868,11 @@ void runner_do_unskip_mapper(void *map_data, int num_elements, ...@@ -868,10 +868,11 @@ void runner_do_unskip_mapper(void *map_data, int num_elements,
void *extra_data) { void *extra_data) {
struct engine *e = (struct engine *)extra_data; struct engine *e = (struct engine *)extra_data;
struct cell *cells = (struct cell *)map_data; struct space *s = e->s;
int *local_cells = (int *)map_data;
for (int ind = 0; ind < num_elements; ind++) { for (int ind = 0; ind < num_elements; ind++) {
struct cell *c = &cells[ind]; struct cell *c = &s->cells_top[local_cells[ind]];
if (c != NULL) runner_do_unskip(c, e); if (c != NULL) runner_do_unskip(c, e);
} }
} }
......
...@@ -389,6 +389,7 @@ void space_regrid(struct space *s, int verbose) { ...@@ -389,6 +389,7 @@ void space_regrid(struct space *s, int verbose) {
/* Free the old cells, if they were allocated. */ /* Free the old cells, if they were allocated. */
if (s->cells_top != NULL) { if (s->cells_top != NULL) {
space_free_cells(s); space_free_cells(s);
free(s->local_cells_top);
free(s->cells_top); free(s->cells_top);
free(s->multipoles_top); free(s->multipoles_top);
} }
...@@ -420,6 +421,12 @@ void space_regrid(struct space *s, int verbose) { ...@@ -420,6 +421,12 @@ void space_regrid(struct space *s, int verbose) {
bzero(s->multipoles_top, s->nr_cells * sizeof(struct gravity_tensors)); bzero(s->multipoles_top, s->nr_cells * sizeof(struct gravity_tensors));
} }
/* Allocate the indices of local cells */
if (posix_memalign((void *)&s->local_cells_top, SWIFT_STRUCT_ALIGNMENT,
s->nr_cells * sizeof(int)) != 0)
error("Failed to allocate indices of local top-level cells.");
bzero(s->local_cells_top, s->nr_cells * sizeof(int));
/* Set the cells' locks */ /* Set the cells' locks */
for (int k = 0; k < s->nr_cells; k++) { for (int k = 0; k < s->nr_cells; k++) {
if (lock_init(&s->cells_top[k].lock) != 0) if (lock_init(&s->cells_top[k].lock) != 0)
...@@ -496,6 +503,16 @@ void space_regrid(struct space *s, int verbose) { ...@@ -496,6 +503,16 @@ void space_regrid(struct space *s, int verbose) {
} }
#endif /* WITH_MPI */ #endif /* WITH_MPI */
/* Let's rebuild the list of local top-level cells */
s->nr_local_cells = 0;
for (int i = 0; i < s->nr_cells; ++i)
if (s->cells_top[i].nodeID == s->nodeID) {
s->local_cells_top[s->nr_local_cells] = i;
++s->nr_local_cells;
}
if (verbose)
message("Have %d local cells (total=%d)", s->nr_local_cells, s->nr_cells);
// message( "rebuilding upper-level cells took %.3f %s." , // message( "rebuilding upper-level cells took %.3f %s." ,
// clocks_from_ticks(double)(getticks() - tic), clocks_getunit()); // clocks_from_ticks(double)(getticks() - tic), clocks_getunit());
...@@ -2652,7 +2669,7 @@ void space_init_sparts(struct space *s) { ...@@ -2652,7 +2669,7 @@ void space_init_sparts(struct space *s) {
* parts with a cutoff below half the cell width are then split * parts with a cutoff below half the cell width are then split
* recursively. * recursively.
*/ */
void space_init(struct space *s, const struct swift_params *params, void space_init(struct space *s, int nodeID, const struct swift_params *params,
double dim[3], struct part *parts, struct gpart *gparts, double dim[3], struct part *parts, struct gpart *gparts,
struct spart *sparts, size_t Npart, size_t Ngpart, struct spart *sparts, size_t Npart, size_t Ngpart,
size_t Nspart, int periodic, int replicate, int gravity, size_t Nspart, int periodic, int replicate, int gravity,
...@@ -2662,6 +2679,7 @@ void space_init(struct space *s, const struct swift_params *params, ...@@ -2662,6 +2679,7 @@ void space_init(struct space *s, const struct swift_params *params,
bzero(s, sizeof(struct space)); bzero(s, sizeof(struct space));
/* Store everything in the space. */ /* Store everything in the space. */
s->nodeID = nodeID;
s->dim[0] = dim[0]; s->dim[0] = dim[0];
s->dim[1] = dim[1]; s->dim[1] = dim[1];
s->dim[2] = dim[2]; s->dim[2] = dim[2];
...@@ -3039,6 +3057,7 @@ void space_clean(struct space *s) { ...@@ -3039,6 +3057,7 @@ void space_clean(struct space *s) {
for (int i = 0; i < s->nr_cells; ++i) cell_clean(&s->cells_top[i]); for (int i = 0; i < s->nr_cells; ++i) cell_clean(&s->cells_top[i]);
free(s->cells_top); free(s->cells_top);
free(s->multipoles_top); free(s->multipoles_top);
free(s->local_cells_top);
free(s->parts); free(s->parts);
free(s->xparts); free(s->xparts);
free(s->gparts); free(s->gparts);
......
...@@ -74,6 +74,9 @@ struct space { ...@@ -74,6 +74,9 @@ struct space {
/*! Extra space information needed for some hydro schemes. */ /*! Extra space information needed for some hydro schemes. */
struct hydro_space hs; struct hydro_space hs;
/*! The MPI rank of this space */
int nodeID;
/*! Are we doing gravity? */ /*! Are we doing gravity? */
int gravity; int gravity;
...@@ -101,6 +104,9 @@ struct space { ...@@ -101,6 +104,9 @@ struct space {
/*! Total number of cells (top- and sub-) */ /*! Total number of cells (top- and sub-) */
int tot_cells; int tot_cells;
/*! Number of *local* top-level cells */
int nr_local_cells;
/*! The (level 0) cells themselves. */ /*! The (level 0) cells themselves. */
struct cell *cells_top; struct cell *cells_top;
...@@ -113,6 +119,9 @@ struct space { ...@@ -113,6 +119,9 @@ struct space {
/*! Buffer of unused multipoles for the sub-cells. */ /*! Buffer of unused multipoles for the sub-cells. */
struct gravity_tensors *multipoles_sub; struct gravity_tensors *multipoles_sub;
/*! The indices of the *local* top-level cells */
int *local_cells_top;
/*! The total number of parts in the space. */ /*! The total number of parts in the space. */
size_t nr_parts, size_parts; size_t nr_parts, size_parts;
...@@ -173,7 +182,7 @@ void space_sparts_sort(struct space *s, int *ind, size_t N, int min, int max, ...@@ -173,7 +182,7 @@ void space_sparts_sort(struct space *s, int *ind, size_t N, int min, int max,
void space_getcells(struct space *s, int nr_cells, struct cell **cells); void space_getcells(struct space *s, int nr_cells, struct cell **cells);
int space_getsid(struct space *s, struct cell **ci, struct cell **cj, int space_getsid(struct space *s, struct cell **ci, struct cell **cj,
double *shift); double *shift);
void space_init(struct space *s, const struct swift_params *params, void space_init(struct space *s, int nodeID, const struct swift_params *params,
double dim[3], struct part *parts, struct gpart *gparts, double dim[3], struct part *parts, struct gpart *gparts,
struct spart *sparts, size_t Npart, size_t Ngpart, struct spart *sparts, size_t Npart, size_t Ngpart,
size_t Nspart, int periodic, int replicate, int gravity, size_t Nspart, int periodic, int replicate, int gravity,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment