Skip to content
Snippets Groups Projects
Commit 1017bd73 authored by Matthieu Schaller's avatar Matthieu Schaller
Browse files

First attempt at MPI distribution of gparts

parent 7c9293d2
No related branches found
No related tags found
2 merge requests!136Master,!116Basic implementation of gparts
...@@ -308,11 +308,6 @@ void space_regrid(struct space *s, double cell_max, int verbose) { ...@@ -308,11 +308,6 @@ void space_regrid(struct space *s, double cell_max, int verbose) {
void space_rebuild(struct space *s, double cell_max, int verbose) { void space_rebuild(struct space *s, double cell_max, int verbose) {
int j, k, cdim[3], nr_parts = s->nr_parts, nr_gparts = s->nr_gparts;
struct cell *restrict c, *restrict cells;
struct part *restrict p;
size_t *ind;
double ih[3], dim[3];
ticks tic = getticks(); ticks tic = getticks();
/* Be verbose about this. */ /* Be verbose about this. */
...@@ -320,13 +315,13 @@ void space_rebuild(struct space *s, double cell_max, int verbose) { ...@@ -320,13 +315,13 @@ void space_rebuild(struct space *s, double cell_max, int verbose) {
/* Re-grid if necessary, or just re-set the cell data. */ /* Re-grid if necessary, or just re-set the cell data. */
space_regrid(s, cell_max, verbose); space_regrid(s, cell_max, verbose);
cells = s->cells;
/* Run through the particles and get their cell index. */ int nr_parts = s->nr_parts;
// tic = getticks(); int nr_gparts = s->nr_gparts;
const size_t ind_size = s->size_parts; struct cell *restrict cells = s->cells;
if ((ind = (size_t *)malloc(sizeof(size_t) * ind_size)) == NULL)
error("Failed to allocate temporary particle indices."); double ih[3], dim[3];
int cdim[3];
ih[0] = s->ih[0]; ih[0] = s->ih[0];
ih[1] = s->ih[1]; ih[1] = s->ih[1];
ih[2] = s->ih[2]; ih[2] = s->ih[2];
...@@ -336,9 +331,16 @@ void space_rebuild(struct space *s, double cell_max, int verbose) { ...@@ -336,9 +331,16 @@ void space_rebuild(struct space *s, double cell_max, int verbose) {
cdim[0] = s->cdim[0]; cdim[0] = s->cdim[0];
cdim[1] = s->cdim[1]; cdim[1] = s->cdim[1];
cdim[2] = s->cdim[2]; cdim[2] = s->cdim[2];
for (k = 0; k < nr_parts; k++) {
p = &s->parts[k]; /* Run through the particles and get their cell index. */
for (j = 0; j < 3; j++) // tic = getticks();
const size_t ind_size = s->size_parts;
size_t *ind;
if ((ind = (size_t *)malloc(sizeof(size_t) * ind_size)) == NULL)
error("Failed to allocate temporary particle indices.");
for (int k = 0; k < nr_parts; k++) {
struct part *restrict p = &s->parts[k];
for (int j = 0; j < 3; j++)
if (p->x[j] < 0.0) if (p->x[j] < 0.0)
p->x[j] += dim[j]; p->x[j] += dim[j];
else if (p->x[j] >= dim[j]) else if (p->x[j] >= dim[j])
...@@ -352,8 +354,8 @@ void space_rebuild(struct space *s, double cell_max, int verbose) { ...@@ -352,8 +354,8 @@ void space_rebuild(struct space *s, double cell_max, int verbose) {
#ifdef WITH_MPI #ifdef WITH_MPI
/* Move non-local parts to the end of the list. */ /* Move non-local parts to the end of the list. */
int nodeID = s->e->nodeID; const int nodeID = s->e->nodeID;
for (k = 0; k < nr_parts; k++) for (int k = 0; k < nr_parts; k++)
if (cells[ind[k]].nodeID != nodeID) { if (cells[ind[k]].nodeID != nodeID) {
cells[ind[k]].count -= 1; cells[ind[k]].count -= 1;
nr_parts -= 1; nr_parts -= 1;
...@@ -385,8 +387,8 @@ void space_rebuild(struct space *s, double cell_max, int verbose) { ...@@ -385,8 +387,8 @@ void space_rebuild(struct space *s, double cell_max, int verbose) {
} }
/* Assign each particle to its cell. */ /* Assign each particle to its cell. */
for (k = nr_parts; k < s->nr_parts; k++) { for (int k = nr_parts; k < s->nr_parts; k++) {
p = &s->parts[k]; struct part *p = &s->parts[k];
ind[k] = ind[k] =
cell_getid(cdim, p->x[0] * ih[0], p->x[1] * ih[1], p->x[2] * ih[2]); cell_getid(cdim, p->x[0] * ih[0], p->x[1] * ih[1], p->x[2] * ih[2]);
cells[ind[k]].count += 1; cells[ind[k]].count += 1;
...@@ -401,7 +403,7 @@ void space_rebuild(struct space *s, double cell_max, int verbose) { ...@@ -401,7 +403,7 @@ void space_rebuild(struct space *s, double cell_max, int verbose) {
space_parts_sort(s, ind, nr_parts, 0, s->nr_cells - 1, verbose); space_parts_sort(s, ind, nr_parts, 0, s->nr_cells - 1, verbose);
/* Re-link the gparts. */ /* Re-link the gparts. */
for (k = 0; k < nr_parts; k++) for (int k = 0; k < nr_parts; k++)
if (s->parts[k].gpart != NULL) s->parts[k].gpart->part = &s->parts[k]; if (s->parts[k].gpart != NULL) s->parts[k].gpart->part = &s->parts[k];
/* Verify space_sort_struct. */ /* Verify space_sort_struct. */
...@@ -419,41 +421,88 @@ void space_rebuild(struct space *s, double cell_max, int verbose) { ...@@ -419,41 +421,88 @@ void space_rebuild(struct space *s, double cell_max, int verbose) {
/* Run through the gravity particles and get their cell index. */ /* Run through the gravity particles and get their cell index. */
// tic = getticks(); // tic = getticks();
if ((ind = (size_t *)malloc(sizeof(size_t) * s->size_gparts)) == NULL) const size_t gind_size = s->size_gparts;
error("Failed to allocate temporary particle indices."); size_t *gind;
for (k = 0; k < nr_gparts; k++) { if ((gind = (size_t *)malloc(sizeof(size_t) * gind_size)) == NULL)
error("Failed to allocate temporary g-particle indices.");
for (int k = 0; k < nr_gparts; k++) {
struct gpart *gp = &s->gparts[k]; struct gpart *gp = &s->gparts[k];
for (j = 0; j < 3; j++) for (int j = 0; j < 3; j++)
if (gp->x[j] < 0.0) if (gp->x[j] < 0.0)
gp->x[j] += dim[j]; gp->x[j] += dim[j];
else if (gp->x[j] >= dim[j]) else if (gp->x[j] >= dim[j])
gp->x[j] -= dim[j]; gp->x[j] -= dim[j];
ind[k] = gind[k] =
cell_getid(cdim, gp->x[0] * ih[0], gp->x[1] * ih[1], gp->x[2] * ih[2]); cell_getid(cdim, gp->x[0] * ih[0], gp->x[1] * ih[1], gp->x[2] * ih[2]);
cells[ind[k]].gcount++; cells[gind[k]].gcount++;
} }
// message( "getting particle indices took %.3f %s." , // message( "getting particle indices took %.3f %s." ,
// clocks_from_ticks(getticks() - tic), clocks_getunit()); // clocks_from_ticks(getticks() - tic), clocks_getunit());
#ifdef WITH_MPI
/* TODO: Here we should exchange the gparts as well! */ /* TODO: Here we should exchange the gparts as well! */
if (gcount > 0) error("gpart exchange not yet implemented");
/* Move non-local gparts to the end of the list. */
for (int k = 0; k < nr_gparts; k++)
if (cells[ind[k]].nodeID != nodeID) {
cells[ind[k]].gcount -= 1;
nr_gparts -= 1;
struct gpart tp = s->gparts[k];
s->gparts[k] = s->gparts[nr_gparts];
s->gparts[nr_gparts] = tp;
int t = ind[k];
ind[k] = ind[nr_gparts];
ind[nr_gparts] = t;
}
/* Exchange the strays, note that this potentially re-allocates
the parts arrays. */
s->nr_gparts =
nr_gparts + engine_exchange_strays(s->e, nr_gparts, &ind[nr_gparts],
s->nr_gparts - nr_gparts);
/* Re-allocate the index array if needed.. */
if (s->nr_gparts > gind_size) {
size_t *gind_new;
if ((gind_new = (size_t *)malloc(sizeof(size_t) * s->nr_gparts)) == NULL)
error("Failed to allocate temporary g-particle indices.");
memcpy(gind_new, gind, sizeof(size_t) * nr_gparts);
free(gind);
gind = gind_new;
}
/* Assign each particle to its cell. */
for (int k = nr_gparts; k < s->gnr_parts; k++) {
struct gpart *p = &s->gparts[k];
gind[k] =
cell_getid(cdim, p->x[0] * ih[0], p->x[1] * ih[1], p->x[2] * ih[2]);
cells[gind[k]].count += 1;
/* if ( cells[ ind[k] ].nodeID != nodeID )
error( "Received part that does not belong to me (nodeID=%i)." , cells[
ind[k] ].nodeID ); */
}
nr_gparts = s->nr_gparts;
#endif
/* Sort the parts according to their cells. */ /* Sort the parts according to their cells. */
space_gparts_sort(s->gparts, ind, nr_gparts, 0, s->nr_cells - 1); space_gparts_sort(s->gparts, gind, nr_gparts, 0, s->nr_cells - 1);
/* Re-link the parts. */ /* Re-link the parts. */
for (k = 0; k < nr_gparts; k++) for (int k = 0; k < nr_gparts; k++)
if (s->gparts[k].id > 0) s->gparts[k].part->gpart = &s->gparts[k]; if (s->gparts[k].id > 0) s->gparts[k].part->gpart = &s->gparts[k];
/* We no longer need the indices as of here. */ /* We no longer need the indices as of here. */
free(ind); free(gind);
/* Hook the cells up to the parts. */ /* Hook the cells up to the parts. */
// tic = getticks(); // tic = getticks();
struct part *finger = s->parts; struct part *finger = s->parts;
struct xpart *xfinger = s->xparts; struct xpart *xfinger = s->xparts;
struct gpart *gfinger = s->gparts; struct gpart *gfinger = s->gparts;
for (k = 0; k < s->nr_cells; k++) { for (int k = 0; k < s->nr_cells; k++) {
c = &cells[k]; struct cell *restrict c = &cells[k];
c->parts = finger; c->parts = finger;
c->xparts = xfinger; c->xparts = xfinger;
c->gparts = gfinger; c->gparts = gfinger;
...@@ -1100,9 +1149,11 @@ void space_do_split(struct space *s, struct cell *c) { ...@@ -1100,9 +1149,11 @@ void space_do_split(struct space *s, struct cell *c) {
/* Set ownership according to the start of the parts array. */ /* Set ownership according to the start of the parts array. */
if (count > 0) if (count > 0)
c->owner = ((c->parts - s->parts) % s->nr_parts) * s->nr_queues / s->nr_parts; c->owner =
((c->parts - s->parts) % s->nr_parts) * s->nr_queues / s->nr_parts;
else else
c->owner = ((c->gparts - s->gparts) % s->nr_gparts) * s->nr_queues / s->nr_gparts; c->owner =
((c->gparts - s->gparts) % s->nr_gparts) * s->nr_queues / s->nr_gparts;
} }
/** /**
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment