Commit a8584ba9 authored by Matthieu Schaller's avatar Matthieu Schaller
Browse files

Also send dx_max_part and dx_max_gpart alongside ti_end_min at the end of a...

Also send dx_max_part and dx_max_gpart alongside ti_end_min at the end of a time-step when running with MPI.
parent bb5ea549
...@@ -82,73 +82,6 @@ int cell_getsize(struct cell *c) { ...@@ -82,73 +82,6 @@ int cell_getsize(struct cell *c) {
return count; return count;
} }
/**
* @brief Unpack the data of a given cell and its sub-cells.
*
* @param pc An array of packed #pcell.
* @param c The #cell in which to unpack the #pcell.
* @param s The #space in which the cells are created.
*
* @return The number of cells created.
*/
int cell_unpack(struct pcell *pc, struct cell *c, struct space *s) {
#ifdef WITH_MPI
/* Unpack the current pcell. */
c->h_max = pc->h_max;
c->ti_end_min = pc->ti_end_min;
c->ti_end_max = pc->ti_end_max;
c->ti_old_part = pc->ti_old_part;
c->ti_old_gpart = pc->ti_old_gpart;
c->count = pc->count;
c->gcount = pc->gcount;
c->scount = pc->scount;
c->tag = pc->tag;
/* Number of new cells created. */
int count = 1;
/* Fill the progeny recursively, depth-first. */
for (int k = 0; k < 8; k++)
if (pc->progeny[k] >= 0) {
struct cell *temp;
space_getcells(s, 1, &temp);
temp->count = 0;
temp->gcount = 0;
temp->scount = 0;
temp->loc[0] = c->loc[0];
temp->loc[1] = c->loc[1];
temp->loc[2] = c->loc[2];
temp->width[0] = c->width[0] / 2;
temp->width[1] = c->width[1] / 2;
temp->width[2] = c->width[2] / 2;
temp->dmin = c->dmin / 2;
if (k & 4) temp->loc[0] += temp->width[0];
if (k & 2) temp->loc[1] += temp->width[1];
if (k & 1) temp->loc[2] += temp->width[2];
temp->depth = c->depth + 1;
temp->split = 0;
temp->dx_max_part = 0.f;
temp->dx_max_gpart = 0.f;
temp->dx_max_sort = 0.f;
temp->nodeID = c->nodeID;
temp->parent = c;
c->progeny[k] = temp;
c->split = 1;
count += cell_unpack(&pc[pc->progeny[k]], temp, s);
}
/* Return the total number of unpacked cells. */
c->pcell_size = count;
return count;
#else
error("SWIFT was not compiled with MPI support.");
return 0;
#endif
}
/** /**
* @brief Link the cells recursively to the given #part array. * @brief Link the cells recursively to the given #part array.
* *
...@@ -233,7 +166,7 @@ int cell_link_sparts(struct cell *c, struct spart *sparts) { ...@@ -233,7 +166,7 @@ int cell_link_sparts(struct cell *c, struct spart *sparts) {
* *
* @return The number of packed cells. * @return The number of packed cells.
*/ */
int cell_pack(struct cell *c, struct pcell *pc) { int cell_pack(struct cell *restrict c, struct pcell *restrict pc) {
#ifdef WITH_MPI #ifdef WITH_MPI
...@@ -267,26 +200,95 @@ int cell_pack(struct cell *c, struct pcell *pc) { ...@@ -267,26 +200,95 @@ int cell_pack(struct cell *c, struct pcell *pc) {
#endif #endif
} }
/**
* @brief Unpack the data of a given cell and its sub-cells.
*
* @param pc An array of packed #pcell.
* @param c The #cell in which to unpack the #pcell.
* @param s The #space in which the cells are created.
*
* @return The number of cells created.
*/
int cell_unpack(struct pcell *restrict pc, struct cell *restrict c, struct space *restrict s) {
#ifdef WITH_MPI
/* Unpack the current pcell. */
c->h_max = pc->h_max;
c->ti_end_min = pc->ti_end_min;
c->ti_end_max = pc->ti_end_max;
c->ti_old_part = pc->ti_old_part;
c->ti_old_gpart = pc->ti_old_gpart;
c->count = pc->count;
c->gcount = pc->gcount;
c->scount = pc->scount;
c->tag = pc->tag;
/* Number of new cells created. */
int count = 1;
/* Fill the progeny recursively, depth-first. */
for (int k = 0; k < 8; k++)
if (pc->progeny[k] >= 0) {
struct cell *temp;
space_getcells(s, 1, &temp);
temp->count = 0;
temp->gcount = 0;
temp->scount = 0;
temp->loc[0] = c->loc[0];
temp->loc[1] = c->loc[1];
temp->loc[2] = c->loc[2];
temp->width[0] = c->width[0] / 2;
temp->width[1] = c->width[1] / 2;
temp->width[2] = c->width[2] / 2;
temp->dmin = c->dmin / 2;
if (k & 4) temp->loc[0] += temp->width[0];
if (k & 2) temp->loc[1] += temp->width[1];
if (k & 1) temp->loc[2] += temp->width[2];
temp->depth = c->depth + 1;
temp->split = 0;
temp->dx_max_part = 0.f;
temp->dx_max_gpart = 0.f;
temp->dx_max_sort = 0.f;
temp->nodeID = c->nodeID;
temp->parent = c;
c->progeny[k] = temp;
c->split = 1;
count += cell_unpack(&pc[pc->progeny[k]], temp, s);
}
/* Return the total number of unpacked cells. */
c->pcell_size = count;
return count;
#else
error("SWIFT was not compiled with MPI support.");
return 0;
#endif
}
/** /**
* @brief Pack the time information of the given cell and all it's sub-cells. * @brief Pack the time information of the given cell and all it's sub-cells.
* *
* @param c The #cell. * @param c The #cell.
* @param ti_ends (output) The time information we pack into * @param pcells (output) The end-of-timestep information we pack into
* *
* @return The number of packed cells. * @return The number of packed cells.
*/ */
int cell_pack_ti_ends(struct cell *c, integertime_t *ti_ends) { int cell_pack_end_step(struct cell *restrict c, struct pcell_step *restrict pcells) {
#ifdef WITH_MPI #ifdef WITH_MPI
/* Pack this cell's data. */ /* Pack this cell's data. */
ti_ends[0] = c->ti_end_min; pcells[0].ti_end_min = c->ti_end_min;
pcells[0].dx_max_part = c->dx_max_part;
pcells[0].dx_max_gpart = c->dx_max_gpart;
/* Fill in the progeny, depth-first recursion. */ /* Fill in the progeny, depth-first recursion. */
int count = 1; int count = 1;
for (int k = 0; k < 8; k++) for (int k = 0; k < 8; k++)
if (c->progeny[k] != NULL) { if (c->progeny[k] != NULL) {
count += cell_pack_ti_ends(c->progeny[k], &ti_ends[count]); count += cell_pack_end_step(c->progeny[k], &pcells[count]);
} }
/* Return the number of packed values. */ /* Return the number of packed values. */
...@@ -302,22 +304,24 @@ int cell_pack_ti_ends(struct cell *c, integertime_t *ti_ends) { ...@@ -302,22 +304,24 @@ int cell_pack_ti_ends(struct cell *c, integertime_t *ti_ends) {
* @brief Unpack the time information of a given cell and its sub-cells. * @brief Unpack the time information of a given cell and its sub-cells.
* *
* @param c The #cell * @param c The #cell
* @param ti_ends The time information to unpack * @param pcells The end-of-timestep information to unpack
* *
* @return The number of cells created. * @return The number of cells created.
*/ */
int cell_unpack_ti_ends(struct cell *c, integertime_t *ti_ends) { int cell_unpack_end_step(struct cell *restrict c, struct pcell_step *restrict pcells) {
#ifdef WITH_MPI #ifdef WITH_MPI
/* Unpack this cell's data. */ /* Unpack this cell's data. */
c->ti_end_min = ti_ends[0]; c->ti_end_min = pcells[0].ti_end_min ;
c->dx_max_part = pcells[0].dx_max_part;
c->dx_max_gpart = pcells[0].dx_max_gpart;
/* Fill in the progeny, depth-first recursion. */ /* Fill in the progeny, depth-first recursion. */
int count = 1; int count = 1;
for (int k = 0; k < 8; k++) for (int k = 0; k < 8; k++)
if (c->progeny[k] != NULL) { if (c->progeny[k] != NULL) {
count += cell_unpack_ti_ends(c->progeny[k], &ti_ends[count]); count += cell_unpack_end_step(c->progeny[k], &pcells[count]);
} }
/* Return the number of packed values. */ /* Return the number of packed values. */
......
...@@ -70,24 +70,64 @@ struct link { ...@@ -70,24 +70,64 @@ struct link {
struct link *next; struct link *next;
}; };
/* Packed cell. */ /**
* @brief Packed cell for information correct at rebuild time.
*
* Contains all the information for a tree walk in a non-local cell.
*/
struct pcell { struct pcell {
/* Stats on this cell's particles. */ /*! Maximal smoothing length. */
double h_max; double h_max;
integertime_t ti_end_min, ti_end_max, ti_beg_max, ti_old_part, ti_old_gpart;
/* Number of particles in this cell. */ /*! Minimal integer end-of-timestep in this cell */
int count, gcount, scount; integertime_t ti_end_min;
/*! Maximal integer end-of-timestep in this cell */
integertime_t ti_end_max;
/*! Maximal integer beginning-of-timestep in this cell */
integertime_t ti_beg_max;
/* tag used for MPI communication. */ /*! Integer time of the last drift of the #part in this cell */
integertime_t ti_old_part;
/*! Integer time of the last drift of the #gpart in this cell */
integertime_t ti_old_gpart;
/*! Number of #part in this cell. */
int count;
/*! Number of #gpart in this cell. */
int gcount;
/*! Number of #spart in this cell. */
int scount;
/*! tag used for MPI communication. */
int tag; int tag;
/* Relative indices of the cell's progeny. */ /*! Relative indices of the cell's progeny. */
int progeny[8]; int progeny[8];
} SWIFT_STRUCT_ALIGN; } SWIFT_STRUCT_ALIGN;
/**
* @brief Cell information at the end of a time-step.
*/
struct pcell_step {
/*! Minimal integer end-of-timestep in this cell */
integertime_t ti_end_min;
/*! Maximal distance any #part has travelled since last rebuild */
float dx_max_part;
/*! Maximal distance any #gpart has travelled since last rebuild */
float dx_max_gpart;
};
/** /**
* @brief Cell within the tree structure. * @brief Cell within the tree structure.
* *
...@@ -389,8 +429,8 @@ int cell_slocktree(struct cell *c); ...@@ -389,8 +429,8 @@ int cell_slocktree(struct cell *c);
void cell_sunlocktree(struct cell *c); void cell_sunlocktree(struct cell *c);
int cell_pack(struct cell *c, struct pcell *pc); int cell_pack(struct cell *c, struct pcell *pc);
int cell_unpack(struct pcell *pc, struct cell *c, struct space *s); int cell_unpack(struct pcell *pc, struct cell *c, struct space *s);
int cell_pack_ti_ends(struct cell *c, integertime_t *ti_ends); int cell_pack_end_step(struct cell *c, struct pcell_step *pcell);
int cell_unpack_ti_ends(struct cell *c, integertime_t *ti_ends); int cell_unpack_end_step(struct cell *c, struct pcell_step *pcell);
int cell_getsize(struct cell *c); int cell_getsize(struct cell *c);
int cell_link_parts(struct cell *c, struct part *parts); int cell_link_parts(struct cell *c, struct part *parts);
int cell_link_gparts(struct cell *c, struct gpart *gparts); int cell_link_gparts(struct cell *c, struct gpart *gparts);
......
...@@ -1961,7 +1961,7 @@ void *runner_main(void *data) { ...@@ -1961,7 +1961,7 @@ void *runner_main(void *data) {
break; break;
case task_type_recv: case task_type_recv:
if (t->subtype == task_subtype_tend) { if (t->subtype == task_subtype_tend) {
cell_unpack_ti_ends(ci, t->buff); cell_unpack_end_step(ci, t->buff);
free(t->buff); free(t->buff);
} else if (t->subtype == task_subtype_xv) { } else if (t->subtype == task_subtype_xv) {
runner_do_recv_part(r, ci, 1, 1); runner_do_recv_part(r, ci, 1, 1);
......
...@@ -1274,8 +1274,8 @@ void scheduler_enqueue(struct scheduler *s, struct task *t) { ...@@ -1274,8 +1274,8 @@ void scheduler_enqueue(struct scheduler *s, struct task *t) {
case task_type_recv: case task_type_recv:
#ifdef WITH_MPI #ifdef WITH_MPI
if (t->subtype == task_subtype_tend) { if (t->subtype == task_subtype_tend) {
t->buff = malloc(sizeof(integertime_t) * t->ci->pcell_size); t->buff = malloc(sizeof(struct pcell_step) * t->ci->pcell_size);
err = MPI_Irecv(t->buff, t->ci->pcell_size * sizeof(integertime_t), err = MPI_Irecv(t->buff, t->ci->pcell_size * sizeof(struct pcell_step),
MPI_BYTE, t->ci->nodeID, t->flags, MPI_COMM_WORLD, MPI_BYTE, t->ci->nodeID, t->flags, MPI_COMM_WORLD,
&t->req); &t->req);
} else if (t->subtype == task_subtype_xv || } else if (t->subtype == task_subtype_xv ||
...@@ -1309,9 +1309,9 @@ void scheduler_enqueue(struct scheduler *s, struct task *t) { ...@@ -1309,9 +1309,9 @@ void scheduler_enqueue(struct scheduler *s, struct task *t) {
case task_type_send: case task_type_send:
#ifdef WITH_MPI #ifdef WITH_MPI
if (t->subtype == task_subtype_tend) { if (t->subtype == task_subtype_tend) {
t->buff = malloc(sizeof(integertime_t) * t->ci->pcell_size); t->buff = malloc(sizeof(struct pcell_step) * t->ci->pcell_size);
cell_pack_ti_ends(t->ci, t->buff); cell_pack_end_step(t->ci, t->buff);
err = MPI_Isend(t->buff, t->ci->pcell_size * sizeof(integertime_t), err = MPI_Isend(t->buff, t->ci->pcell_size * sizeof(struct pcell_step),
MPI_BYTE, t->cj->nodeID, t->flags, MPI_COMM_WORLD, MPI_BYTE, t->cj->nodeID, t->flags, MPI_COMM_WORLD,
&t->req); &t->req);
} else if (t->subtype == task_subtype_xv || } else if (t->subtype == task_subtype_xv ||
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment