Commit 0a61ab22 authored by Matthieu Schaller's avatar Matthieu Schaller
Browse files

Count the number of particles in foregin cells that have tasks only. Allocated...

Count the number of particles in foregin cells that have tasks only. Allocated memory based on that counter. Link the cells recursively based on that counter.
parent f5b9d23d
......@@ -69,6 +69,11 @@
/* Global variables. */
int cell_next_tag = 0;
int depth_0_counter = 0;
int depth_1_counter = 0;
int depth_2_counter = 0;
int depth_3_counter = 0;
/**
* @brief Get the size of the cell subtree.
*
......@@ -96,21 +101,46 @@ int cell_getsize(struct cell *c) {
*
* @return The number of particles linked.
*/
int cell_link_parts(struct cell *c, struct part *parts) {
int cell_link_parts(struct cell *c, struct part *parts, int link) {
#ifdef SWIFT_DEBUG_CHECKS
if (c->nodeID == engine_rank)
error("Linking foreign particles in a local cell!");
#endif
/* Do we have a hydro task at this level? */
if (c->hydro.density != NULL) {
link = 1;
}
c->hydro.parts = parts;
/* Ok, link the particles at this level */
if (link) {
c->hydro.parts = parts;
}
/* Fill the progeny recursively, depth-first. */
if (c->split) {
int offset = 0;
int count = 0;
for (int k = 0; k < 8; k++) {
if (c->progeny[k] != NULL)
offset += cell_link_parts(c->progeny[k], &parts[offset]);
if (c->progeny[k] != NULL) {
count += cell_link_parts(c->progeny[k], &parts[count], link);
}
}
#ifdef SWIFT_DEBUG_CHECKS
if (link && (count != c->hydro.count))
error("Something is wrong with the foreign part counts.");
#endif
return count;
}
/* Return the total number of linked particles. */
return c->hydro.count;
if (link)
return c->hydro.count;
else
return 0;
}
/**
......@@ -163,6 +193,35 @@ int cell_link_sparts(struct cell *c, struct spart *sparts) {
return c->stars.count;
}
int cell_count_parts_for_tasks(const struct cell *c) {
#ifdef SWIFT_DEBUG_CHECKS
if (c->nodeID == engine_rank)
error("Counting foreign particles in a local cell!");
#endif
/* Do we have a hydro task at this level? */
if (c->hydro.density != NULL) {
if (c->depth == 0) ++depth_0_counter;
if (c->depth == 1) ++depth_1_counter;
if (c->depth == 2) ++depth_2_counter;
if (c->depth == 3) ++depth_3_counter;
return c->hydro.count;
}
if (c->split) {
int count = 0;
for (int k = 0; k < 8; ++k) {
if (c->progeny[k] != NULL) {
count += cell_count_parts_for_tasks(c->progeny[k]);
}
}
return count;
} else {
return 0;
}
}
/**
* @brief Pack the data of the given cell and all it's sub-cells.
*
......
......@@ -688,9 +688,10 @@ int cell_unpack_end_step(struct cell *c, struct pcell_step *pcell);
int cell_pack_multipoles(struct cell *c, struct gravity_tensors *m);
int cell_unpack_multipoles(struct cell *c, struct gravity_tensors *m);
int cell_getsize(struct cell *c);
int cell_link_parts(struct cell *c, struct part *parts);
int cell_link_parts(struct cell *c, struct part *parts, int link);
int cell_link_gparts(struct cell *c, struct gpart *gparts);
int cell_link_sparts(struct cell *c, struct spart *sparts);
int cell_count_parts_for_tasks(const struct cell *c);
void cell_clean_links(struct cell *c, void *data);
void cell_make_multipoles(struct cell *c, integertime_t ti_current);
void cell_check_multipole(struct cell *c);
......
......@@ -1130,84 +1130,12 @@ void engine_exchange_cells(struct engine *e) {
#ifdef WITH_MPI
struct space *s = e->s;
const int nr_proxies = e->nr_proxies;
const int with_gravity = e->policy & engine_policy_self_gravity;
const ticks tic = getticks();
/* Exchange the cell structure with neighbouring ranks. */
proxy_cells_exchange(e->proxies, e->nr_proxies, e->s, with_gravity);
ticks tic2 = getticks();
/* Count the number of particles we need to import and re-allocate
the buffer if needed. */
size_t count_parts_in = 0, count_gparts_in = 0, count_sparts_in = 0;
for (int k = 0; k < nr_proxies; k++)
for (int j = 0; j < e->proxies[k].nr_cells_in; j++) {
if (e->proxies[k].cells_in_type[j] & proxy_cell_type_hydro)
count_parts_in += e->proxies[k].cells_in[j]->hydro.count;
if (e->proxies[k].cells_in_type[j] & proxy_cell_type_gravity)
count_gparts_in += e->proxies[k].cells_in[j]->grav.count;
count_sparts_in += e->proxies[k].cells_in[j]->stars.count;
}
if (count_parts_in > s->size_parts_foreign) {
if (s->parts_foreign != NULL) free(s->parts_foreign);
s->size_parts_foreign = 1.1 * count_parts_in;
if (posix_memalign((void **)&s->parts_foreign, part_align,
sizeof(struct part) * s->size_parts_foreign) != 0)
error("Failed to allocate foreign part data.");
}
if (count_gparts_in > s->size_gparts_foreign) {
if (s->gparts_foreign != NULL) free(s->gparts_foreign);
s->size_gparts_foreign = 1.1 * count_gparts_in;
if (posix_memalign((void **)&s->gparts_foreign, gpart_align,
sizeof(struct gpart) * s->size_gparts_foreign) != 0)
error("Failed to allocate foreign gpart data.");
}
if (count_sparts_in > s->size_sparts_foreign) {
if (s->sparts_foreign != NULL) free(s->sparts_foreign);
s->size_sparts_foreign = 1.1 * count_sparts_in;
if (posix_memalign((void **)&s->sparts_foreign, spart_align,
sizeof(struct spart) * s->size_sparts_foreign) != 0)
error("Failed to allocate foreign spart data.");
}
if (e->verbose)
message("Counting and allocating arrays took %.3f %s.",
clocks_from_ticks(getticks() - tic2), clocks_getunit());
tic2 = getticks();
/* Unpack the cells and link to the particle data. */
struct part *parts = s->parts_foreign;
struct gpart *gparts = s->gparts_foreign;
struct spart *sparts = s->sparts_foreign;
for (int k = 0; k < nr_proxies; k++) {
for (int j = 0; j < e->proxies[k].nr_cells_in; j++) {
if (e->proxies[k].cells_in_type[j] & proxy_cell_type_hydro) {
cell_link_parts(e->proxies[k].cells_in[j], parts);
parts = &parts[e->proxies[k].cells_in[j]->hydro.count];
}
if (e->proxies[k].cells_in_type[j] & proxy_cell_type_gravity) {
cell_link_gparts(e->proxies[k].cells_in[j], gparts);
gparts = &gparts[e->proxies[k].cells_in[j]->grav.count];
}
cell_link_sparts(e->proxies[k].cells_in[j], sparts);
sparts = &sparts[e->proxies[k].cells_in[j]->stars.count];
}
}
s->nr_parts_foreign = parts - s->parts_foreign;
s->nr_gparts_foreign = gparts - s->gparts_foreign;
s->nr_sparts_foreign = sparts - s->sparts_foreign;
if (e->verbose)
message("Recursively linking arrays took %.3f %s.",
clocks_from_ticks(getticks() - tic2), clocks_getunit());
if (e->verbose)
message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
clocks_getunit());
......@@ -1822,6 +1750,123 @@ void engine_exchange_proxy_multipoles(struct engine *e) {
#endif
}
extern int depth_0_counter;
extern int depth_1_counter;
extern int depth_2_counter;
extern int depth_3_counter;
void engine_allocate_foreign_particles(struct engine *e) {
#ifdef WITH_MPI
const int nr_proxies = e->nr_proxies;
struct space *s = e->s;
// ticks tic = getticks();
/* Count the number of particles we need to import and re-allocate
the buffer if needed. */
size_t count_parts_in_old = 0;
size_t count_parts_in = 0;
for (int k = 0; k < nr_proxies; k++) {
for (int j = 0; j < e->proxies[k].nr_cells_in; j++) {
if (e->proxies[k].cells_in_type[j] & proxy_cell_type_hydro) {
count_parts_in_old += e->proxies[k].cells_in[j]->hydro.count;
count_parts_in += cell_count_parts_for_tasks(e->proxies[k].cells_in[j]);
}
}
}
message("New count: %zd old count: %zd", count_parts_in, count_parts_in_old);
message("counters: %d %d %d %d", depth_0_counter, depth_1_counter,
depth_2_counter, depth_3_counter);
/* Allocate space for the foreign particles we will receive */
if (count_parts_in > s->size_parts_foreign) {
if (s->parts_foreign != NULL) free(s->parts_foreign);
s->size_parts_foreign = 1.1 * count_parts_in;
if (posix_memalign((void **)&s->parts_foreign, part_align,
sizeof(struct part) * s->size_parts_foreign) != 0)
error("Failed to allocate foreign part data.");
}
struct part *parts = s->parts_foreign;
size_t count_parts = 0;
for (int k = 0; k < nr_proxies; k++) {
for (int j = 0; j < e->proxies[k].nr_cells_in; j++) {
if (e->proxies[k].cells_in_type[j] & proxy_cell_type_hydro) {
count_parts +=
cell_link_parts(e->proxies[k].cells_in[j], parts, /*link=*/0);
parts += count_parts;
}
}
}
message("count_parts: %zd %zd", count_parts_in, count_parts);
return;
/* if (count_gparts_in > s->size_gparts_foreign) { */
/* if (s->gparts_foreign != NULL) free(s->gparts_foreign); */
/* s->size_gparts_foreign = 1.1 * count_gparts_in; */
/* if (posix_memalign((void **)&s->gparts_foreign, gpart_align, */
/* sizeof(struct gpart) * s->size_gparts_foreign) != 0)
*/
/* error("Failed to allocate foreign gpart data."); */
/* } */
/* if (count_sparts_in > s->size_sparts_foreign) { */
/* if (s->sparts_foreign != NULL) free(s->sparts_foreign); */
/* s->size_sparts_foreign = 1.1 * count_sparts_in; */
/* if (posix_memalign((void **)&s->sparts_foreign, spart_align, */
/* sizeof(struct spart) * s->size_sparts_foreign) != 0)
*/
/* error("Failed to allocate foreign spart data."); */
/* } */
/* if (e->verbose) */
/* message("Counting and allocating arrays took %.3f %s.", */
/* clocks_from_ticks(getticks() - tic), clocks_getunit()); */
/* tic = getticks(); */
/* /\* Unpack the cells and link to the particle data. *\/ */
/* struct part *parts = s->parts_foreign; */
/* struct gpart *gparts = s->gparts_foreign; */
/* struct spart *sparts = s->sparts_foreign; */
/* for (int k = 0; k < nr_proxies; k++) { */
/* for (int j = 0; j < e->proxies[k].nr_cells_in; j++) { */
/* if (e->proxies[k].cells_in_type[j] & proxy_cell_type_hydro) { */
/* cell_link_parts(e->proxies[k].cells_in[j], parts); */
/* parts = &parts[e->proxies[k].cells_in[j]->hydro.count]; */
/* } */
/* if (e->proxies[k].cells_in_type[j] & proxy_cell_type_gravity) { */
/* cell_link_gparts(e->proxies[k].cells_in[j], gparts); */
/* gparts = &gparts[e->proxies[k].cells_in[j]->grav.count]; */
/* } */
/* cell_link_sparts(e->proxies[k].cells_in[j], sparts); */
/* sparts = &sparts[e->proxies[k].cells_in[j]->stars.count]; */
/* } */
/* } */
/* s->nr_parts_foreign = parts - s->parts_foreign; */
/* s->nr_gparts_foreign = gparts - s->gparts_foreign; */
/* s->nr_sparts_foreign = sparts - s->sparts_foreign; */
/* if (e->verbose) */
/* message("Recursively linking arrays took %.3f %s.", */
/* clocks_from_ticks(getticks() - tic), clocks_getunit()); */
#else
error("SWIFT was not compiled with MPI support.");
#endif
}
/**
* @brief Prints the number of tasks in the engine
*
......
......@@ -400,6 +400,7 @@ void engine_unskip(struct engine *e);
void engine_drift_all(struct engine *e, const int drift_mpoles);
void engine_drift_top_multipoles(struct engine *e);
void engine_reconstruct_multipoles(struct engine *e);
void engine_allocate_foreign_particles(struct engine *e);
void engine_print_stats(struct engine *e);
void engine_check_for_dumps(struct engine *e);
void engine_dump_snapshot(struct engine *e);
......
......@@ -2485,6 +2485,8 @@ void engine_maketasks(struct engine *e) {
}
#endif
engine_allocate_foreign_particles(e);
tic2 = getticks();
/* Set the unlocks per task. */
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment