diff --git a/src/cell.h b/src/cell.h index 1f5b308c729ac1f87d3e6fe81ed122ecf4842057..aa296e5d6c0bceb77cd2cd9f1c7186e434aff6ac 100644 --- a/src/cell.h +++ b/src/cell.h @@ -263,6 +263,9 @@ struct cell { /*! Number of #part updated in this cell. */ int updated; + /*! Number of #part inhibited in this cell. */ + int inhibited; + /*! Is the #part data of this cell being used in a sub-cell? */ int hold; @@ -379,6 +382,9 @@ struct cell { /*! Number of #gpart updated in this cell. */ int updated; + /*! Number of #gpart inhibited in this cell. */ + int inhibited; + /*! Is the #gpart data of this cell being used in a sub-cell? */ int phold; @@ -447,6 +453,9 @@ struct cell { /*! Number of #spart updated in this cell. */ int updated; + /*! Number of #spart inhibited in this cell. */ + int inhibited; + /*! Is the #spart data of this cell being used in a sub-cell? */ int hold; diff --git a/src/collectgroup.c b/src/collectgroup.c index c83d7bef3f03e672e8b5c9036e5daaab26b5d190..630c383f12891b652d60771fbbf990557d68bc65 100644 --- a/src/collectgroup.c +++ b/src/collectgroup.c @@ -36,7 +36,8 @@ /* Local collections for MPI reduces. */ struct mpicollectgroup1 { - long long updates, g_updates, s_updates; + long long updated, g_updated, s_updated; + long long inhibited, g_inhibited, s_inhibited; integertime_t ti_hydro_end_min; integertime_t ti_gravity_end_min; int forcerebuild; @@ -85,9 +86,12 @@ void collectgroup1_apply(struct collectgroup1 *grp1, struct engine *e) { e->ti_end_min = min(e->ti_hydro_end_min, e->ti_gravity_end_min); e->ti_end_max = max(e->ti_hydro_end_max, e->ti_gravity_end_max); e->ti_beg_max = max(e->ti_hydro_beg_max, e->ti_gravity_beg_max); - e->updates = grp1->updates; - e->g_updates = grp1->g_updates; - e->s_updates = grp1->s_updates; + e->updates = grp1->updated; + e->g_updates = grp1->g_updated; + e->s_updates = grp1->s_updated; + e->nr_inhibited_parts = grp1->inhibited; + e->nr_inhibited_gparts = grp1->g_inhibited; + e->nr_inhibited_sparts = grp1->s_inhibited; e->forcerebuild = grp1->forcerebuild; } @@ -113,17 +117,22 @@ void collectgroup1_apply(struct collectgroup1 *grp1, struct engine *e) { * after this step. * @param forcerebuild whether a rebuild is required after this step. */ -void collectgroup1_init(struct collectgroup1 *grp1, size_t updates, - size_t g_updates, size_t s_updates, +void collectgroup1_init(struct collectgroup1 *grp1, size_t updated, + size_t g_updated, size_t s_updated, size_t inhibited, + size_t g_inhibited, size_t s_inhibited, integertime_t ti_hydro_end_min, integertime_t ti_hydro_end_max, integertime_t ti_hydro_beg_max, integertime_t ti_gravity_end_min, integertime_t ti_gravity_end_max, integertime_t ti_gravity_beg_max, int forcerebuild) { - grp1->updates = updates; - grp1->g_updates = g_updates; - grp1->s_updates = s_updates; + + grp1->updated = updated; + grp1->g_updated = g_updated; + grp1->s_updated = s_updated; + grp1->inhibited = inhibited; + grp1->g_inhibited = g_inhibited; + grp1->s_inhibited = s_inhibited; grp1->ti_hydro_end_min = ti_hydro_end_min; grp1->ti_hydro_end_max = ti_hydro_end_max; grp1->ti_hydro_beg_max = ti_hydro_beg_max; @@ -147,9 +156,12 @@ void collectgroup1_reduce(struct collectgroup1 *grp1) { /* Populate an MPI group struct and reduce this across all nodes. */ struct mpicollectgroup1 mpigrp11; - mpigrp11.updates = grp1->updates; - mpigrp11.g_updates = grp1->g_updates; - mpigrp11.s_updates = grp1->s_updates; + mpigrp11.updated = grp1->updated; + mpigrp11.g_updated = grp1->g_updated; + mpigrp11.s_updated = grp1->s_updated; + mpigrp11.inhibited = grp1->inhibited; + mpigrp11.g_inhibited = grp1->g_inhibited; + mpigrp11.s_inhibited = grp1->s_inhibited; mpigrp11.ti_hydro_end_min = grp1->ti_hydro_end_min; mpigrp11.ti_gravity_end_min = grp1->ti_gravity_end_min; mpigrp11.forcerebuild = grp1->forcerebuild; @@ -160,9 +172,12 @@ void collectgroup1_reduce(struct collectgroup1 *grp1) { error("Failed to reduce mpicollection1."); /* And update. */ - grp1->updates = mpigrp12.updates; - grp1->g_updates = mpigrp12.g_updates; - grp1->s_updates = mpigrp12.s_updates; + grp1->updated = mpigrp12.updated; + grp1->g_updated = mpigrp12.g_updated; + grp1->s_updated = mpigrp12.s_updated; + grp1->inhibited = mpigrp12.inhibited; + grp1->g_inhibited = mpigrp12.g_inhibited; + grp1->s_inhibited = mpigrp12.s_inhibited; grp1->ti_hydro_end_min = mpigrp12.ti_hydro_end_min; grp1->ti_gravity_end_min = mpigrp12.ti_gravity_end_min; grp1->forcerebuild = mpigrp12.forcerebuild; @@ -182,9 +197,14 @@ static void doreduce1(struct mpicollectgroup1 *mpigrp11, /* Do what is needed for each part of the collection. */ /* Sum of updates. */ - mpigrp11->updates += mpigrp12->updates; - mpigrp11->g_updates += mpigrp12->g_updates; - mpigrp11->s_updates += mpigrp12->s_updates; + mpigrp11->updated += mpigrp12->updated; + mpigrp11->g_updated += mpigrp12->g_updated; + mpigrp11->s_updated += mpigrp12->s_updated; + + /* Sum of inhibited */ + mpigrp11->inhibited += mpigrp12->inhibited; + mpigrp11->g_inhibited += mpigrp12->g_inhibited; + mpigrp11->s_inhibited += mpigrp12->s_inhibited; /* Minimum end time. */ mpigrp11->ti_hydro_end_min = diff --git a/src/collectgroup.h b/src/collectgroup.h index 8bf8a9d1b75f9a5ddb3f19fa9cdb4103e044ea59..b6e8769ac993cc023ae402cdfc4b0169406f6181 100644 --- a/src/collectgroup.h +++ b/src/collectgroup.h @@ -35,7 +35,10 @@ struct engine; struct collectgroup1 { /* Number of particles updated */ - long long updates, g_updates, s_updates; + long long updated, g_updated, s_updated; + + /* Number of particles inhibited */ + long long inhibited, g_inhibited, s_inhibited; /* Times for the time-step */ integertime_t ti_hydro_end_min, ti_hydro_end_max, ti_hydro_beg_max; @@ -47,8 +50,9 @@ struct collectgroup1 { void collectgroup_init(void); void collectgroup1_apply(struct collectgroup1 *grp1, struct engine *e); -void collectgroup1_init(struct collectgroup1 *grp1, size_t updates, - size_t g_updates, size_t s_updates, +void collectgroup1_init(struct collectgroup1 *grp1, size_t updated, + size_t g_updated, size_t s_updated, size_t inhibited, + size_t g_inhibited, size_t s_inhibited, integertime_t ti_hydro_end_min, integertime_t ti_hydro_end_max, integertime_t ti_hydro_beg_max, diff --git a/src/engine.c b/src/engine.c index 57cfaf1067230c8415eb787e8bada535be13fd27..18302e1d20fd3b3a0794f6ef0fe910508e663334 100644 --- a/src/engine.c +++ b/src/engine.c @@ -121,7 +121,8 @@ int engine_rank; */ struct end_of_step_data { - size_t updates, g_updates, s_updates; + size_t updated, g_updated, s_updated; + size_t inhibited, g_inhibited, s_inhibited; integertime_t ti_hydro_end_min, ti_hydro_end_max, ti_hydro_beg_max; integertime_t ti_gravity_end_min, ti_gravity_end_max, ti_gravity_beg_max; struct engine *e; @@ -4434,11 +4435,10 @@ void engine_rebuild(struct engine *e, int clean_smoothing_length_values) { e->total_nr_gparts = num_particles[1]; e->total_nr_sparts = num_particles[2]; -#ifdef SWIFT_DEBUG_CHECKS - e->count_inhibited_parts = 0; - e->count_inhibited_gparts = 0; - e->count_inhibited_sparts = 0; -#endif + /* Flag that there are no inhibited particles */ + e->nr_inhibited_parts = 0; + e->nr_inhibited_gparts = 0; + e->nr_inhibited_sparts = 0; /* Re-compute the mesh forces */ if ((e->policy & engine_policy_self_gravity) && e->s->periodic) @@ -4625,6 +4625,7 @@ void engine_collect_end_of_step_recurse(struct cell *c, /* Counters for the different quantities. */ size_t updated = 0, g_updated = 0, s_updated = 0; + size_t inhibited = 0, g_inhibited = 0, s_inhibited = 0; integertime_t ti_hydro_end_min = max_nr_timesteps, ti_hydro_end_max = 0, ti_hydro_beg_max = 0; integertime_t ti_gravity_end_min = max_nr_timesteps, ti_gravity_end_max = 0, @@ -4652,6 +4653,10 @@ void engine_collect_end_of_step_recurse(struct cell *c, g_updated += cp->grav.updated; s_updated += cp->stars.updated; + inhibited += cp->hydro.inhibited; + g_inhibited += cp->grav.inhibited; + s_inhibited += cp->stars.inhibited; + /* Collected, so clear for next time. */ cp->hydro.updated = 0; cp->grav.updated = 0; @@ -4669,6 +4674,9 @@ void engine_collect_end_of_step_recurse(struct cell *c, c->hydro.updated = updated; c->grav.updated = g_updated; c->stars.updated = s_updated; + c->hydro.inhibited = inhibited; + c->grav.inhibited = g_inhibited; + c->stars.inhibited = s_inhibited; } /** @@ -4691,7 +4699,8 @@ void engine_collect_end_of_step_mapper(void *map_data, int num_elements, int *local_cells = (int *)map_data; /* Local collectible */ - size_t updates = 0, g_updates = 0, s_updates = 0; + size_t updated = 0, g_updated = 0, s_updated = 0; + size_t inhibited = 0, g_inhibited = 0, s_inhibited = 0; integertime_t ti_hydro_end_min = max_nr_timesteps, ti_hydro_end_max = 0, ti_hydro_beg_max = 0; integertime_t ti_gravity_end_min = max_nr_timesteps, ti_gravity_end_max = 0, @@ -4716,9 +4725,13 @@ void engine_collect_end_of_step_mapper(void *map_data, int num_elements, ti_gravity_end_max = max(ti_gravity_end_max, c->grav.ti_end_max); ti_gravity_beg_max = max(ti_gravity_beg_max, c->grav.ti_beg_max); - updates += c->hydro.updated; - g_updates += c->grav.updated; - s_updates += c->stars.updated; + updated += c->hydro.updated; + g_updated += c->grav.updated; + s_updated += c->stars.updated; + + inhibited += c->hydro.inhibited; + g_inhibited += c->grav.inhibited; + s_inhibited += c->stars.inhibited; /* Collected, so clear for next time. */ c->hydro.updated = 0; @@ -4730,9 +4743,13 @@ void engine_collect_end_of_step_mapper(void *map_data, int num_elements, /* Let's write back to the global data. * We use the space lock to garanty single access*/ if (lock_lock(&s->lock) == 0) { - data->updates += updates; - data->g_updates += g_updates; - data->s_updates += s_updates; + data->updated += updated; + data->g_updated += g_updated; + data->s_updated += s_updated; + + data->inhibited += inhibited; + data->g_inhibited += g_inhibited; + data->s_inhibited += s_inhibited; if (ti_hydro_end_min > e->ti_current) data->ti_hydro_end_min = min(ti_hydro_end_min, data->ti_hydro_end_min); @@ -4773,7 +4790,8 @@ void engine_collect_end_of_step(struct engine *e, int apply) { const ticks tic = getticks(); const struct space *s = e->s; struct end_of_step_data data; - data.updates = 0, data.g_updates = 0, data.s_updates = 0; + data.updated = 0, data.g_updated = 0, data.s_updated = 0; + data.inhibited = 0, data.g_inhibited = 0, data.s_inhibited = 0; data.ti_hydro_end_min = max_nr_timesteps, data.ti_hydro_end_max = 0, data.ti_hydro_beg_max = 0; data.ti_gravity_end_min = max_nr_timesteps, data.ti_gravity_end_max = 0, @@ -4786,11 +4804,11 @@ void engine_collect_end_of_step(struct engine *e, int apply) { sizeof(int), 0, &data); /* Store these in the temporary collection group. */ - collectgroup1_init(&e->collect_group1, data.updates, data.g_updates, - data.s_updates, data.ti_hydro_end_min, - data.ti_hydro_end_max, data.ti_hydro_beg_max, - data.ti_gravity_end_min, data.ti_gravity_end_max, - data.ti_gravity_beg_max, e->forcerebuild); + collectgroup1_init( + &e->collect_group1, data.updated, data.g_updated, data.s_updated, + data.inhibited, data.g_inhibited, data.s_inhibited, data.ti_hydro_end_min, + data.ti_hydro_end_max, data.ti_hydro_beg_max, data.ti_gravity_end_min, + data.ti_gravity_end_max, data.ti_gravity_beg_max, e->forcerebuild); /* Aggregate collective data from the different nodes for this step. */ #ifdef WITH_MPI @@ -4815,21 +4833,37 @@ void engine_collect_end_of_step(struct engine *e, int apply) { in_i[1], e->collect_group1.ti_gravity_end_min); long long in_ll[3], out_ll[3]; - out_ll[0] = data.updates; - out_ll[1] = data.g_updates; - out_ll[2] = data.s_updates; + out_ll[0] = data.updated; + out_ll[1] = data.g_updated; + out_ll[2] = data.s_updated; + if (MPI_Allreduce(out_ll, in_ll, 3, MPI_LONG_LONG_INT, MPI_SUM, + MPI_COMM_WORLD) != MPI_SUCCESS) + error("Failed to aggregate particle counts."); + if (in_ll[0] != (long long)e->collect_group1.updated) + error("Failed to get same updated, is %lld, should be %lld", in_ll[0], + e->collect_group1.updated); + if (in_ll[1] != (long long)e->collect_group1.g_updated) + error("Failed to get same g_updated, is %lld, should be %lld", in_ll[1], + e->collect_group1.g_updated); + if (in_ll[2] != (long long)e->collect_group1.s_updated) + error("Failed to get same s_updated, is %lld, should be %lld", in_ll[2], + e->collect_group1.s_updated); + + out_ll[0] = data.inhibited; + out_ll[1] = data.g_inhibited; + out_ll[2] = data.s_inhibited; if (MPI_Allreduce(out_ll, in_ll, 3, MPI_LONG_LONG_INT, MPI_SUM, MPI_COMM_WORLD) != MPI_SUCCESS) error("Failed to aggregate particle counts."); - if (in_ll[0] != (long long)e->collect_group1.updates) - error("Failed to get same updates, is %lld, should be %lld", in_ll[0], - e->collect_group1.updates); - if (in_ll[1] != (long long)e->collect_group1.g_updates) - error("Failed to get same g_updates, is %lld, should be %lld", in_ll[1], - e->collect_group1.g_updates); - if (in_ll[2] != (long long)e->collect_group1.s_updates) - error("Failed to get same s_updates, is %lld, should be %lld", in_ll[2], - e->collect_group1.s_updates); + if (in_ll[0] != (long long)e->collect_group1.inhibited) + error("Failed to get same inhibited, is %lld, should be %lld", in_ll[0], + e->collect_group1.inhibited); + if (in_ll[1] != (long long)e->collect_group1.g_inhibited) + error("Failed to get same g_inhibited, is %lld, should be %lld", in_ll[1], + e->collect_group1.g_inhibited); + if (in_ll[2] != (long long)e->collect_group1.s_inhibited) + error("Failed to get same s_inhibited, is %lld, should be %lld", in_ll[2], + e->collect_group1.s_inhibited); int buff = 0; if (MPI_Allreduce(&e->forcerebuild, &buff, 1, MPI_INT, MPI_MAX, @@ -5366,9 +5400,9 @@ void engine_step(struct engine *e) { /* Collect information about the next time-step */ engine_collect_end_of_step(e, 1); e->forcerebuild = e->collect_group1.forcerebuild; - e->updates_since_rebuild += e->collect_group1.updates; - e->g_updates_since_rebuild += e->collect_group1.g_updates; - e->s_updates_since_rebuild += e->collect_group1.s_updates; + e->updates_since_rebuild += e->collect_group1.updated; + e->g_updates_since_rebuild += e->collect_group1.g_updated; + e->s_updates_since_rebuild += e->collect_group1.s_updated; #ifdef SWIFT_DEBUG_CHECKS if (e->ti_end_min == e->ti_current && e->ti_end_min < max_nr_timesteps) diff --git a/src/engine.h b/src/engine.h index 012021a1712fb6748ee44cec50b61c70615c0906..5c31fcc51e11351dade2075deaa40aa35848737d 100644 --- a/src/engine.h +++ b/src/engine.h @@ -205,6 +205,9 @@ struct engine { /* Total numbers of particles in the system. */ long long total_nr_parts, total_nr_gparts, total_nr_sparts; + /* The total number of inhibted particles in the system. */ + size_t nr_inhibited_parts, nr_inhibited_gparts, nr_inhibited_sparts; + #ifdef SWIFT_DEBUG_CHECKS /* Total number of particles removed from the system since the last rebuild */ long long count_inhibited_parts, count_inhibited_gparts, diff --git a/src/space.c b/src/space.c index fe8e2a1bd647c4531fd5d65030dc891fb3c22a7e..efb3e966d7c935737519518a935a32aec8072e20 100644 --- a/src/space.c +++ b/src/space.c @@ -178,8 +178,14 @@ void space_rebuild_recycle_mapper(void *map_data, int num_elements, c->hydro.dx_max_sort = 0.0f; c->hydro.sorted = 0; c->hydro.count = 0; + c->hydro.updated = 0; + c->hydro.inhibited = 0; c->grav.count = 0; + c->grav.updated = 0; + c->grav.inhibited = 0; c->stars.count = 0; + c->stars.updated = 0; + c->stars.inhibited = 0; c->grav.init = NULL; c->grav.init_out = NULL; c->hydro.extra_ghost = NULL;