diff --git a/src/atomic.h b/src/atomic.h index 97e0935f5b2dd76ec0dd6cb14699216e1e25d8b7..43da9823466b200c11dd55d43c9762c93df364ea 100644 --- a/src/atomic.h +++ b/src/atomic.h @@ -29,6 +29,7 @@ #define atomic_add(v, i) __sync_fetch_and_add(v, i) #define atomic_sub(v, i) __sync_fetch_and_sub(v, i) #define atomic_or(v, i) __sync_fetch_and_or(v, i) +#define atomic_and(v, i) __sync_fetch_and_and(v, i) #define atomic_inc(v) atomic_add(v, 1) #define atomic_dec(v) atomic_sub(v, 1) #define atomic_cas(v, o, n) __sync_val_compare_and_swap(v, o, n) diff --git a/src/cell.c b/src/cell.c index 85763ec1605de6a6cba1aa03e1f62f529042cd73..b4d2c9f62b74270024f25256e23ea76ee2aac949 100644 --- a/src/cell.c +++ b/src/cell.c @@ -2153,20 +2153,18 @@ void cell_clean(struct cell *c) { * @brief Clear the drift flags on the given cell. */ void cell_clear_drift_flags(struct cell *c, void *data) { - c->hydro.do_drift = 0; - c->hydro.do_sub_drift = 0; - c->grav.do_drift = 0; - c->grav.do_sub_drift = 0; - c->stars.do_drift = 0; - c->stars.do_sub_drift = 0; + cell_clear_flag(c, cell_flag_do_hydro_drift | cell_flag_do_hydro_sub_drift | + cell_flag_do_grav_drift | cell_flag_do_grav_sub_drift | + cell_flag_do_stars_drift | + cell_flag_do_stars_sub_drift); } /** * @brief Clear the limiter flags on the given cell. */ void cell_clear_limiter_flags(struct cell *c, void *data) { - c->hydro.do_limiter = 0; - c->hydro.do_sub_limiter = 0; + cell_clear_flag(c, + cell_flag_do_hydro_limiter | cell_flag_do_hydro_sub_limiter); } /** @@ -2199,10 +2197,10 @@ void cell_activate_super_spart_drifts(struct cell *c, struct scheduler *s) { */ void cell_activate_drift_part(struct cell *c, struct scheduler *s) { /* If this cell is already marked for drift, quit early. */ - if (c->hydro.do_drift) return; + if (cell_get_flag(c, cell_flag_do_hydro_drift)) return; /* Mark this cell for drifting. */ - c->hydro.do_drift = 1; + cell_set_flag(c, cell_flag_do_hydro_drift); /* Set the do_sub_drifts all the way up and activate the super drift if this has not yet been done. */ @@ -2214,10 +2212,10 @@ void cell_activate_drift_part(struct cell *c, struct scheduler *s) { scheduler_activate(s, c->hydro.drift); } else { for (struct cell *parent = c->parent; - parent != NULL && !parent->hydro.do_sub_drift; + parent != NULL && !cell_get_flag(parent, cell_flag_do_hydro_sub_drift); parent = parent->parent) { /* Mark this cell for drifting */ - parent->hydro.do_sub_drift = 1; + cell_set_flag(parent, cell_flag_do_hydro_sub_drift); if (parent == c->hydro.super) { #ifdef SWIFT_DEBUG_CHECKS @@ -2236,10 +2234,10 @@ void cell_activate_drift_part(struct cell *c, struct scheduler *s) { */ void cell_activate_drift_gpart(struct cell *c, struct scheduler *s) { /* If this cell is already marked for drift, quit early. */ - if (c->grav.do_drift) return; + if (cell_get_flag(c, cell_flag_do_grav_drift)) return; /* Mark this cell for drifting. */ - c->grav.do_drift = 1; + cell_set_flag(c, cell_flag_do_grav_drift); if (c->grav.drift_out != NULL) scheduler_activate(s, c->grav.drift_out); @@ -2253,9 +2251,9 @@ void cell_activate_drift_gpart(struct cell *c, struct scheduler *s) { scheduler_activate(s, c->grav.drift); } else { for (struct cell *parent = c->parent; - parent != NULL && !parent->grav.do_sub_drift; + parent != NULL && !cell_get_flag(parent, cell_flag_do_grav_sub_drift); parent = parent->parent) { - parent->grav.do_sub_drift = 1; + cell_set_flag(parent, cell_flag_do_grav_sub_drift); if (parent->grav.drift_out) { scheduler_activate(s, parent->grav.drift_out); @@ -2278,10 +2276,10 @@ void cell_activate_drift_gpart(struct cell *c, struct scheduler *s) { */ void cell_activate_drift_spart(struct cell *c, struct scheduler *s) { /* If this cell is already marked for drift, quit early. */ - if (c->stars.do_drift) return; + if (cell_get_flag(c, cell_flag_do_stars_drift)) return; /* Mark this cell for drifting. */ - c->stars.do_drift = 1; + cell_set_flag(c, cell_flag_do_stars_drift); /* Set the do_stars_sub_drifts all the way up and activate the super drift if this has not yet been done. */ @@ -2293,10 +2291,10 @@ void cell_activate_drift_spart(struct cell *c, struct scheduler *s) { scheduler_activate(s, c->stars.drift); } else { for (struct cell *parent = c->parent; - parent != NULL && !parent->stars.do_sub_drift; + parent != NULL && !cell_get_flag(parent, cell_flag_do_stars_sub_drift); parent = parent->parent) { /* Mark this cell for drifting */ - parent->stars.do_sub_drift = 1; + cell_set_flag(parent, cell_flag_do_stars_sub_drift); if (parent == c->hydro.super) { #ifdef SWIFT_DEBUG_CHECKS @@ -2315,10 +2313,10 @@ void cell_activate_drift_spart(struct cell *c, struct scheduler *s) { */ void cell_activate_limiter(struct cell *c, struct scheduler *s) { /* If this cell is already marked for limiting, quit early. */ - if (c->hydro.do_limiter) return; + if (cell_get_flag(c, cell_flag_do_hydro_limiter)) return; /* Mark this cell for limiting. */ - c->hydro.do_limiter = 1; + cell_set_flag(c, cell_flag_do_hydro_limiter); /* Set the do_sub_limiter all the way up and activate the super limiter if this has not yet been done. */ @@ -2330,10 +2328,11 @@ void cell_activate_limiter(struct cell *c, struct scheduler *s) { scheduler_activate(s, c->timestep_limiter); } else { for (struct cell *parent = c->parent; - parent != NULL && !parent->hydro.do_sub_limiter; + parent != NULL && + !cell_get_flag(parent, cell_flag_do_hydro_sub_limiter); parent = parent->parent) { /* Mark this cell for limiting */ - parent->hydro.do_sub_limiter = 1; + cell_set_flag(parent, cell_flag_do_hydro_sub_limiter); if (parent == c->super) { #ifdef SWIFT_DEBUG_CHECKS @@ -2360,9 +2359,9 @@ void cell_activate_hydro_sorts_up(struct cell *c, struct scheduler *s) { if (c->nodeID == engine_rank) cell_activate_drift_part(c, s); } else { for (struct cell *parent = c->parent; - parent != NULL && !parent->hydro.do_sub_sort; + parent != NULL && !cell_get_flag(parent, cell_flag_do_hydro_sub_sort); parent = parent->parent) { - parent->hydro.do_sub_sort = 1; + cell_set_flag(parent, cell_flag_do_hydro_sub_sort); if (parent == c->hydro.super) { #ifdef SWIFT_DEBUG_CHECKS if (parent->hydro.sorts == NULL) @@ -2415,9 +2414,9 @@ void cell_activate_stars_sorts_up(struct cell *c, struct scheduler *s) { } } else { for (struct cell *parent = c->parent; - parent != NULL && !parent->stars.do_sub_sort; + parent != NULL && !cell_get_flag(parent, cell_flag_do_stars_sub_sort); parent = parent->parent) { - parent->stars.do_sub_sort = 1; + cell_set_flag(parent, cell_flag_do_stars_sub_sort); if (parent == c->hydro.super) { #ifdef SWIFT_DEBUG_CHECKS if (parent->stars.sorts == NULL) @@ -3516,7 +3515,7 @@ void cell_drift_part(struct cell *c, const struct engine *e, int force) { float cell_h_max = 0.f; /* Drift irrespective of cell flags? */ - force |= c->hydro.do_drift; + force = (force || cell_get_flag(c, cell_flag_do_hydro_drift)); #ifdef SWIFT_DEBUG_CHECKS /* Check that we only drift local cells. */ @@ -3529,8 +3528,7 @@ void cell_drift_part(struct cell *c, const struct engine *e, int force) { /* Early abort? */ if (c->hydro.count == 0) { /* Clear the drift flags. */ - c->hydro.do_drift = 0; - c->hydro.do_sub_drift = 0; + cell_clear_flag(c, cell_flag_do_hydro_drift | cell_flag_do_hydro_sub_drift); /* Update the time of the last drift */ c->hydro.ti_old_part = ti_current; @@ -3541,7 +3539,8 @@ void cell_drift_part(struct cell *c, const struct engine *e, int force) { /* Ok, we have some particles somewhere in the hierarchy to drift */ /* Are we not in a leaf ? */ - if (c->split && (force || c->hydro.do_sub_drift)) { + if (c->split && (force || cell_get_flag(c, cell_flag_do_hydro_sub_drift))) { + /* Loop over the progeny and collect their data. */ for (int k = 0; k < 8; k++) { if (c->progeny[k] != NULL) { @@ -3674,8 +3673,7 @@ void cell_drift_part(struct cell *c, const struct engine *e, int force) { } /* Clear the drift flags. */ - c->hydro.do_drift = 0; - c->hydro.do_sub_drift = 0; + cell_clear_flag(c, cell_flag_do_hydro_drift | cell_flag_do_hydro_sub_drift); } /** @@ -3694,7 +3692,7 @@ void cell_drift_gpart(struct cell *c, const struct engine *e, int force) { struct gpart *const gparts = c->grav.parts; /* Drift irrespective of cell flags? */ - force |= c->grav.do_drift; + force = (force || cell_get_flag(c, cell_flag_do_grav_drift)); #ifdef SWIFT_DEBUG_CHECKS /* Check that we only drift local cells. */ @@ -3707,8 +3705,7 @@ void cell_drift_gpart(struct cell *c, const struct engine *e, int force) { /* Early abort? */ if (c->grav.count == 0) { /* Clear the drift flags. */ - c->grav.do_drift = 0; - c->grav.do_sub_drift = 0; + cell_clear_flag(c, cell_flag_do_grav_drift | cell_flag_do_grav_sub_drift); /* Update the time of the last drift */ c->grav.ti_old_part = ti_current; @@ -3719,7 +3716,8 @@ void cell_drift_gpart(struct cell *c, const struct engine *e, int force) { /* Ok, we have some particles somewhere in the hierarchy to drift */ /* Are we not in a leaf ? */ - if (c->split && (force || c->grav.do_sub_drift)) { + if (c->split && (force || cell_get_flag(c, cell_flag_do_grav_sub_drift))) { + /* Loop over the progeny and collect their data. */ for (int k = 0; k < 8; k++) { if (c->progeny[k] != NULL) { @@ -3792,8 +3790,7 @@ void cell_drift_gpart(struct cell *c, const struct engine *e, int force) { } /* Clear the drift flags. */ - c->grav.do_drift = 0; - c->grav.do_sub_drift = 0; + cell_clear_flag(c, cell_flag_do_grav_drift | cell_flag_do_grav_sub_drift); } /** @@ -3818,7 +3815,7 @@ void cell_drift_spart(struct cell *c, const struct engine *e, int force) { float cell_h_max = 0.f; /* Drift irrespective of cell flags? */ - force |= c->stars.do_drift; + force = (force || cell_get_flag(c, cell_flag_do_stars_drift)); #ifdef SWIFT_DEBUG_CHECKS /* Check that we only drift local cells. */ @@ -3831,8 +3828,7 @@ void cell_drift_spart(struct cell *c, const struct engine *e, int force) { /* Early abort? */ if (c->stars.count == 0) { /* Clear the drift flags. */ - c->stars.do_drift = 0; - c->stars.do_sub_drift = 0; + cell_clear_flag(c, cell_flag_do_stars_drift | cell_flag_do_stars_sub_drift); /* Update the time of the last drift */ c->stars.ti_old_part = ti_current; @@ -3843,7 +3839,8 @@ void cell_drift_spart(struct cell *c, const struct engine *e, int force) { /* Ok, we have some particles somewhere in the hierarchy to drift */ /* Are we not in a leaf ? */ - if (c->split && (force || c->stars.do_sub_drift)) { + if (c->split && (force || cell_get_flag(c, cell_flag_do_stars_sub_drift))) { + /* Loop over the progeny and collect their data. */ for (int k = 0; k < 8; k++) { if (c->progeny[k] != NULL) { @@ -3956,8 +3953,7 @@ void cell_drift_spart(struct cell *c, const struct engine *e, int force) { } /* Clear the drift flags. */ - c->stars.do_drift = 0; - c->stars.do_sub_drift = 0; + cell_clear_flag(c, cell_flag_do_stars_drift | cell_flag_do_stars_sub_drift); } /** @@ -3982,7 +3978,7 @@ void cell_drift_bpart(struct cell *c, const struct engine *e, int force) { float cell_h_max = 0.f; /* Drift irrespective of cell flags? */ - force |= c->black_holes.do_drift; + force = (force || cell_get_flag(c, cell_flag_do_bh_drift)); #ifdef SWIFT_DEBUG_CHECKS /* Check that we only drift local cells. */ @@ -3996,8 +3992,7 @@ void cell_drift_bpart(struct cell *c, const struct engine *e, int force) { if (c->black_holes.count == 0) { /* Clear the drift flags. */ - c->black_holes.do_drift = 0; - c->black_holes.do_sub_drift = 0; + cell_clear_flag(c, cell_flag_do_bh_drift | cell_flag_do_bh_sub_drift); /* Update the time of the last drift */ c->black_holes.ti_old_part = ti_current; @@ -4008,7 +4003,7 @@ void cell_drift_bpart(struct cell *c, const struct engine *e, int force) { /* Ok, we have some particles somewhere in the hierarchy to drift */ /* Are we not in a leaf ? */ - if (c->split && (force || c->black_holes.do_sub_drift)) { + if (c->split && (force || cell_get_flag(c, cell_flag_do_bh_sub_drift))) { /* Loop over the progeny and collect their data. */ for (int k = 0; k < 8; k++) { @@ -4114,8 +4109,7 @@ void cell_drift_bpart(struct cell *c, const struct engine *e, int force) { } /* Clear the drift flags. */ - c->black_holes.do_drift = 0; - c->black_holes.do_sub_drift = 0; + cell_clear_flag(c, cell_flag_do_bh_drift | cell_flag_do_bh_sub_drift); } /** diff --git a/src/cell.h b/src/cell.h index cf01e9666fb7bb46fb0fc4b222f974e04f2b3a3b..e9f58eb7991656ee501d018983757e2aa7211e6d 100644 --- a/src/cell.h +++ b/src/cell.h @@ -28,6 +28,7 @@ /* Includes. */ #include +#include /* Local includes. */ #include "align.h" @@ -232,6 +233,24 @@ struct pcell_step_black_holes { float dx_max_part; }; +/** Bitmasks for the cell flags. Beware when adding flags that you don't exceed + the size of the flags variable in the struct cell. */ +enum cell_flags { + cell_flag_split = (1UL << 0), + cell_flag_do_hydro_drift = (1UL << 1), + cell_flag_do_hydro_sub_drift = (1UL << 2), + cell_flag_do_hydro_sub_sort = (1UL << 3), + cell_flag_do_hydro_limiter = (1UL << 4), + cell_flag_do_hydro_sub_limiter = (1UL << 5), + cell_flag_do_grav_drift = (1UL << 6), + cell_flag_do_grav_sub_drift = (1UL << 7), + cell_flag_do_stars_sub_sort = (1UL << 8), + cell_flag_do_stars_drift = (1UL << 9), + cell_flag_do_stars_sub_drift = (1UL << 10), + cell_flag_do_bh_drift = (1UL << 11), + cell_flag_do_bh_sub_drift = (1UL << 12) +}; + /** * @brief Cell within the tree structure. * @@ -260,6 +279,9 @@ struct cell { /*! Super cell, i.e. the highest-level parent cell with *any* task */ struct cell *super; + /*! Cell flags bit-mask. */ + volatile uint32_t flags; + /*! Hydro variables */ struct { @@ -366,28 +388,13 @@ struct cell { int hold; /*! Bit mask of sort directions that will be needed in the next timestep. */ - unsigned int requires_sorts; + uint16_t requires_sorts; /*! Bit mask of sorts that need to be computed for this cell. */ - unsigned int do_sort; + uint16_t do_sort; /*! Bit-mask indicating the sorted directions */ - unsigned int sorted; - - /*! Does this cell need to be drifted (hydro)? */ - char do_drift; - - /*! Do any of this cell's sub-cells need to be drifted (hydro)? */ - char do_sub_drift; - - /*! Do any of this cell's sub-cells need to be sorted? */ - char do_sub_sort; - - /*! Does this cell need to be limited? */ - char do_limiter; - - /*! Do any of this cell's sub-cells need to be limited? */ - char do_sub_limiter; + uint16_t sorted; #ifdef SWIFT_DEBUG_CHECKS @@ -487,12 +494,6 @@ struct cell { /*! Number of M-M tasks that are associated with this cell. */ short int nr_mm_tasks; - /*! Does this cell need to be drifted (gravity)? */ - char do_drift; - - /*! Do any of this cell's sub-cells need to be drifted (gravity)? */ - char do_sub_drift; - } grav; /*! Stars variables */ @@ -556,21 +557,18 @@ struct cell { /*! Values of dx_max_sort before the drifts, used for sub-cell tasks. */ float dx_max_sort_old; - /*! Bit mask of sort directions that will be needed in the next timestep. */ - unsigned int requires_sorts; - /*! Pointer for the sorted indices. */ struct entry *sort[13]; struct entry *sortptr; + /*! Bit mask of sort directions that will be needed in the next timestep. */ + uint16_t requires_sorts; + /*! Bit-mask indicating the sorted directions */ - unsigned int sorted; + uint16_t sorted; /*! Bit mask of sorts that need to be computed for this cell. */ - unsigned int do_sort; - - /*! Do any of this cell's sub-cells need to be sorted? */ - char do_sub_sort; + uint16_t do_sort; /*! Maximum end of (integer) time step in this cell for star tasks. */ integertime_t ti_end_min; @@ -591,12 +589,6 @@ struct cell { /*! Is the #spart data of this cell being used in a sub-cell? */ int hold; - /*! Does this cell need to be drifted (stars)? */ - char do_drift; - - /*! Do any of this cell's sub-cells need to be drifted (stars)? */ - char do_sub_drift; - /*! Star formation history struct */ struct star_formation_history sfh; @@ -657,12 +649,6 @@ struct cell { /*! Is the #bpart data of this cell being used in a sub-cell? */ int hold; - /*! Does this cell need to be drifted (black holes)? */ - char do_drift; - - /*! Do any of this cell's sub-cells need to be drifted (black holes)? */ - char do_sub_drift; - } black_holes; #ifdef WITH_MPI @@ -1248,4 +1234,22 @@ __attribute__((always_inline)) INLINE static void cell_free_stars_sorts( } } +/** Set the given flag for the given cell. */ +__attribute__((always_inline)) INLINE static void cell_set_flag(struct cell *c, + uint32_t flag) { + atomic_or(&c->flags, flag); +} + +/** Clear the given flag for the given cell. */ +__attribute__((always_inline)) INLINE static void cell_clear_flag( + struct cell *c, uint32_t flag) { + atomic_and(&c->flags, ~flag); +} + +/** Get the given flag for the given cell. */ +__attribute__((always_inline)) INLINE static int cell_get_flag( + const struct cell *c, uint32_t flag) { + return (c->flags & flag) > 0; +} + #endif /* SWIFT_CELL_H */ diff --git a/src/runner.c b/src/runner.c index e0dfbacebd9a074b515c25316f8d43272d2b3b14..ff2564d8b4b2fb4d6699b295c805cbfd2e6cbd41 100644 --- a/src/runner.c +++ b/src/runner.c @@ -908,7 +908,7 @@ void runner_do_hydro_sort(struct runner *r, struct cell *c, int flags, } else { flags &= ~c->hydro.sorted; } - if (flags == 0 && !c->hydro.do_sub_sort) return; + if (flags == 0 && !cell_get_flag(c, cell_flag_do_hydro_sub_sort)) return; /* Check that the particles have been moved to the current time */ if (flags && !cell_are_part_drifted(c, r->e)) @@ -1091,7 +1091,7 @@ void runner_do_hydro_sort(struct runner *r, struct cell *c, int flags, /* Clear the cell's sort flags. */ c->hydro.do_sort = 0; - c->hydro.do_sub_sort = 0; + cell_clear_flag(c, cell_flag_do_hydro_sub_sort); c->hydro.requires_sorts = 0; if (clock) TIMER_TOC(timer_dosort); @@ -1130,7 +1130,7 @@ void runner_do_stars_sort(struct runner *r, struct cell *c, int flags, } else { flags &= ~c->stars.sorted; } - if (flags == 0 && !c->stars.do_sub_sort) return; + if (flags == 0 && !cell_get_flag(c, cell_flag_do_stars_sub_sort)) return; /* Check that the particles have been moved to the current time */ if (flags && !cell_are_spart_drifted(c, r->e)) { @@ -1308,7 +1308,7 @@ void runner_do_stars_sort(struct runner *r, struct cell *c, int flags, /* Clear the cell's sort flags. */ c->stars.do_sort = 0; - c->stars.do_sub_sort = 0; + cell_clear_flag(c, cell_flag_do_stars_sub_sort); c->stars.requires_sorts = 0; if (clock) TIMER_TOC(timer_do_stars_sort); @@ -2937,19 +2937,19 @@ void runner_do_limiter(struct runner *r, struct cell *c, int force, int timer) { ti_gravity_beg_max = 0; /* Limit irrespective of cell flags? */ - force |= c->hydro.do_limiter; + force = (force || cell_get_flag(c, cell_flag_do_hydro_limiter)); /* Early abort? */ if (c->hydro.count == 0) { /* Clear the limiter flags. */ - c->hydro.do_limiter = 0; - c->hydro.do_sub_limiter = 0; + cell_clear_flag( + c, cell_flag_do_hydro_limiter | cell_flag_do_hydro_sub_limiter); return; } /* Loop over the progeny ? */ - if (c->split && (force || c->hydro.do_sub_limiter)) { + if (c->split && (force || cell_get_flag(c, cell_flag_do_hydro_sub_limiter))) { for (int k = 0; k < 8; k++) { if (c->progeny[k] != NULL) { struct cell *restrict cp = c->progeny[k]; @@ -3039,8 +3039,8 @@ void runner_do_limiter(struct runner *r, struct cell *c, int force, int timer) { } /* Clear the limiter flags. */ - c->hydro.do_limiter = 0; - c->hydro.do_sub_limiter = 0; + cell_clear_flag(c, + cell_flag_do_hydro_limiter | cell_flag_do_hydro_sub_limiter); if (timer) TIMER_TOC(timer_do_limiter); } diff --git a/src/space.c b/src/space.c index eac346cef7eb86cf85d9d123a270000bb759cf9f..4351c934d1bd7c69e646dc4810a04ad27617b61e 100644 --- a/src/space.c +++ b/src/space.c @@ -262,14 +262,7 @@ void space_rebuild_recycle_mapper(void *map_data, int num_elements, c->grav.parts = NULL; c->stars.parts = NULL; c->black_holes.parts = NULL; - c->hydro.do_sub_sort = 0; - c->stars.do_sub_sort = 0; - c->hydro.do_sub_drift = 0; - c->grav.do_sub_drift = 0; - c->stars.do_sub_drift = 0; - c->black_holes.do_sub_drift = 0; - c->hydro.do_sub_limiter = 0; - c->hydro.do_limiter = 0; + c->flags = 0; c->hydro.ti_end_min = -1; c->hydro.ti_end_max = -1; c->grav.ti_end_min = -1; @@ -3330,14 +3323,7 @@ void space_split_recursive(struct space *s, struct cell *c, cp->super = NULL; cp->hydro.super = NULL; cp->grav.super = NULL; - cp->hydro.do_sub_sort = 0; - cp->stars.do_sub_sort = 0; - cp->grav.do_sub_drift = 0; - cp->hydro.do_sub_drift = 0; - cp->stars.do_sub_drift = 0; - cp->black_holes.do_sub_drift = 0; - cp->hydro.do_sub_limiter = 0; - cp->hydro.do_limiter = 0; + cp->flags = 0; #ifdef WITH_MPI cp->mpi.tag = -1; #endif // WITH_MPI