From 85af978e039dfe154abe90aecb4379fa07b11e7c Mon Sep 17 00:00:00 2001
From: Matthieu Schaller <schaller@strw.leidenuniv.nl>
Date: Tue, 31 Mar 2020 11:40:48 +0200
Subject: [PATCH] Apply the same behaviour to the accumulators used to check
the accuracy of the gravity calculation.
---
src/accumulate.h | 55 +++++++++++++++++++++++++
src/multipole.h | 6 +--
src/runner_doiact_grav.c | 86 +++++++++++++++++++++-------------------
3 files changed, 104 insertions(+), 43 deletions(-)
diff --git a/src/accumulate.h b/src/accumulate.h
index 4968b331ef..a7e8e6c1bf 100644
--- a/src/accumulate.h
+++ b/src/accumulate.h
@@ -51,6 +51,25 @@ __attribute__((always_inline)) INLINE static void accumulate_add_i(
#endif
}
+/**
+ * @brief Add x to the value stored at the location address (long long version)
+ *
+ * When SWIFT_TASKS_WITHOUT_ATOMICS is *not* defined this function uses an
+ * atomic operation.
+ *
+ * @param address The address to update.
+ * @param x The value to add to *address.
+ */
+__attribute__((always_inline)) INLINE static void accumulate_add_ll(
+ volatile long long *const address, const long long x) {
+
+#ifdef SWIFT_TASKS_WITHOUT_ATOMICS
+ *address += x;
+#else
+ atomic_add(address, x);
+#endif
+}
+
/**
* @brief Add x to the value stored at the location address (float version)
*
@@ -89,4 +108,40 @@ __attribute__((always_inline)) INLINE static void accumulate_add_d(
#endif
}
+/**
+ * @brief Add 1 to the value stored at the location address (int version)
+ *
+ * When SWIFT_TASKS_WITHOUT_ATOMICS is *not* defined this function uses an
+ * atomic operation.
+ *
+ * @param address The address to update.
+ */
+__attribute__((always_inline)) INLINE static void accumulate_inc_i(
+ volatile int *const address) {
+
+#ifdef SWIFT_TASKS_WITHOUT_ATOMICS
+ (*address)++;
+#else
+ atomic_inc(address);
+#endif
+}
+
+/**
+ * @brief Add 1 to the value stored at the location address (long long version)
+ *
+ * When SWIFT_TASKS_WITHOUT_ATOMICS is *not* defined this function uses an
+ * atomic operation.
+ *
+ * @param address The address to update.
+ */
+__attribute__((always_inline)) INLINE static void accumulate_inc_ll(
+ volatile long long *const address) {
+
+#ifdef SWIFT_TASKS_WITHOUT_ATOMICS
+ (*address)++;
+#else
+ atomic_inc(address);
+#endif
+}
+
#endif /* SWIFT_ACCUMULATE_H */
diff --git a/src/multipole.h b/src/multipole.h
index fede08c09f..867244d284 100644
--- a/src/multipole.h
+++ b/src/multipole.h
@@ -2467,12 +2467,12 @@ INLINE static void gravity_L2P(const struct grav_tensor *lb,
#ifdef SWIFT_DEBUG_CHECKS
if (lb->num_interacted == 0) error("Interacting with empty field tensor");
- atomic_add(&gp->num_interacted, lb->num_interacted);
+ accumulate_add_ll(&gp->num_interacted, lb->num_interacted);
#endif
#ifdef SWIFT_GRAVITY_FORCE_CHECKS
- atomic_add(&gp->num_interacted_m2l, lb->num_interacted_tree);
- atomic_add(&gp->num_interacted_pm, lb->num_interacted_pm);
+ accumulate_add_ll(&gp->num_interacted_m2l, lb->num_interacted_tree);
+ accumulate_add_ll(&gp->num_interacted_pm, lb->num_interacted_pm);
#endif
/* Local accumulator */
diff --git a/src/runner_doiact_grav.c b/src/runner_doiact_grav.c
index bb914dc47a..91abe9e910 100644
--- a/src/runner_doiact_grav.c
+++ b/src/runner_doiact_grav.c
@@ -264,13 +264,13 @@ static INLINE void runner_dopair_grav_pp_full(
#ifdef SWIFT_DEBUG_CHECKS
/* Update the interaction counter if it's not a padded gpart */
if (pjd < gcount_j && !gpart_is_inhibited(&gparts_j[pjd], e))
- atomic_inc(&gparts_i[pid].num_interacted);
+ accumulate_inc_ll(&gparts_i[pid].num_interacted);
#endif
#ifdef SWIFT_GRAVITY_FORCE_CHECKS
/* Update the p2p interaction counter if it's not a padded gpart */
if (pjd < gcount_j && !gpart_is_inhibited(&gparts_j[pjd], e))
- atomic_inc(&gparts_i[pid].num_interacted_p2p);
+ accumulate_inc_ll(&gparts_i[pid].num_interacted_p2p);
#endif
}
@@ -281,9 +281,9 @@ static INLINE void runner_dopair_grav_pp_full(
ci_cache->pot[pid] += pot;
#ifdef SWIFT_GRAVITY_FORCE_CHECKS
- atomic_add_f(&gparts_i[pid].a_grav_p2p[0], a_x);
- atomic_add_f(&gparts_i[pid].a_grav_p2p[1], a_y);
- atomic_add_f(&gparts_i[pid].a_grav_p2p[2], a_z);
+ accumulate_add_f(&gparts_i[pid].a_grav_p2p[0], a_x);
+ accumulate_add_f(&gparts_i[pid].a_grav_p2p[1], a_y);
+ accumulate_add_f(&gparts_i[pid].a_grav_p2p[2], a_z);
#endif
}
}
@@ -420,13 +420,13 @@ static INLINE void runner_dopair_grav_pp_truncated(
#ifdef SWIFT_DEBUG_CHECKS
/* Update the interaction counter if it's not a padded gpart */
if (pjd < gcount_j && !gpart_is_inhibited(&gparts_j[pjd], e))
- atomic_inc(&gparts_i[pid].num_interacted);
+ accumulate_inc_ll(&gparts_i[pid].num_interacted);
#endif
#ifdef SWIFT_GRAVITY_FORCE_CHECKS
/* Update the p2p interaction counter if it's not a padded gpart */
if (pjd < gcount_j && !gpart_is_inhibited(&gparts_j[pjd], e))
- atomic_inc(&gparts_i[pid].num_interacted_p2p);
+ accumulate_inc_ll(&gparts_i[pid].num_interacted_p2p);
#endif
}
@@ -437,9 +437,9 @@ static INLINE void runner_dopair_grav_pp_truncated(
ci_cache->pot[pid] += pot;
#ifdef SWIFT_GRAVITY_FORCE_CHECKS
- atomic_add_f(&gparts_i[pid].a_grav_p2p[0], a_x);
- atomic_add_f(&gparts_i[pid].a_grav_p2p[1], a_y);
- atomic_add_f(&gparts_i[pid].a_grav_p2p[2], a_z);
+ accumulate_add_f(&gparts_i[pid].a_grav_p2p[0], a_x);
+ accumulate_add_f(&gparts_i[pid].a_grav_p2p[1], a_y);
+ accumulate_add_f(&gparts_i[pid].a_grav_p2p[2], a_z);
#endif
}
}
@@ -565,18 +565,18 @@ static INLINE void runner_dopair_grav_pm_full(
#ifdef SWIFT_DEBUG_CHECKS
/* Update the interaction counter */
if (pid < gcount_i)
- atomic_add(&gparts_i[pid].num_interacted,
- cj->grav.multipole->m_pole.num_gpart);
+ accumulate_add_ll(&gparts_i[pid].num_interacted,
+ cj->grav.multipole->m_pole.num_gpart);
#endif
#ifdef SWIFT_GRAVITY_FORCE_CHECKS
/* Update the M2P interaction counter and forces. */
if (pid < gcount_i) {
- atomic_add(&gparts_i[pid].num_interacted_m2p,
- cj->grav.multipole->m_pole.num_gpart);
- atomic_add_f(&gparts_i[pid].a_grav_m2p[0], f_x);
- atomic_add_f(&gparts_i[pid].a_grav_m2p[1], f_y);
- atomic_add_f(&gparts_i[pid].a_grav_m2p[2], f_z);
+ accumulate_add_ll(&gparts_i[pid].num_interacted_m2p,
+ cj->grav.multipole->m_pole.num_gpart);
+ accumulate_add_f(&gparts_i[pid].a_grav_m2p[0], f_x);
+ accumulate_add_f(&gparts_i[pid].a_grav_m2p[1], f_y);
+ accumulate_add_f(&gparts_i[pid].a_grav_m2p[2], f_z);
}
#endif
}
@@ -708,18 +708,18 @@ static INLINE void runner_dopair_grav_pm_truncated(
#ifdef SWIFT_DEBUG_CHECKS
/* Update the interaction counter */
if (pid < gcount_i)
- atomic_add(&gparts_i[pid].num_interacted,
- cj->grav.multipole->m_pole.num_gpart);
+ accumulate_add_ll(&gparts_i[pid].num_interacted,
+ cj->grav.multipole->m_pole.num_gpart);
#endif
#ifdef SWIFT_GRAVITY_FORCE_CHECKS
/* Update the M2P interaction counter and forces. */
if (pid < gcount_i) {
- atomic_add(&gparts_i[pid].num_interacted_m2p,
- cj->grav.multipole->m_pole.num_gpart);
- atomic_add_f(&gparts_i[pid].a_grav_m2p[0], f_x);
- atomic_add_f(&gparts_i[pid].a_grav_m2p[1], f_y);
- atomic_add_f(&gparts_i[pid].a_grav_m2p[2], f_z);
+ accumulate_add_ll(&gparts_i[pid].num_interacted_m2p,
+ cj->grav.multipole->m_pole.num_gpart);
+ accumulate_add_f(&gparts_i[pid].a_grav_m2p[0], f_x);
+ accumulate_add_f(&gparts_i[pid].a_grav_m2p[1], f_y);
+ accumulate_add_f(&gparts_i[pid].a_grav_m2p[2], f_z);
}
#endif
}
@@ -1045,13 +1045,13 @@ static INLINE void runner_doself_grav_pp_full(
#ifdef SWIFT_DEBUG_CHECKS
/* Update the interaction counter if it's not a padded gpart */
if (pjd < gcount && !gpart_is_inhibited(&gparts[pjd], e))
- atomic_inc(&gparts[pid].num_interacted);
+ accumulate_inc_ll(&gparts[pid].num_interacted);
#endif
#ifdef SWIFT_GRAVITY_FORCE_CHECKS
/* Update the P2P interaction counter if it's not a padded gpart */
if (pjd < gcount && !gpart_is_inhibited(&gparts[pjd], e))
- atomic_inc(&gparts[pid].num_interacted_p2p);
+ accumulate_inc_ll(&gparts[pid].num_interacted_p2p);
#endif
}
@@ -1062,9 +1062,9 @@ static INLINE void runner_doself_grav_pp_full(
ci_cache->pot[pid] += pot;
#ifdef SWIFT_GRAVITY_FORCE_CHECKS
- atomic_add_f(&gparts[pid].a_grav_p2p[0], a_x);
- atomic_add_f(&gparts[pid].a_grav_p2p[1], a_y);
- atomic_add_f(&gparts[pid].a_grav_p2p[2], a_z);
+ accumulate_add_f(&gparts[pid].a_grav_p2p[0], a_x);
+ accumulate_add_f(&gparts[pid].a_grav_p2p[1], a_y);
+ accumulate_add_f(&gparts[pid].a_grav_p2p[2], a_z);
#endif
}
}
@@ -1184,13 +1184,13 @@ static INLINE void runner_doself_grav_pp_truncated(
#ifdef SWIFT_DEBUG_CHECKS
/* Update the interaction counter if it's not a padded gpart */
if (pjd < gcount && !gpart_is_inhibited(&gparts[pjd], e))
- atomic_inc(&gparts[pid].num_interacted);
+ accumulate_inc_ll(&gparts[pid].num_interacted);
#endif
#ifdef SWIFT_GRAVITY_FORCE_CHECKS
/* Update the P2P interaction counter if it's not a padded gpart */
if (pjd < gcount && !gpart_is_inhibited(&gparts[pjd], e))
- atomic_inc(&gparts[pid].num_interacted_p2p);
+ accumulate_inc_ll(&gparts[pid].num_interacted_p2p);
#endif
}
@@ -1201,9 +1201,9 @@ static INLINE void runner_doself_grav_pp_truncated(
ci_cache->pot[pid] += pot;
#ifdef SWIFT_GRAVITY_FORCE_CHECKS
- atomic_add_f(&gparts[pid].a_grav_p2p[0], a_x);
- atomic_add_f(&gparts[pid].a_grav_p2p[1], a_y);
- atomic_add_f(&gparts[pid].a_grav_p2p[2], a_z);
+ accumulate_add_f(&gparts[pid].a_grav_p2p[0], a_x);
+ accumulate_add_f(&gparts[pid].a_grav_p2p[1], a_y);
+ accumulate_add_f(&gparts[pid].a_grav_p2p[2], a_z);
#endif
}
}
@@ -1678,17 +1678,21 @@ void runner_dopair_recursive_grav(struct runner *r, struct cell *ci,
#ifdef SWIFT_DEBUG_CHECKS
if (cell_is_active_gravity(ci, e))
- atomic_add(&multi_i->pot.num_interacted, multi_j->m_pole.num_gpart);
+ accumulate_add_ll(&multi_i->pot.num_interacted,
+ multi_j->m_pole.num_gpart);
if (cell_is_active_gravity(cj, e))
- atomic_add(&multi_j->pot.num_interacted, multi_i->m_pole.num_gpart);
+ accumulate_add_ll(&multi_j->pot.num_interacted,
+ multi_i->m_pole.num_gpart);
#endif
#ifdef SWIFT_GRAVITY_FORCE_CHECKS
/* Need to account for the interactions we missed */
if (cell_is_active_gravity(ci, e))
- atomic_add(&multi_i->pot.num_interacted_pm, multi_j->m_pole.num_gpart);
+ accumulate_add_ll(&multi_i->pot.num_interacted_pm,
+ multi_j->m_pole.num_gpart);
if (cell_is_active_gravity(cj, e))
- atomic_add(&multi_j->pot.num_interacted_pm, multi_i->m_pole.num_gpart);
+ accumulate_add_ll(&multi_j->pot.num_interacted_pm,
+ multi_i->m_pole.num_gpart);
#endif
return;
}
@@ -1894,12 +1898,14 @@ void runner_do_grav_long_range(struct runner *r, struct cell *ci, int timer) {
#ifdef SWIFT_DEBUG_CHECKS
/* Need to account for the interactions we missed */
- atomic_add(&multi_i->pot.num_interacted, multi_j->m_pole.num_gpart);
+ accumulate_add_ll(&multi_i->pot.num_interacted,
+ multi_j->m_pole.num_gpart);
#endif
#ifdef SWIFT_GRAVITY_FORCE_CHECKS
/* Need to account for the interactions we missed */
- atomic_add(&multi_i->pot.num_interacted_pm, multi_j->m_pole.num_gpart);
+ accumulate_add_ll(&multi_i->pot.num_interacted_pm,
+ multi_j->m_pole.num_gpart);
#endif
/* Record that this multipole received a contribution */
--
GitLab