From b9403b81a41591f33bd4ecec577ca87298418c8f Mon Sep 17 00:00:00 2001
From: Matthieu Schaller <matthieu.schaller@durham.ac.uk>
Date: Wed, 8 Aug 2018 13:24:40 +0100
Subject: [PATCH] Reduce the gravity mesh over MPI before performing the FFT.
---
src/mesh_gravity.c | 17 +++++++++++++++--
1 file changed, 15 insertions(+), 2 deletions(-)
diff --git a/src/mesh_gravity.c b/src/mesh_gravity.c
index 2359b8a9cd..68c6d07272 100644
--- a/src/mesh_gravity.c
+++ b/src/mesh_gravity.c
@@ -314,7 +314,7 @@ void pm_mesh_compute_potential(struct pm_mesh* mesh, const struct space* s,
fftw_plan inverse_plan = fftw_plan_dft_c2r_3d(
N, N, N, frho, rho, FFTW_ESTIMATE | FFTW_DESTROY_INPUT);
- const ticks tic = getticks();
+ ticks tic = getticks();
/* Zero everything */
bzero(rho, N * N * N * sizeof(double));
@@ -324,9 +324,22 @@ void pm_mesh_compute_potential(struct pm_mesh* mesh, const struct space* s,
gpart_to_mesh_CIC(&s->gparts[i], rho, N, cell_fac, dim);
if (verbose)
- message("gpart assignment took %.3f %s.",
+ message("Gpart assignment took %.3f %s.",
clocks_from_ticks(getticks() - tic), clocks_getunit());
+#ifdef WITH_MPI
+
+ MPI_Barrier(MPI_COMM_WORLD);
+ tic = getticks();
+
+ /* Merge everybody's share of the density mesh */
+ MPI_Allreduce(MPI_IN_PLACE, rho, N * N * N, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);
+
+ if (verbose)
+ message("Mesh comunication took %.3f %s.",
+ clocks_from_ticks(getticks() - tic), clocks_getunit());
+#endif
+
/* message("\n\n\n DENSITY"); */
/* print_array(rho, N); */
--
GitLab