From 150aed3782f99237667dd67bd7576d0c7304efc2 Mon Sep 17 00:00:00 2001
From: Matthieu Schaller <schaller@strw.leidenuniv.nl>
Date: Sun, 21 Jun 2020 19:26:49 +0200
Subject: [PATCH] Do the cell unpacking in parallel by having one thread in the
threapool per request/proxy
---
src/proxy.c | 52 ++++++++++++++++++++++++++++++++++++++++------------
1 file changed, 40 insertions(+), 12 deletions(-)
diff --git a/src/proxy.c b/src/proxy.c
index ed002132df..efbd14c325 100644
--- a/src/proxy.c
+++ b/src/proxy.c
@@ -323,6 +323,40 @@ void proxy_cells_exchange_first_mapper(void *map_data, int num_elements,
}
}
+struct unpack_mapper_data {
+
+ struct space *s;
+ const int with_gravity;
+ MPI_Request *reqs_in;
+ struct proxy *proxies;
+};
+
+void proxy_cells_wait_and_unpack_mapper(void *map_data, int num_elements,
+ void *extra_data) {
+
+ MPI_Request *reqs = (MPI_Request *)map_data;
+ struct unpack_mapper_data *data = (struct unpack_mapper_data *)extra_data;
+ struct space *s = data->s;
+ const int with_gravity = data->with_gravity;
+ MPI_Request *reqs_in = data->reqs_in;
+ struct proxy *proxies = data->proxies;
+
+ for (int k = 0; k < num_elements; ++k) {
+
+ /* Wait for the data to arrive */
+ MPI_Status status;
+ if (MPI_Wait(&reqs[k], &status) != MPI_SUCCESS) error("MPI_Wait failed!");
+
+ const int i = &reqs[k] - reqs_in;
+
+ /* Un-pack the cells received in this proxy */
+ int count = 0;
+ for (int j = 0; j < proxies[i].nr_cells_in; j++)
+ count += cell_unpack(&proxies[i].pcells_in[count], proxies[i].cells_in[j],
+ s, with_gravity);
+ }
+}
+
#endif // WITH_MPI
/**
@@ -417,18 +451,12 @@ void proxy_cells_exchange(struct proxy *proxies, int num_proxies,
tic2 = getticks();
- /* Wait for each pcell array to come in from the proxies. */
- for (int k = 0; k < num_proxies; k++) {
- int pid = MPI_UNDEFINED;
- MPI_Status status;
- if (MPI_Waitany(num_proxies, reqs_in, &pid, &status) != MPI_SUCCESS ||
- pid == MPI_UNDEFINED)
- error("MPI_Waitany failed.");
- // message( "cell data from proxy %i has arrived." , pid );
- for (int count = 0, j = 0; j < proxies[pid].nr_cells_in; j++)
- count += cell_unpack(&proxies[pid].pcells_in[count],
- proxies[pid].cells_in[j], s, with_gravity);
- }
+ /* Wait for each pcell array to come in from the proxies
+ * and unpack the cells. */
+ struct unpack_mapper_data unpack_data = {s, with_gravity, reqs_in, proxies};
+ threadpool_map(&s->e->threadpool, proxy_cells_wait_and_unpack_mapper, reqs_in,
+ num_proxies, sizeof(MPI_Request), /*chunk_size=*/1,
+ /*extra_data=*/&unpack_data);
if (s->e->verbose)
message("Un-packing cells took %.3f %s.",
--
GitLab