From 0342d542a9830dd70f9333bf1a61240e318c31ed Mon Sep 17 00:00:00 2001
From: "Peter W. Draper" <p.w.draper@durham.ac.uk>
Date: Tue, 10 Mar 2020 14:47:21 +0000
Subject: [PATCH] Add a loop to simulate multiple steps from a single data file
---
swiftmpiproxies.c | 222 +++++++++++++++++++++++++---------------------
1 file changed, 119 insertions(+), 103 deletions(-)
diff --git a/swiftmpiproxies.c b/swiftmpiproxies.c
index e33e47e..e05c9ef 100644
--- a/swiftmpiproxies.c
+++ b/swiftmpiproxies.c
@@ -35,6 +35,9 @@ int myrank = -1;
/* Are we verbose. */
static int verbose = 0;
+/* Maximum main loops. */
+static int maxloops = 1000;
+
/* Integer types of send and recv tasks, must match log. */
static const int task_type_send = 23;
static const int task_type_recv = 24;
@@ -145,8 +148,8 @@ static void pick_logs(void) {
* @brief usage help.
*/
static void usage(char *argv[]) {
- fprintf(stderr, "Usage: %s [-vf] SWIFT_mpiuse-log-file.dat\n", argv[0]);
- fprintf(stderr, " options: -v verbose\n");
+ fprintf(stderr, "Usage: %s [-vn] SWIFT_mpiuse-log-file.dat\n", argv[0]);
+ fprintf(stderr, " options: -n maxloops, -v verbose\n");
fflush(stderr);
}
@@ -169,11 +172,14 @@ int main(int argc, char *argv[]) {
if (res != MPI_SUCCESS)
error("Call to MPI_Comm_rank failed with error %i.", res);
- /* Handle the command-line, we expect a mpiuse data file to read and various
- * options. */
+ /* Handle the command-line, we expect a mpiuse data file to read and
+ * various options. */
int opt;
- while ((opt = getopt(argc, argv, "v")) != -1) {
+ while ((opt = getopt(argc, argv, "vn:")) != -1) {
switch (opt) {
+ case 'n':
+ maxloops = atoi(optarg);
+ break;
case 'v':
verbose = 1;
break;
@@ -213,110 +219,120 @@ int main(int argc, char *argv[]) {
MPI_Request req_pcells_out[nr_send_pcells];
int pcells_size[nr_send_pcells];
- /* XXX note in SWIFT we use the threadpool to launch these. That may
- * matter. */
- for (int k = 0; k < nr_send_pcells; k++) {
- struct mpiuse_log_entry *log = send_pcells[k];
-
- /* Need to regenerate the tags for each other communication type. */
- int basetag = myrank * proxy_tag_shift;
-
- /* Start Isend counts of pcells. Really just the size of the buffer we're
- * about to send, SWIFT sends the count. */
- int size = log->size;
- res =
+ /* Loop over next section a number of times to simulate rebuild steps in
+ * SWIFT. */
+ for (int nloop = 0; nloop < maxloops; nloop++) {
+ MPI_Barrier(MPI_COMM_WORLD);
+ if (myrank == 0)
+ message("*** Proxy simulation exchange loop: %d ***", nloop);
+
+ /* XXX note in SWIFT we use the threadpool to launch these. That may
+ * matter. */
+ for (int k = 0; k < nr_send_pcells; k++) {
+ struct mpiuse_log_entry *log = send_pcells[k];
+
+ /* Need to regenerate the tags for each other communication type. */
+ int basetag = myrank * proxy_tag_shift;
+
+ /* Start Isend counts of pcells. Really just the size of the buffer we're
+ * about to send, SWIFT sends the count. */
+ int size = log->size;
+ res =
MPI_Isend(&size, 1, MPI_INT, log->otherrank, basetag + proxy_tag_count,
MPI_COMM_WORLD, &req_send_counts[k]);
- if (res != MPI_SUCCESS) error("Counts MPI_Isend failed.");
-
- /* Start Isend of pcells. */
- log->data = calloc(log->size, 1);
-
- /* Fill data with a pattern for checking on arrival. */
- datacheck_fill(0, log->size, log->data);
- res = MPI_Isend(log->data, log->size, MPI_BYTE, log->otherrank,
- basetag + proxy_tag_cells, MPI_COMM_WORLD,
- &req_pcells_out[k]);
- if (res != MPI_SUCCESS) error("Pcell MPI_Isend failed.");
-
- /* Start Irecv counts of pcells from other rank. */
- basetag = log->otherrank * proxy_tag_shift;
- res = MPI_Irecv(&pcells_size[k], 1, MPI_INT, log->otherrank,
- basetag + proxy_tag_count, MPI_COMM_WORLD,
- &req_recv_counts[k]);
- if (res != MPI_SUCCESS) error("Counts MPI_Irecv failed.");
- }
- message("All counts requests and pcell sends are launched");
-
- /* Now wait for any of the counts irecvs to complete and then create the
- * irecv for the pcells. */
- void *pcells_in[nr_send_pcells];
- MPI_Request req_pcells_in[nr_send_pcells];
- for (int k = 0; k < nr_send_pcells; k++) {
- int pid = MPI_UNDEFINED;
- MPI_Status status;
- res = MPI_Waitany(nr_send_pcells, req_recv_counts, &pid, &status);
- if (res != MPI_SUCCESS || pid == MPI_UNDEFINED)
- error("MPI_Waitany failed.");
- if (verbose) message("Counts received for proxy %d", pid);
-
- struct mpiuse_log_entry *log = send_pcells[pid];
- int basetag = log->otherrank * proxy_tag_shift;
-
- pcells_in[pid] = calloc(pcells_size[pid], 1);
-
- /* Fill data with a pattern for checking when overwritten. */
- datacheck_fill(1, pcells_size[pid], pcells_in[pid]);
- res = MPI_Irecv(pcells_in[pid], pcells_size[pid], MPI_BYTE, log->otherrank,
- basetag + proxy_tag_cells, MPI_COMM_WORLD,
- &req_pcells_in[pid]);
-
- if (res != MPI_SUCCESS) error("Pcell MPI_Irecv failed.");
- }
- message("All proxy cell counts have arrived");
-
- /* Waitall for all Isend counts to complete. */
- res = MPI_Waitall(nr_send_pcells, req_send_counts, MPI_STATUSES_IGNORE);
- if (res != MPI_SUCCESS) error("Waitall for counts Isend failed.");
-
- /* Now wait for the pcell irecvs to complete, so we receive the pcells,
- * which would be unpacked in SWIFT. */
- for (int k = 0; k < nr_send_pcells; k++) {
- int pid = MPI_UNDEFINED;
- MPI_Status status;
- res = MPI_Waitany(nr_send_pcells, req_pcells_in, &pid, &status);
- if (res != MPI_SUCCESS || pid == MPI_UNDEFINED)
- error("MPI_Waitany failed.");
-
- /* Check the data received is correct. */
- struct mpiuse_log_entry *log = send_pcells[pid];
- if (!datacheck_test(0, pcells_size[pid], pcells_in[pid])) {
- if (!datacheck_test(1, pcells_size[pid], pcells_in[pid])) {
- error("Received data is not modified");
+ if (res != MPI_SUCCESS) error("Counts MPI_Isend failed.");
+
+ /* Start Isend of pcells. */
+ log->data = calloc(log->size, 1);
+
+ /* Fill data with a pattern for checking on arrival. */
+ datacheck_fill(0, log->size, log->data);
+ res = MPI_Isend(log->data, log->size, MPI_BYTE, log->otherrank,
+ basetag + proxy_tag_cells, MPI_COMM_WORLD,
+ &req_pcells_out[k]);
+ if (res != MPI_SUCCESS) error("Pcell MPI_Isend failed.");
+
+ /* Start Irecv counts of pcells from other rank. */
+ basetag = log->otherrank * proxy_tag_shift;
+ res = MPI_Irecv(&pcells_size[k], 1, MPI_INT, log->otherrank,
+ basetag + proxy_tag_count, MPI_COMM_WORLD,
+ &req_recv_counts[k]);
+ if (res != MPI_SUCCESS) error("Counts MPI_Irecv failed.");
+ }
+ message("All counts requests and pcell sends are launched");
+
+ /* Now wait for any of the counts irecvs to complete and then create the
+ * irecv for the pcells. */
+ void *pcells_in[nr_send_pcells];
+ MPI_Request req_pcells_in[nr_send_pcells];
+ for (int k = 0; k < nr_send_pcells; k++) {
+ int pid = MPI_UNDEFINED;
+ MPI_Status status;
+ res = MPI_Waitany(nr_send_pcells, req_recv_counts, &pid, &status);
+ if (res != MPI_SUCCESS || pid == MPI_UNDEFINED)
+ error("MPI_Waitany failed.");
+ if (verbose) message("Counts received for proxy %d", pid);
+
+ struct mpiuse_log_entry *log = send_pcells[pid];
+ int basetag = log->otherrank * proxy_tag_shift;
+
+ pcells_in[pid] = calloc(pcells_size[pid], 1);
+
+ /* Fill data with a pattern for checking when overwritten. */
+ datacheck_fill(1, pcells_size[pid], pcells_in[pid]);
+ res = MPI_Irecv(pcells_in[pid], pcells_size[pid], MPI_BYTE, log->otherrank,
+ basetag + proxy_tag_cells, MPI_COMM_WORLD,
+ &req_pcells_in[pid]);
+
+ if (res != MPI_SUCCESS) error("Pcell MPI_Irecv failed.");
+ }
+ message("All proxy cell counts have arrived");
+
+ /* Waitall for all Isend counts to complete. */
+ res = MPI_Waitall(nr_send_pcells, req_send_counts, MPI_STATUSES_IGNORE);
+ if (res != MPI_SUCCESS) error("Waitall for counts Isend failed.");
+
+ /* Now wait for the pcell irecvs to complete, so we receive the pcells,
+ * which would be unpacked in SWIFT. */
+ for (int k = 0; k < nr_send_pcells; k++) {
+ int pid = MPI_UNDEFINED;
+ MPI_Status status;
+ res = MPI_Waitany(nr_send_pcells, req_pcells_in, &pid, &status);
+ if (res != MPI_SUCCESS || pid == MPI_UNDEFINED)
+ error("MPI_Waitany failed.");
+
+ /* Check the data received is correct. */
+ if (!datacheck_test(0, pcells_size[pid], pcells_in[pid])) {
+ if (!datacheck_test(1, pcells_size[pid], pcells_in[pid])) {
+ error("Received data is not modified");
+ } else {
+ error("Received data is corrupt");
+ }
} else {
- error("Received data is corrupt");
+ message("Received data is correct");
}
- } else {
- message("Received data is correct");
+ free(pcells_in[pid]);
+ pcells_in[pid] = NULL;
}
- free(pcells_in[pid]);
- }
- message("All proxy cells have arrived");
-
- /* Waitall for Isend of pcells to complete. */
- res = MPI_Waitall(nr_send_pcells, req_pcells_out, MPI_STATUSES_IGNORE);
- if (res != MPI_SUCCESS) error("Waitall for pcells Isend failed.");
-
- /* Check data is unmodified. */
- for (int k = 0; k < nr_send_pcells; k++) {
- struct mpiuse_log_entry *log = send_pcells[k];
- if (!datacheck_test(0, log->size, log->data)) {
- error("Sent data has been corrupted");
- } else {
- message("Sent data is correct");
+ message("All proxy cells have arrived");
+
+ /* Waitall for Isend of pcells to complete. */
+ res = MPI_Waitall(nr_send_pcells, req_pcells_out, MPI_STATUSES_IGNORE);
+ if (res != MPI_SUCCESS) error("Waitall for pcells Isend failed.");
+
+ /* Check data is unmodified. */
+ for (int k = 0; k < nr_send_pcells; k++) {
+ struct mpiuse_log_entry *log = send_pcells[k];
+ if (!datacheck_test(0, log->size, log->data)) {
+ error("Sent data has been corrupted");
+ } else {
+ message("Sent data is correct");
+ }
+ free(log->data);
+ log->data = NULL;
}
- free(log->data);
- }
+
+ }/* nloop */
/* Shutdown MPI. */
res = MPI_Finalize();
--
GitLab