Skip to content
Snippets Groups Projects
Commit 0342d542 authored by Peter W. Draper's avatar Peter W. Draper
Browse files

Add a loop to simulate multiple steps from a single data file

parent 82473678
Branches
Tags
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 119 additions and 103 deletions
swiftmpiproxies.c
+ 119
− 103
View file @ 0342d542
......@@ -35,6 +35,9 @@ int myrank = -1;
/* Are we verbose. */
static int verbose = 0;
/* Maximum main loops. */
static int maxloops = 1000;
/* Integer types of send and recv tasks, must match log. */
static const int task_type_send = 23;
static const int task_type_recv = 24;
......@@ -145,8 +148,8 @@ static void pick_logs(void) {
* @brief usage help.
*/
static void usage(char *argv[]) {
fprintf(stderr, "Usage: %s [-vf] SWIFT_mpiuse-log-file.dat\n", argv[0]);
fprintf(stderr, " options: -v verbose\n");
fprintf(stderr, "Usage: %s [-vn] SWIFT_mpiuse-log-file.dat\n", argv[0]);
fprintf(stderr, " options: -n maxloops, -v verbose\n");
fflush(stderr);
}
......@@ -169,11 +172,14 @@ int main(int argc, char *argv[]) {
if (res != MPI_SUCCESS)
error("Call to MPI_Comm_rank failed with error %i.", res);
/* Handle the command-line, we expect a mpiuse data file to read and various
* options. */
/* Handle the command-line, we expect a mpiuse data file to read and
* various options. */
int opt;
while ((opt = getopt(argc, argv, "v")) != -1) {
while ((opt = getopt(argc, argv, "vn:")) != -1) {
switch (opt) {
case 'n':
maxloops = atoi(optarg);
break;
case 'v':
verbose = 1;
break;
......@@ -213,110 +219,120 @@ int main(int argc, char *argv[]) {
MPI_Request req_pcells_out[nr_send_pcells];
int pcells_size[nr_send_pcells];
/* XXX note in SWIFT we use the threadpool to launch these. That may
* matter. */
for (int k = 0; k < nr_send_pcells; k++) {
struct mpiuse_log_entry *log = send_pcells[k];
/* Need to regenerate the tags for each other communication type. */
int basetag = myrank * proxy_tag_shift;
/* Start Isend counts of pcells. Really just the size of the buffer we're
* about to send, SWIFT sends the count. */
int size = log->size;
res =
/* Loop over next section a number of times to simulate rebuild steps in
* SWIFT. */
for (int nloop = 0; nloop < maxloops; nloop++) {
MPI_Barrier(MPI_COMM_WORLD);
if (myrank == 0)
message("*** Proxy simulation exchange loop: %d ***", nloop);
/* XXX note in SWIFT we use the threadpool to launch these. That may
* matter. */
for (int k = 0; k < nr_send_pcells; k++) {
struct mpiuse_log_entry *log = send_pcells[k];
/* Need to regenerate the tags for each other communication type. */
int basetag = myrank * proxy_tag_shift;
/* Start Isend counts of pcells. Really just the size of the buffer we're
* about to send, SWIFT sends the count. */
int size = log->size;
res =
MPI_Isend(&size, 1, MPI_INT, log->otherrank, basetag + proxy_tag_count,
MPI_COMM_WORLD, &req_send_counts[k]);
if (res != MPI_SUCCESS) error("Counts MPI_Isend failed.");
/* Start Isend of pcells. */
log->data = calloc(log->size, 1);
/* Fill data with a pattern for checking on arrival. */
datacheck_fill(0, log->size, log->data);
res = MPI_Isend(log->data, log->size, MPI_BYTE, log->otherrank,
basetag + proxy_tag_cells, MPI_COMM_WORLD,
&req_pcells_out[k]);
if (res != MPI_SUCCESS) error("Pcell MPI_Isend failed.");
/* Start Irecv counts of pcells from other rank. */
basetag = log->otherrank * proxy_tag_shift;
res = MPI_Irecv(&pcells_size[k], 1, MPI_INT, log->otherrank,
basetag + proxy_tag_count, MPI_COMM_WORLD,
&req_recv_counts[k]);
if (res != MPI_SUCCESS) error("Counts MPI_Irecv failed.");
}
message("All counts requests and pcell sends are launched");
/* Now wait for any of the counts irecvs to complete and then create the
* irecv for the pcells. */
void *pcells_in[nr_send_pcells];
MPI_Request req_pcells_in[nr_send_pcells];
for (int k = 0; k < nr_send_pcells; k++) {
int pid = MPI_UNDEFINED;
MPI_Status status;
res = MPI_Waitany(nr_send_pcells, req_recv_counts, &pid, &status);
if (res != MPI_SUCCESS || pid == MPI_UNDEFINED)
error("MPI_Waitany failed.");
if (verbose) message("Counts received for proxy %d", pid);
struct mpiuse_log_entry *log = send_pcells[pid];
int basetag = log->otherrank * proxy_tag_shift;
pcells_in[pid] = calloc(pcells_size[pid], 1);
/* Fill data with a pattern for checking when overwritten. */
datacheck_fill(1, pcells_size[pid], pcells_in[pid]);
res = MPI_Irecv(pcells_in[pid], pcells_size[pid], MPI_BYTE, log->otherrank,
basetag + proxy_tag_cells, MPI_COMM_WORLD,
&req_pcells_in[pid]);
if (res != MPI_SUCCESS) error("Pcell MPI_Irecv failed.");
}
message("All proxy cell counts have arrived");
/* Waitall for all Isend counts to complete. */
res = MPI_Waitall(nr_send_pcells, req_send_counts, MPI_STATUSES_IGNORE);
if (res != MPI_SUCCESS) error("Waitall for counts Isend failed.");
/* Now wait for the pcell irecvs to complete, so we receive the pcells,
* which would be unpacked in SWIFT. */
for (int k = 0; k < nr_send_pcells; k++) {
int pid = MPI_UNDEFINED;
MPI_Status status;
res = MPI_Waitany(nr_send_pcells, req_pcells_in, &pid, &status);
if (res != MPI_SUCCESS || pid == MPI_UNDEFINED)
error("MPI_Waitany failed.");
/* Check the data received is correct. */
struct mpiuse_log_entry *log = send_pcells[pid];
if (!datacheck_test(0, pcells_size[pid], pcells_in[pid])) {
if (!datacheck_test(1, pcells_size[pid], pcells_in[pid])) {
error("Received data is not modified");
if (res != MPI_SUCCESS) error("Counts MPI_Isend failed.");
/* Start Isend of pcells. */
log->data = calloc(log->size, 1);
/* Fill data with a pattern for checking on arrival. */
datacheck_fill(0, log->size, log->data);
res = MPI_Isend(log->data, log->size, MPI_BYTE, log->otherrank,
basetag + proxy_tag_cells, MPI_COMM_WORLD,
&req_pcells_out[k]);
if (res != MPI_SUCCESS) error("Pcell MPI_Isend failed.");
/* Start Irecv counts of pcells from other rank. */
basetag = log->otherrank * proxy_tag_shift;
res = MPI_Irecv(&pcells_size[k], 1, MPI_INT, log->otherrank,
basetag + proxy_tag_count, MPI_COMM_WORLD,
&req_recv_counts[k]);
if (res != MPI_SUCCESS) error("Counts MPI_Irecv failed.");
}
message("All counts requests and pcell sends are launched");
/* Now wait for any of the counts irecvs to complete and then create the
* irecv for the pcells. */
void *pcells_in[nr_send_pcells];
MPI_Request req_pcells_in[nr_send_pcells];
for (int k = 0; k < nr_send_pcells; k++) {
int pid = MPI_UNDEFINED;
MPI_Status status;
res = MPI_Waitany(nr_send_pcells, req_recv_counts, &pid, &status);
if (res != MPI_SUCCESS || pid == MPI_UNDEFINED)
error("MPI_Waitany failed.");
if (verbose) message("Counts received for proxy %d", pid);
struct mpiuse_log_entry *log = send_pcells[pid];
int basetag = log->otherrank * proxy_tag_shift;
pcells_in[pid] = calloc(pcells_size[pid], 1);
/* Fill data with a pattern for checking when overwritten. */
datacheck_fill(1, pcells_size[pid], pcells_in[pid]);
res = MPI_Irecv(pcells_in[pid], pcells_size[pid], MPI_BYTE, log->otherrank,
basetag + proxy_tag_cells, MPI_COMM_WORLD,
&req_pcells_in[pid]);
if (res != MPI_SUCCESS) error("Pcell MPI_Irecv failed.");
}
message("All proxy cell counts have arrived");
/* Waitall for all Isend counts to complete. */
res = MPI_Waitall(nr_send_pcells, req_send_counts, MPI_STATUSES_IGNORE);
if (res != MPI_SUCCESS) error("Waitall for counts Isend failed.");
/* Now wait for the pcell irecvs to complete, so we receive the pcells,
* which would be unpacked in SWIFT. */
for (int k = 0; k < nr_send_pcells; k++) {
int pid = MPI_UNDEFINED;
MPI_Status status;
res = MPI_Waitany(nr_send_pcells, req_pcells_in, &pid, &status);
if (res != MPI_SUCCESS || pid == MPI_UNDEFINED)
error("MPI_Waitany failed.");
/* Check the data received is correct. */
if (!datacheck_test(0, pcells_size[pid], pcells_in[pid])) {
if (!datacheck_test(1, pcells_size[pid], pcells_in[pid])) {
error("Received data is not modified");
} else {
error("Received data is corrupt");
}
} else {
error("Received data is corrupt");
message("Received data is correct");
}
} else {
message("Received data is correct");
free(pcells_in[pid]);
pcells_in[pid] = NULL;
}
free(pcells_in[pid]);
}
message("All proxy cells have arrived");
/* Waitall for Isend of pcells to complete. */
res = MPI_Waitall(nr_send_pcells, req_pcells_out, MPI_STATUSES_IGNORE);
if (res != MPI_SUCCESS) error("Waitall for pcells Isend failed.");
/* Check data is unmodified. */
for (int k = 0; k < nr_send_pcells; k++) {
struct mpiuse_log_entry *log = send_pcells[k];
if (!datacheck_test(0, log->size, log->data)) {
error("Sent data has been corrupted");
} else {
message("Sent data is correct");
message("All proxy cells have arrived");
/* Waitall for Isend of pcells to complete. */
res = MPI_Waitall(nr_send_pcells, req_pcells_out, MPI_STATUSES_IGNORE);
if (res != MPI_SUCCESS) error("Waitall for pcells Isend failed.");
/* Check data is unmodified. */
for (int k = 0; k < nr_send_pcells; k++) {
struct mpiuse_log_entry *log = send_pcells[k];
if (!datacheck_test(0, log->size, log->data)) {
error("Sent data has been corrupted");
} else {
message("Sent data is correct");
}
free(log->data);
log->data = NULL;
}
free(log->data);
}
}/* nloop */
/* Shutdown MPI. */
res = MPI_Finalize();
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment