Add command-line options and output deadtime, waiting seems to now work

mpistalls.c

@@ -10,6 +10,7 @@
  */
 #include <mpi.h>
 #include <pthread.h>
+#include <unistd.h>
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -68,6 +69,7 @@ static void *inject_thread(void *arg) {
   /* Ticks of our last attempt and ticks the first loop takes (usetics == 1). */
   ticks basetic = reqs_queue[0]->tic;
   ticks looptics = 0;
+  double deadtime = 0.0;
 
   while (ind_req < nr_reqs) {
     struct mpiuse_log_entry *log = reqs_queue[ind_req];
@@ -99,6 +101,7 @@ static void *inject_thread(void *arg) {
           message("wait greater than one second");
         }
         nanosleep(&sleep, NULL);
+        deadtime += sleep.tv_nsec;
       }
     }
 
@@ -129,7 +132,7 @@ static void *inject_thread(void *arg) {
     /* Set looptics on the first pass. Assumes MPI_Isend and MPI_Irecv are
      * equally timed. */
     if (looptics == 0 && usetics) {
-      looptics = starttics - getticks();
+      looptics = getticks() - starttics;
       if (verbose)
         message("injection loop took %.3f %s.", clocks_from_ticks(looptics),
                 clocks_getunit());
@@ -141,6 +144,7 @@ static void *inject_thread(void *arg) {
     message("%d injections completed, sends = %d, recvs = %d", ind_req,
             nr_sends, nr_recvs);
     message("remaining sends = %d, recvs = %d", todo_send, todo_recv);
+    if (usetics) message("deadtime %.3f ms", deadtime / 1.0e6);
   }
   message("took %.3f %s.", clocks_from_ticks(getticks() - starttics),
           clocks_getunit());
@@ -317,17 +321,20 @@ static void pick_logs(void) {
   qsort(reqs_queue, nr_reqs, sizeof(struct mpiuse_log_entry *), cmp_logs);
 }
 
+/**
+ * @brief usage help.
+ */
+static void usage(char *argv[]) {
+  fprintf(stderr, "Usage: %s [-vf] SWIFT_mpiuse-log-file.dat\n", argv[0]);
+  fprintf(stderr, " options: -v verbose, -f fast injections\n");
+  fflush(stderr);
+}
+
 /**
  * @brief main function.
  */
 int main(int argc, char *argv[]) {
 
-  /* First we read the SWIFT MPI logger output that defines the communcations
-   * we will undertake and the time differences between injections into the
-   * queues. */
-  mpiuse_log_restore("testdata/mpiuse_report-step1.dat");
-  int nranks = mpiuse_nr_ranks();
-
   /* Initiate MPI. */
   int prov = 0;
   int res = MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &prov);
@@ -338,15 +345,40 @@ int main(int argc, char *argv[]) {
   res = MPI_Comm_size(MPI_COMM_WORLD, &nr_nodes);
   if (res != MPI_SUCCESS) error("MPI_Comm_size failed with error %i.", res);
 
+  res = MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
+  if (res != MPI_SUCCESS)
+    error("Call to MPI_Comm_rank failed with error %i.", res);
+
+  /* Handle the command-line, we expect a mpiuse data file to read and various
+   * options. */
+  int opt;
+  while ((opt = getopt(argc, argv, "vf")) != -1) {
+    switch (opt) {
+    case 'f': usetics = 0; break;
+    case 'v': verbose = 1; break;
+    default:
+      if (myrank == 0) usage(argv);
+      return 1;
+    }
+  }
+  if (optind >= argc) {
+    if (myrank == 0) usage(argv);
+    return 1;
+  }
+  char *infile = argv[optind];
+
+  /* Now we read the SWIFT MPI logger output that defines the communcations
+   * we will undertake and the time differences between injections into the
+   * queues. Note this has all ranks for a single steps, SWIFT outputs one MPI
+   * log per rank per step, so you need to combine all ranks from a step. */
+  mpiuse_log_restore(infile);
+  int nranks = mpiuse_nr_ranks();
+
   /* This should match the expected size. */
   if (nr_nodes != nranks)
     error("The number of MPI ranks %d does not match the expected value %d",
           nranks, nr_nodes);
 
-  res = MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
-  if (res != MPI_SUCCESS)
-    error("Call to MPI_Comm_rank failed with error %i.", res);
-
   /* Create communicators for each subtype of the tasks. */
   for (int i = 0; i < task_subtype_count; i++) {
     MPI_Comm_dup(MPI_COMM_WORLD, &subtypeMPI_comms[i]);