Merge branch 'multi-injectors' into multi-injectors-many

Conflicts:
	Makefile
	swiftmpistepsim.c

Makefile

 CFLAGS = -g -O0 -Wall
 
-
 all: swiftmpistepsim
 
-swiftmpistepsim: swiftmpistepsim.c mpiuse.c mpiuse.h atomic.h cycle.h clocks.h clocks.c Makefile
-	$(CC) $(CFLAGS) -o swiftmpistepsim swiftmpistepsim.c mpiuse.c clocks.c -I/usr/include/mpi -lmpi -lpthread
+swiftmpistepsim: swiftmpistepsim.c mpiuse.c mpiuse.h atomic.h cycle.h clocks.h clocks.c
+	mpicc $(CFLAGS) -o swiftmpistepsim swiftmpistepsim.c mpiuse.c clocks.c
 
 clean:
-	rm swiftmpistepsim
+	rm -f swiftmpistepsim

swiftmpistepsim.c

@@ -41,6 +41,9 @@ static int usetics = 1;
 /* The wait between injections, nanosecs. */
 static long long waitns = 0;
 
+/* Set a data pattern and check we get this back, slow... */
+static int datacheck = 0;
+
 /* Integer types of send and recv tasks, must match log. */
 static const int task_type_send = 22;
 static const int task_type_recv = 23;
@@ -70,7 +73,37 @@ static int volatile todo_send = 0;
 // XXX need to store this in the data file.
 static double log_clocks_cpufreq = 2194844448.0;
 
+/**
+ * @brief fill a data area with a pattern that can be checked for changes.
+ *
+ * @param size size of data in bytes.
+ * @param data the data to fill.
+ */
+static void datacheck_fill(size_t size, void *data) {
+  unsigned char *p = (unsigned char *)data;
+  for (size_t i = 0; i < size; i++) {
+      p[i] = 170; /* 10101010 in bits. */
+  }
+}
+
+/**
+ * @brief test a filled data area for our pattern.
+ *
+ * @param size size of data in bytes.
+ * @param data the data to fill.
+ *
+ * @result 1 on success, 0 otherwise.
+ */
+static int datacheck_test(size_t size, void *data) {
+  unsigned char *p = (unsigned char *)data;
+  for (size_t i = 0; i < size; i++) {
+    if (p[i] != 170) return 0;
+  }
+  return 1;
+}
+
 static void injection_runner(int qid) {
+
   if (verbose) message("%d: injection thread starts", qid);
   ticks starttics = getticks();
   struct mpiuse_log_entry **reqs = reqs_queue[qid];
@@ -132,6 +165,11 @@ static void injection_runner(int qid) {
     int err = 0;
     if (log->type == task_type_send) {
       log->data = calloc(log->size, 1);
+
+      /* Fill data with pattern. */
+      if (datacheck) datacheck_fill(log->size, log->data);
+
+      /* And send. */
       err = MPI_Isend(log->data, log->size, MPI_BYTE, log->otherrank, log->tag,
                       subtypeMPI_comms[log->subtype], &log->req);
 
@@ -141,6 +179,8 @@ static void injection_runner(int qid) {
       atomic_inc(&todo_send);
 
     } else {
+
+      /* Ready to receive. */
       log->data = calloc(log->size, 1);
       err = MPI_Irecv(log->data, log->size, MPI_BYTE, log->otherrank, log->tag,
                       subtypeMPI_comms[log->subtype], &log->req);
@@ -244,6 +284,12 @@ static void queue_runner(struct mpiuse_log_entry **logs, int volatile *nr_logs,
         if (dt > lmaxt) lmaxt = dt;
 
         if (res) {
+          /* Check data sent data is unchanged and received data is as
+           * expected. */
+          if (datacheck && !datacheck_test(log->size, log->data)) {
+            error("Data mismatch on completion");
+          }
+
           /* Done, clean up. */
           log->done = 1;
           log->endtic = getticks();
@@ -355,9 +401,8 @@ static void pick_logs(void) {
 
         /* And keep this log. */
         log->data = NULL;
-        reqs_queue[nr_reqs] = log;
-        nr_reqs++;
-
+        reqs[nreqs] = log;
+        nreqs++;
       } else {
         error("task type '%d' is not a known send or recv task", log->type);
       }
@@ -430,8 +475,11 @@ int main(int argc, char *argv[]) {
   /* Handle the command-line, we expect a mpiuse data file to read and various
    * options. */
   int opt;
-  while ((opt = getopt(argc, argv, "vfn:")) != -1) {
+  while ((opt = getopt(argc, argv, "vfdn:")) != -1) {
     switch (opt) {
+      case 'd':
+        datacheck = 1;
+        break;
       case 'f':
         usetics = 0;
         break;
@@ -480,6 +528,10 @@ int main(int argc, char *argv[]) {
     message("Start of MPI tests");
     message("==================");
     if (waitns > 0) message("adding fixed waits of %lld ns", waitns);
+    if (verbose) {
+      if (!usetics) message("using fast untimed injections");
+      if (datacheck) message("checking data pattern on send and recv completion");
+    }
   }
 
   /* Make three threads, one for injecting tasks and two to check for