From 6248a70e1859153fa21cdebd5fe9f5b738a68caf Mon Sep 17 00:00:00 2001
From: "Peter W. Draper" <p.w.draper@durham.ac.uk>
Date: Tue, 13 Aug 2019 17:52:42 +0100
Subject: [PATCH] Add function to dump all the queues and add to the dumper
 thread

---
 src/engine.c    |  1 +
 src/queue.c     | 31 ++++++++++++++++++++++++++
 src/queue.h     |  2 ++
 src/scheduler.c | 58 +++++++++++++++++++++++++++++++++++++++++++++++++
 src/scheduler.h |  1 +
 5 files changed, 93 insertions(+)

diff --git a/src/engine.c b/src/engine.c
index 21a3ba29fe..056f19b2a9 100644
--- a/src/engine.c
+++ b/src/engine.c
@@ -4886,6 +4886,7 @@ static void *engine_dumper_poll(void *p) {
 #endif
 
       /* Add more interesting diagnostics. */
+      scheduler_dump_queues(e);
 
       /* Delete the file. */
       unlink(".dump");
diff --git a/src/queue.c b/src/queue.c
index 3a9919163a..f6dbdcd0e0 100644
--- a/src/queue.c
+++ b/src/queue.c
@@ -301,3 +301,34 @@ void queue_clean(struct queue *q) {
   free(q->tid);
   free(q->tid_incoming);
 }
+
+
+/**
+ * @brief Dump a formatted list of tasks in the queue to the given file stream.
+ *
+ * @param nodeID the node id of this rank.
+ * @param index a number for this queue, added to the output.
+ * @param file the FILE stream, should opened for write.
+ * @param q The task #queue.
+ */
+void queue_dump(int nodeID, int index, FILE *file, struct queue *q) {
+
+  swift_lock_type *qlock = &q->lock;
+
+  /* Grab the queue lock. */
+  if (lock_lock(qlock) != 0) error("Locking the qlock failed.\n");
+
+  /* Fill any tasks from the incoming DEQ. */
+  queue_get_incoming(q);
+
+  /* Loop over the queue entries. */
+  for (int k = 0; k < q->count; k++) {
+    struct task *t = &q->tasks[q->tid[k]];
+
+    fprintf(file, "%d %d %d %s %s %d\n", nodeID, index, k,
+            taskID_names[t->type], subtaskID_names[t->subtype], t->wait);
+  }
+
+  /* Release the task lock. */
+  if (lock_unlock(qlock) != 0) error("Unlocking the qlock failed.\n");
+}
diff --git a/src/queue.h b/src/queue.h
index c85cf0cabe..e3c7b6f7b2 100644
--- a/src/queue.h
+++ b/src/queue.h
@@ -67,4 +67,6 @@ void queue_init(struct queue *q, struct task *tasks);
 void queue_insert(struct queue *q, struct task *t);
 void queue_clean(struct queue *q);
 
+void queue_dump(int nodeID, int index, FILE *file, struct queue *q);
+
 #endif /* SWIFT_QUEUE_H */
diff --git a/src/scheduler.c b/src/scheduler.c
index 392c868e7e..7bc0d4cb38 100644
--- a/src/scheduler.c
+++ b/src/scheduler.c
@@ -2265,3 +2265,61 @@ void scheduler_write_task_level(const struct scheduler *s) {
   fclose(f);
   free(count);
 }
+/**
+ * @brief dump all the active queues of all the known schedulers into a file.
+ *
+ * @param e the #scheduler
+ */
+void scheduler_dump_queues(struct engine *e) {
+
+  struct scheduler *s = &e->sched;
+
+#ifdef WITH_MPI
+
+  /* Make sure output file is empty, only on one rank. */
+  char dumpfile[35];
+  snprintf(dumpfile, sizeof(dumpfile), "queue_dump_MPI-step%d.dat", e->step);
+  FILE *file_thread;
+  if (engine_rank == 0) {
+    file_thread = fopen(dumpfile, "w");
+    fprintf(file_thread, "# rank queue index type subtype waits\n");
+    fclose(file_thread);
+  }
+  MPI_Barrier(MPI_COMM_WORLD);
+
+  for (int i = 0; i < e->nr_nodes; i++) {
+
+    /* Rank 0 decides the index of the writing node, this happens
+     * one-by-one. */
+    int kk = i;
+    MPI_Bcast(&kk, 1, MPI_INT, 0, MPI_COMM_WORLD);
+
+    if (i == engine_rank) {
+
+      /* Open file and position at end. */
+      file_thread = fopen(dumpfile, "a");
+
+      for (int l = 0; l < s->nr_queues; l++) {
+        queue_dump(engine_rank, l, file_thread, &s->queues[l]);
+      }
+      fclose(file_thread);
+    }
+
+    /* And we wait for all to synchronize. */
+    MPI_Barrier(MPI_COMM_WORLD);
+  }
+
+#else
+
+  /* Non-MPI, so just a single schedulers worth of queues to dump. */
+  char dumpfile[32];
+  snprintf(dumpfile, sizeof(dumpfile), "queue_dump-step%d.dat", e->step);
+  FILE *file_thread;
+  file_thread = fopen(dumpfile, "w");
+  fprintf(file_thread, "# rank queue index type subtype waits\n");
+  for (int l = 0; l < s->nr_queues; l++) {
+    queue_dump(engine_rank, l, file_thread, &s->queues[l]);
+  }
+  fclose(file_thread);
+#endif  // WITH_MPI
+}
diff --git a/src/scheduler.h b/src/scheduler.h
index ac9bc754db..c12d2e9806 100644
--- a/src/scheduler.h
+++ b/src/scheduler.h
@@ -201,5 +201,6 @@ void scheduler_clean(struct scheduler *s);
 void scheduler_free_tasks(struct scheduler *s);
 void scheduler_write_dependencies(struct scheduler *s, int verbose);
 void scheduler_write_task_level(const struct scheduler *s);
+void scheduler_dump_queues(struct engine *e);
 
 #endif /* SWIFT_SCHEDULER_H */
-- 
GitLab