Add an optional task dumper thread

Creates an optional thread whose job is simply to poll for the existence of the file .dump in the current directory. When this file is found dumps of all the unskipped tasks and uncompleted entries in the queues is made, as is a dump of memory use, if --enable-memuse-reports was configured. Once the dump is complete the .dump file is removed and the application will just continue so must be killed normally, or can be dumped at a later time (this feature is aimed at checking deadlocks of the engine tasks, so not clear if that would be useful).

Uses configure flag --enable-dumper to enable, so not always available, so you need to catch the problem at least twice, but reliably works, unlike the signals version !857 (closed).

src/engine.c

@@ -37,6 +37,7 @@
 #include <sys/types.h>
 #include <unistd.h>
 
+
 /* MPI headers. */
 #ifdef WITH_MPI
 
@@ -3383,6 +3384,67 @@ void engine_unpin(void) {
 #endif
 }
 
+#ifdef SWIFT_DUMPER_THREAD
+/**
+ * @brief dumper thread action, checks got the existence of the .dump file
+ * every 5 seconds and does the dump if found.
+ *
+ * @param p the #engine
+ */
+static void *engine_dumper_poll(void *p) {
+  struct engine *e = (struct engine *)p;
+  while (1) {
+    if (access(".dump", F_OK) == 0) {
+
+      /* OK, do our work. */
+      message("Dumping engine tasks in step: %d", e->step);
+      task_dump_active(e);
+
+#ifdef SWIFT_MEMUSE_REPORTS
+      /* Dump the currently logged memory. */
+      message("Dumping memory use report");
+      memuse_log_dump_error(e->nodeID);
+#endif
+
+      /* Add more interesting diagnostics. */
+      scheduler_dump_queues(e);
+
+      /* Delete the file. */
+      unlink(".dump");
+      message("Dumping completed");
+      fflush(stdout);
+    }
+
+    /* Take a breath. */
+    sleep(5);
+  }
+  return NULL;
+}
+#endif /* SWIFT_DUMPER_THREAD */
+
+#ifdef SWIFT_DUMPER_THREAD
+/**
+ * @brief creates the dumper thread.
+ *
+ * This watches for the creation of a ".dump" file in the current directory
+ * and if found dumps the current state of the tasks and memory use (if also
+ * configured).
+ *
+ * @param e the #engine
+ *
+ */
+static void engine_dumper_init(struct engine *e) {
+  pthread_t dumper;
+
+  /* Make sure the .dump file is not present, that is bad when starting up. */
+  struct stat buf;
+  if (stat(".dump", &buf) == 0) unlink(".dump");
+
+  /* Thread does not exit, so nothing to do but create it. */
+  pthread_create(&dumper, NULL, &engine_dumper_poll, e);
+}
+#endif /* SWIFT_DUMPER_THREAD */
+
 /**
  * @brief init an engine struct with the necessary properties for the
  *        simulation.
@@ -4283,6 +4345,12 @@ void engine_config(int restart, int fof, struct engine *e,
   }
 #endif
 
+#ifdef SWIFT_DUMPER_THREAD
+
+  /* Start the dumper thread.*/
+  engine_dumper_init(e);
+#endif
+
   /* Wait for the runner threads to be in place. */
   swift_barrier_wait(&e->wait_barrier);
 }