From cd06f8450e8592eec5ad16de004fa6f783d04805 Mon Sep 17 00:00:00 2001
From: Matthieu Schaller <matthieu.schaller@durham.ac.uk>
Date: Thu, 27 Apr 2017 18:42:59 +0100
Subject: [PATCH] Add a separate timer for subset sub-cell tasks. Write the
 timers to a file. Make sure we can compile without timers. Add a configure
 option to switch the timers on.

---
 configure.ac         | 13 +++++++++++
 examples/Makefile.am |  2 +-
 examples/main.c      | 21 +++++------------
 src/Makefile.am      |  2 +-
 src/atomic.h         |  3 +++
 src/engine.c         |  5 ++--
 src/inline.h         |  3 +++
 src/runner_doiact.h  |  8 +++----
 src/timers.c         | 54 +++++++++++++++++++++++++++++++++++++++++---
 src/timers.h         | 31 ++++++++++++++++---------
 tests/Makefile.am    |  3 +--
 11 files changed, 106 insertions(+), 39 deletions(-)

diff --git a/configure.ac b/configure.ac
index e979580470..8a2d0f30ae 100644
--- a/configure.ac
+++ b/configure.ac
@@ -189,6 +189,18 @@ if test "$enable_task_debugging" = "yes"; then
    AC_DEFINE([SWIFT_DEBUG_TASKS],1,[Enable task debugging])
 fi
 
+# Check if the general timers are switched on.
+AC_ARG_ENABLE([timers],
+   [AS_HELP_STRING([--enable-timers],
+     [Activate the basic timers @<:@yes/no@:>@]
+   )],
+   [enable_timers="$enableval"],
+   [enable_timers="no"]
+)
+if test "$enable_timers" = "yes"; then
+   AC_DEFINE([SWIFT_USE_TIMERS],1,[Enable individual timers])
+fi
+
 # Check if expensive debugging is on.
 AC_ARG_ENABLE([debugging-checks],
    [AS_HELP_STRING([--enable-debugging-checks],
@@ -874,6 +886,7 @@ AC_MSG_RESULT([
    Multipole order     : $with_multipole_order
    No gravity below ID : $no_gravity_below_id
 
+   Individual timers : $enable_timers
    Task debugging    : $enable_task_debugging
    Debugging checks  : $enable_debugging_checks
    Gravity checks    : $gravity_force_checks
diff --git a/examples/Makefile.am b/examples/Makefile.am
index dd13fb7eb4..1dd240fb60 100644
--- a/examples/Makefile.am
+++ b/examples/Makefile.am
@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 # Common flags
-MYFLAGS = -DTIMER
+MYFLAGS = 
 
 # Add the source directory and debug to CFLAGS
 AM_CFLAGS = -I$(top_srcdir)/src $(HDF5_CPPFLAGS)
diff --git a/examples/main.c b/examples/main.c
index 66d9b2d40a..afe8265c62 100644
--- a/examples/main.c
+++ b/examples/main.c
@@ -610,35 +610,25 @@ int main(int argc, char *argv[]) {
   engine_dump_snapshot(&e);
 
   /* Legend */
-  if (myrank == 0) {
+  if (myrank == 0)
     printf("# %6s %14s %14s %10s %10s %10s %16s [%s]\n", "Step", "Time",
            "Time-step", "Updates", "g-Updates", "s-Updates", "Wall-clock time",
            clocks_getunit());
 
-    if (with_verbose_timers) {
-      printf("timers: ");
-      for (int k = 0; k < timer_count; k++) printf("%s\t", timers_names[k]);
-      printf("\n");
-    }
-  }
+  /* File for the timers */
+  if (with_verbose_timers) timers_open_file();
 
   /* Main simulation loop */
   for (int j = 0; !engine_is_done(&e) && e.step - 1 != nsteps; j++) {
 
     /* Reset timers */
-    timers_reset(timers_mask_all);
+    timers_reset_all();
 
     /* Take a step. */
     engine_step(&e);
 
     /* Print the timers. */
-    if (with_verbose_timers) {
-      printf("timers: ");
-      for (int k = 0; k < timer_count; k++)
-        printf("%.3f\t", clocks_from_ticks(timers[k]));
-      printf("\n");
-      timers_reset(timers_mask_all);
-    }
+    if (with_verbose_timers) timers_print(e.step);
 
 #ifdef SWIFT_DEBUG_TASKS
     /* Dump the task data using the given frequency. */
@@ -744,6 +734,7 @@ int main(int argc, char *argv[]) {
 #endif
 
   /* Clean everything */
+  if (with_verbose_timers) timers_close_file();
   engine_clean(&e);
   free(params);
 
diff --git a/src/Makefile.am b/src/Makefile.am
index 644094eefb..7bec5327f4 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 # Add the debug flag to the whole thing
-AM_CFLAGS = -DTIMER $(HDF5_CPPFLAGS)
+AM_CFLAGS = $(HDF5_CPPFLAGS)
 
 # Assign a "safe" version number
 AM_LDFLAGS = $(HDF5_LDFLAGS) $(FFTW_LIBS) -version-info 0:0:0
diff --git a/src/atomic.h b/src/atomic.h
index be24f96e5a..4f180c643a 100644
--- a/src/atomic.h
+++ b/src/atomic.h
@@ -19,6 +19,9 @@
 #ifndef SWIFT_ATOMIC_H
 #define SWIFT_ATOMIC_H
 
+/* Config parameters. */
+#include "../config.h"
+
 /* Includes. */
 #include "inline.h"
 
diff --git a/src/engine.c b/src/engine.c
index e6d5484781..07fd121a14 100644
--- a/src/engine.c
+++ b/src/engine.c
@@ -2742,7 +2742,8 @@ void engine_rebuild(struct engine *e) {
  */
 void engine_prepare(struct engine *e) {
 
-  TIMER_TIC;
+  TIMER_TIC2;
+  const ticks tic = getticks();
 
 #ifdef SWIFT_DEBUG_CHECKS
   if (e->forcerepart || e->forcerebuild) {
@@ -2769,7 +2770,7 @@ void engine_prepare(struct engine *e) {
   }
   e->tasks_age += 1;
 
-  TIMER_TOC(timer_prepare);
+  TIMER_TOC2(timer_prepare);
 
   if (e->verbose)
     message("took %.3f %s (including unskip and reweight).",
diff --git a/src/inline.h b/src/inline.h
index c4dd9d59be..538f6f520c 100644
--- a/src/inline.h
+++ b/src/inline.h
@@ -20,6 +20,9 @@
 #ifndef SWIFT_INLINE_H
 #define SWIFT_INLINE_H
 
+/* Config parameters. */
+#include "../config.h"
+
 /**
  * @brief Defines inline
  */
diff --git a/src/runner_doiact.h b/src/runner_doiact.h
index 21b7ba5535..46cdbcf090 100644
--- a/src/runner_doiact.h
+++ b/src/runner_doiact.h
@@ -255,7 +255,7 @@ void DOPAIR2_NAIVE(struct runner *r, struct cell *restrict ci,
   const struct engine *e = r->e;
 
 #ifndef SWIFT_DEBUG_CHECKS
-// error("Don't use in actual runs ! Slow code !");
+  error("Don't use in actual runs ! Slow code !");
 #endif
 
 #ifdef WITH_OLD_VECTORIZATION
@@ -562,7 +562,7 @@ void DOPAIR_SUBSET_NAIVE(struct runner *r, struct cell *restrict ci,
       IACT_NONSYM(r2q[k], &dxq[3 * k], hiq[k], hjq[k], piq[k], pjq[k]);
 #endif
 
-  TIMER_TOC(timer_dopair_subset);
+  TIMER_TOC(timer_dopair_subset_naive);
 }
 
 /**
@@ -857,7 +857,7 @@ void DOSELF_SUBSET(struct runner *r, struct cell *restrict ci,
       IACT_NONSYM(r2q[k], &dxq[3 * k], hiq[k], hjq[k], piq[k], pjq[k]);
 #endif
 
-  TIMER_TOC(timer_dopair_subset);
+  TIMER_TOC(timer_doself_subset);
 }
 
 /**
@@ -3179,5 +3179,5 @@ void DOSUB_SUBSET(struct runner *r, struct cell *ci, struct part *parts,
 
   } /* otherwise, pair interaction. */
 
-  if (gettimer) TIMER_TOC(TIMER_DOSUB_PAIR);
+  if (gettimer) TIMER_TOC(timer_dosub_subset);
 }
diff --git a/src/timers.c b/src/timers.c
index 558ac109c2..153f782da4 100644
--- a/src/timers.c
+++ b/src/timers.c
@@ -21,16 +21,21 @@
  ******************************************************************************/
 
 /* Config parameters. */
-#include "../config.h"
 
 /* This object's header. */
 #include "timers.h"
 
+/* Some standard headers. */
+#include <stdio.h>
+
+/* Local includes. */
+#include "clocks.h"
+
 /* The timers. */
 ticks timers[timer_count];
 
 /* Timer names. */
-char *timers_names[timer_count] = {
+const char* timers_names[timer_count] = {
     "none",
     "prepare",
     "init",
@@ -63,20 +68,27 @@ char *timers_names[timer_count] = {
     "dosub_pair_gradient",
     "dosub_pair_force",
     "dosub_pair_grav",
+    "doself_subset",
     "dopair_subset",
+    "dopair_subset_naive",
+    "dosub_subset",
     "do_ghost",
     "do_extra_ghost",
     "dorecv_part",
     "dorecv_gpart",
     "dorecv_spart",
+    "do_cooling",
     "gettask",
     "qget",
     "qsteal",
+    "locktree",
     "runners",
     "step",
-    "do_cooling",
 };
 
+/* File to store the timers */
+static FILE* timers_file;
+
 /**
  * @brief Re-set the timers.
  *
@@ -90,3 +102,39 @@ void timers_reset(unsigned long long mask) {
   for (int k = 0; k < timer_count; k++)
     if (mask & (1ull << k)) timers[k] = 0;
 }
+
+/**
+ * @brief Re-set all the timers.
+ *
+ */
+void timers_reset_all() { timers_reset(timers_mask_all); }
+
+/**
+ * @brief Outputs all the timers to the timers dump file.
+ *
+ * @param step The current step.
+ */
+void timers_print(int step) {
+  fprintf(timers_file, "%d\t", step);
+  for (int k = 0; k < timer_count; k++)
+    fprintf(timers_file, "%.3f\t", clocks_from_ticks(timers[k]));
+  fprintf(timers_file, "\n");
+}
+
+/**
+ * @brief Opens the file to contain the timers info and print a header
+ */
+void timers_open_file() {
+
+  timers_file = fopen("timers.txt", "w");
+
+  fprintf(timers_file, "# timers: \n# step | ");
+  for (int k = 0; k < timer_count; k++)
+    fprintf(timers_file, "%s\t", timers_names[k]);
+  fprintf(timers_file, "\n");
+}
+
+/**
+ * @brief Close the file containing the timer info.
+ */
+void timers_close_file() { fclose(timers_file); }
diff --git a/src/timers.h b/src/timers.h
index ef84a82eab..23ae61b83f 100644
--- a/src/timers.h
+++ b/src/timers.h
@@ -22,7 +22,10 @@
 #ifndef SWIFT_TIMERS_H
 #define SWIFT_TIMERS_H
 
-/* Includes. */
+/* Config parameters. */
+#include "../config.h"
+
+/* Local includes. */
 #include "atomic.h"
 #include "cycle.h"
 #include "inline.h"
@@ -66,18 +69,22 @@ enum {
   timer_dosub_pair_gradient,
   timer_dosub_pair_force,
   timer_dosub_pair_grav,
+  timer_doself_subset,
   timer_dopair_subset,
+  timer_dopair_subset_naive,
+  timer_dosub_subset,
   timer_do_ghost,
   timer_do_extra_ghost,
   timer_dorecv_part,
   timer_dorecv_gpart,
   timer_dorecv_spart,
+  timer_do_cooling,
   timer_gettask,
   timer_qget,
   timer_qsteal,
+  timer_locktree,
   timer_runners,
   timer_step,
-  timer_do_cooling,
   timer_count,
 };
 
@@ -85,32 +92,34 @@ enum {
 extern ticks timers[timer_count];
 
 /* The timer names. */
-extern char *timers_names[];
+extern const char *timers_names[];
 
 /* Mask for all timers. */
 #define timers_mask_all ((1ull << timer_count) - 1)
 
 /* Define the timer macros. */
-#ifdef TIMER
-#define TIMER_TIC_ND tic = getticks();
-#define TIMER_TIC2_ND ticks tic2 = getticks();
-#define TIMER_TIC ticks tic = getticks();
+#ifdef SWIFT_USE_TIMERS
+#define TIMER_TIC const ticks tic = getticks();
 #define TIMER_TOC(t) timers_toc(t, tic)
-#define TIMER_TIC2 ticks tic2 = getticks();
+#define TIMER_TIC2 const ticks tic2 = getticks();
 #define TIMER_TOC2(t) timers_toc(t, tic2)
 INLINE static ticks timers_toc(unsigned int t, ticks tic) {
-  ticks d = (getticks() - tic);
+  const ticks d = (getticks() - tic);
   atomic_add(&timers[t], d);
   return d;
 }
 #else
 #define TIMER_TIC
-#define TIMER_TOC(t)
+#define TIMER_TOC(t) (void)0
 #define TIMER_TIC2
-#define TIMER_TOC2(t)
+#define TIMER_TOC2(t) (void)0
 #endif
 
 /* Function prototypes. */
+void timers_reset_all();
 void timers_reset(unsigned long long mask);
+void timers_open_file();
+void timers_close_file();
+void timers_print(int step);
 
 #endif /* SWIFT_TIMERS_H */
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 10343c05db..a51b8eb82a 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -15,8 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 # Add the source directory and debug to CFLAGS
-AM_CFLAGS = -I$(top_srcdir)/src $(HDF5_CPPFLAGS) -DTIMER
-
+AM_CFLAGS = -I$(top_srcdir)/src $(HDF5_CPPFLAGS)
 
 AM_LDFLAGS = ../src/.libs/libswiftsim.a $(HDF5_LDFLAGS) $(HDF5_LIBS) $(FFTW_LIBS)
 
-- 
GitLab