Add a separate timer for subset sub-cell tasks. Write the timers to a file....

Add a separate timer for subset sub-cell tasks. Write the timers to a file. Make sure we can compile without timers. Add a configure option to switch the timers on.

configure.ac

@@ -189,6 +189,18 @@ if test "$enable_task_debugging" = "yes"; then
    AC_DEFINE([SWIFT_DEBUG_TASKS],1,[Enable task debugging])
 fi
 
+# Check if the general timers are switched on.
+AC_ARG_ENABLE([timers],
+   [AS_HELP_STRING([--enable-timers],
+     [Activate the basic timers @<:@yes/no@:>@]
+   )],
+   [enable_timers="$enableval"],
+   [enable_timers="no"]
+)
+if test "$enable_timers" = "yes"; then
+   AC_DEFINE([SWIFT_USE_TIMERS],1,[Enable individual timers])
+fi
+
 # Check if expensive debugging is on.
 AC_ARG_ENABLE([debugging-checks],
    [AS_HELP_STRING([--enable-debugging-checks],
@@ -874,6 +886,7 @@ AC_MSG_RESULT([
    Multipole order     : $with_multipole_order
    No gravity below ID : $no_gravity_below_id
 
+   Individual timers : $enable_timers
    Task debugging    : $enable_task_debugging
    Debugging checks  : $enable_debugging_checks
    Gravity checks    : $gravity_force_checks

examples/Makefile.am

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 # Common flags
-MYFLAGS = -DTIMER
+MYFLAGS = 
 
 # Add the source directory and debug to CFLAGS
 AM_CFLAGS = -I$(top_srcdir)/src $(HDF5_CPPFLAGS)

examples/main.c

@@ -610,35 +610,25 @@ int main(int argc, char *argv[]) {
   engine_dump_snapshot(&e);
 
   /* Legend */
-  if (myrank == 0) {
+  if (myrank == 0)
     printf("# %6s %14s %14s %10s %10s %10s %16s [%s]\n", "Step", "Time",
            "Time-step", "Updates", "g-Updates", "s-Updates", "Wall-clock time",
            clocks_getunit());
 
-    if (with_verbose_timers) {
-      printf("timers: ");
-      for (int k = 0; k < timer_count; k++) printf("%s\t", timers_names[k]);
-      printf("\n");
-    }
-  }
+  /* File for the timers */
+  if (with_verbose_timers) timers_open_file();
 
   /* Main simulation loop */
   for (int j = 0; !engine_is_done(&e) && e.step - 1 != nsteps; j++) {
 
     /* Reset timers */
-    timers_reset(timers_mask_all);
+    timers_reset_all();
 
     /* Take a step. */
     engine_step(&e);
 
     /* Print the timers. */
-    if (with_verbose_timers) {
-      printf("timers: ");
-      for (int k = 0; k < timer_count; k++)
-        printf("%.3f\t", clocks_from_ticks(timers[k]));
-      printf("\n");
-      timers_reset(timers_mask_all);
-    }
+    if (with_verbose_timers) timers_print(e.step);
 
 #ifdef SWIFT_DEBUG_TASKS
     /* Dump the task data using the given frequency. */
@@ -744,6 +734,7 @@ int main(int argc, char *argv[]) {
 #endif
 
   /* Clean everything */
+  if (with_verbose_timers) timers_close_file();
   engine_clean(&e);
   free(params);
 

src/Makefile.am

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 # Add the debug flag to the whole thing
-AM_CFLAGS = -DTIMER $(HDF5_CPPFLAGS)
+AM_CFLAGS = $(HDF5_CPPFLAGS)
 
 # Assign a "safe" version number
 AM_LDFLAGS = $(HDF5_LDFLAGS) $(FFTW_LIBS) -version-info 0:0:0

src/atomic.h

@@ -19,6 +19,9 @@
 #ifndef SWIFT_ATOMIC_H
 #define SWIFT_ATOMIC_H
 
+/* Config parameters. */
+#include "../config.h"
+
 /* Includes. */
 #include "inline.h"
 

src/engine.c

@@ -2742,7 +2742,8 @@ void engine_rebuild(struct engine *e) {
  */
 void engine_prepare(struct engine *e) {
 
-  TIMER_TIC;
+  TIMER_TIC2;
+  const ticks tic = getticks();
 
 #ifdef SWIFT_DEBUG_CHECKS
   if (e->forcerepart || e->forcerebuild) {
@@ -2769,7 +2770,7 @@ void engine_prepare(struct engine *e) {
   }
   e->tasks_age += 1;
 
-  TIMER_TOC(timer_prepare);
+  TIMER_TOC2(timer_prepare);
 
   if (e->verbose)
     message("took %.3f %s (including unskip and reweight).",

src/inline.h

@@ -20,6 +20,9 @@
 #ifndef SWIFT_INLINE_H
 #define SWIFT_INLINE_H
 
+/* Config parameters. */
+#include "../config.h"
+
 /**
  * @brief Defines inline
  */

src/runner_doiact.h

@@ -255,7 +255,7 @@ void DOPAIR2_NAIVE(struct runner *r, struct cell *restrict ci,
   const struct engine *e = r->e;
 
 #ifndef SWIFT_DEBUG_CHECKS
-// error("Don't use in actual runs ! Slow code !");
+  error("Don't use in actual runs ! Slow code !");
 #endif
 
 #ifdef WITH_OLD_VECTORIZATION
@@ -562,7 +562,7 @@ void DOPAIR_SUBSET_NAIVE(struct runner *r, struct cell *restrict ci,
       IACT_NONSYM(r2q[k], &dxq[3 * k], hiq[k], hjq[k], piq[k], pjq[k]);
 #endif
 
-  TIMER_TOC(timer_dopair_subset);
+  TIMER_TOC(timer_dopair_subset_naive);
 }
 
 /**
@@ -857,7 +857,7 @@ void DOSELF_SUBSET(struct runner *r, struct cell *restrict ci,
       IACT_NONSYM(r2q[k], &dxq[3 * k], hiq[k], hjq[k], piq[k], pjq[k]);
 #endif
 
-  TIMER_TOC(timer_dopair_subset);
+  TIMER_TOC(timer_doself_subset);
 }
 
 /**
@@ -3179,5 +3179,5 @@ void DOSUB_SUBSET(struct runner *r, struct cell *ci, struct part *parts,
 
   } /* otherwise, pair interaction. */
 
-  if (gettimer) TIMER_TOC(TIMER_DOSUB_PAIR);
+  if (gettimer) TIMER_TOC(timer_dosub_subset);
 }

src/timers.c

@@ -21,16 +21,21 @@
  ******************************************************************************/
 
 /* Config parameters. */
-#include "../config.h"
 
 /* This object's header. */
 #include "timers.h"
 
+/* Some standard headers. */
+#include <stdio.h>
+
+/* Local includes. */
+#include "clocks.h"
+
 /* The timers. */
 ticks timers[timer_count];
 
 /* Timer names. */
-char *timers_names[timer_count] = {
+const char* timers_names[timer_count] = {
     "none",
     "prepare",
     "init",
@@ -63,20 +68,27 @@ char *timers_names[timer_count] = {
     "dosub_pair_gradient",
     "dosub_pair_force",
     "dosub_pair_grav",
+    "doself_subset",
     "dopair_subset",
+    "dopair_subset_naive",
+    "dosub_subset",
     "do_ghost",
     "do_extra_ghost",
     "dorecv_part",
     "dorecv_gpart",
     "dorecv_spart",
+    "do_cooling",
     "gettask",
     "qget",
     "qsteal",
+    "locktree",
     "runners",
     "step",
-    "do_cooling",
 };
 
+/* File to store the timers */
+static FILE* timers_file;
+
 /**
  * @brief Re-set the timers.
  *
@@ -90,3 +102,39 @@ void timers_reset(unsigned long long mask) {
   for (int k = 0; k < timer_count; k++)
     if (mask & (1ull << k)) timers[k] = 0;
 }
+
+/**
+ * @brief Re-set all the timers.
+ *
+ */
+void timers_reset_all() { timers_reset(timers_mask_all); }
+
+/**
+ * @brief Outputs all the timers to the timers dump file.
+ *
+ * @param step The current step.
+ */
+void timers_print(int step) {
+  fprintf(timers_file, "%d\t", step);
+  for (int k = 0; k < timer_count; k++)
+    fprintf(timers_file, "%.3f\t", clocks_from_ticks(timers[k]));
+  fprintf(timers_file, "\n");
+}
+
+/**
+ * @brief Opens the file to contain the timers info and print a header
+ */
+void timers_open_file() {
+
+  timers_file = fopen("timers.txt", "w");
+
+  fprintf(timers_file, "# timers: \n# step | ");
+  for (int k = 0; k < timer_count; k++)
+    fprintf(timers_file, "%s\t", timers_names[k]);
+  fprintf(timers_file, "\n");
+}
+
+/**
+ * @brief Close the file containing the timer info.
+ */
+void timers_close_file() { fclose(timers_file); }

src/timers.h

@@ -22,7 +22,10 @@
 #ifndef SWIFT_TIMERS_H
 #define SWIFT_TIMERS_H
 
-/* Includes. */
+/* Config parameters. */
+#include "../config.h"
+
+/* Local includes. */
 #include "atomic.h"
 #include "cycle.h"
 #include "inline.h"
@@ -66,18 +69,22 @@ enum {
   timer_dosub_pair_gradient,
   timer_dosub_pair_force,
   timer_dosub_pair_grav,
+  timer_doself_subset,
   timer_dopair_subset,
+  timer_dopair_subset_naive,
+  timer_dosub_subset,
   timer_do_ghost,
   timer_do_extra_ghost,
   timer_dorecv_part,
   timer_dorecv_gpart,
   timer_dorecv_spart,
+  timer_do_cooling,
   timer_gettask,
   timer_qget,
   timer_qsteal,
+  timer_locktree,
   timer_runners,
   timer_step,
-  timer_do_cooling,
   timer_count,
 };
 
@@ -85,32 +92,34 @@ enum {
 extern ticks timers[timer_count];
 
 /* The timer names. */
-extern char *timers_names[];
+extern const char *timers_names[];
 
 /* Mask for all timers. */
 #define timers_mask_all ((1ull << timer_count) - 1)
 
 /* Define the timer macros. */
-#ifdef TIMER
-#define TIMER_TIC_ND tic = getticks();
-#define TIMER_TIC2_ND ticks tic2 = getticks();
-#define TIMER_TIC ticks tic = getticks();
+#ifdef SWIFT_USE_TIMERS
+#define TIMER_TIC const ticks tic = getticks();
 #define TIMER_TOC(t) timers_toc(t, tic)
-#define TIMER_TIC2 ticks tic2 = getticks();
+#define TIMER_TIC2 const ticks tic2 = getticks();
 #define TIMER_TOC2(t) timers_toc(t, tic2)
 INLINE static ticks timers_toc(unsigned int t, ticks tic) {
-  ticks d = (getticks() - tic);
+  const ticks d = (getticks() - tic);
   atomic_add(&timers[t], d);
   return d;
 }
 #else
 #define TIMER_TIC
-#define TIMER_TOC(t)
+#define TIMER_TOC(t) (void)0
 #define TIMER_TIC2
-#define TIMER_TOC2(t)
+#define TIMER_TOC2(t) (void)0
 #endif
 
 /* Function prototypes. */
+void timers_reset_all();
 void timers_reset(unsigned long long mask);
+void timers_open_file();
+void timers_close_file();
+void timers_print(int step);
 
 #endif /* SWIFT_TIMERS_H */

tests/Makefile.am

@@ -15,8 +15,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 # Add the source directory and debug to CFLAGS
-AM_CFLAGS = -I$(top_srcdir)/src $(HDF5_CPPFLAGS) -DTIMER
-
+AM_CFLAGS = -I$(top_srcdir)/src $(HDF5_CPPFLAGS)
 
 AM_LDFLAGS = ../src/.libs/libswiftsim.a $(HDF5_LDFLAGS) $(HDF5_LIBS) $(FFTW_LIBS)