Added more timers to the main sections of the engine to understand where the un-accounted time is.

src/engine.c

@@ -1044,6 +1044,8 @@ void engine_repartition_trigger(struct engine *e) {
 
 #ifdef WITH_MPI
 
+  const ticks tic = getticks();
+
   /* Do nothing if there have not been enough steps since the last
    * repartition, don't want to repeat this too often or immediately after
    * a repartition step. Also nothing to do when requested. */
@@ -1112,6 +1114,10 @@ void engine_repartition_trigger(struct engine *e) {
   /* We always reset CPU time for next check, unless it will not be used. */
   if (e->reparttype->type != REPART_NONE)
     e->cputime_last_step = clocks_get_cputime_used();
+
+  if (e->verbose)
+    message("took %.3f %s", clocks_from_ticks(getticks() - tic),
+            clocks_getunit());
 #endif
 }
 
@@ -1591,6 +1597,8 @@ void engine_exchange_top_multipoles(struct engine *e) {
 
 #ifdef WITH_MPI
 
+  ticks tic = getticks();
+
 #ifdef SWIFT_DEBUG_CHECKS
   for (int i = 0; i < e->s->nr_cells; ++i) {
     const struct gravity_tensors *m = &e->s->multipoles_top[i];
@@ -1659,6 +1667,9 @@ void engine_exchange_top_multipoles(struct engine *e) {
         counter, e->total_nr_gparts);
 #endif
 
+  if (e->verbose)
+    message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+            clocks_getunit());
 #else
   error("SWIFT was not compiled with MPI support.");
 #endif
@@ -1993,6 +2004,8 @@ void engine_rebuild(struct engine *e, int repartitioned,
   /* Re-build the space. */
   space_rebuild(e->s, repartitioned, e->verbose);
 
+  const ticks tic2 = getticks();
+
   /* Update the global counters of particles */
   long long num_particles[3] = {e->s->nr_parts, e->s->nr_gparts,
                                 e->s->nr_sparts};
@@ -2009,6 +2022,10 @@ void engine_rebuild(struct engine *e, int repartitioned,
   e->nr_inhibited_gparts = 0;
   e->nr_inhibited_sparts = 0;
 
+  if (e->verbose)
+    message("updating particle counts took %.3f %s.",
+            clocks_from_ticks(getticks() - tic2), clocks_getunit());
+
   /* Re-compute the mesh forces */
   if ((e->policy & engine_policy_self_gravity) && e->s->periodic)
     pm_mesh_compute_potential(e->mesh, e->s, &e->threadpool, e->verbose);
@@ -3862,6 +3879,8 @@ void engine_makeproxies(struct engine *e) {
 void engine_split(struct engine *e, struct partition *initial_partition) {
 
 #ifdef WITH_MPI
+  const ticks tic = getticks();
+
   struct space *s = e->s;
 
   /* Do the initial partition of the cells. */
@@ -3942,6 +3961,10 @@ void engine_split(struct engine *e, struct partition *initial_partition) {
                     s->nr_sparts, e->verbose);
 #endif
 
+  if (e->verbose)
+    message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+            clocks_getunit());
+
 #else
   error("SWIFT was not compiled with MPI support.");
 #endif
@@ -5114,6 +5137,8 @@ void engine_init_output_lists(struct engine *e, struct swift_params *params) {
  */
 void engine_recompute_displacement_constraint(struct engine *e) {
 
+  const ticks tic = getticks();
+
   /* Get the cosmological information */
   const struct cosmology *cosmo = e->cosmology;
   const float Om = cosmo->Omega_m;
@@ -5224,6 +5249,10 @@ void engine_recompute_displacement_constraint(struct engine *e) {
 
   if (e->verbose)
     message("max_dt_RMS_displacement = %e", e->dt_max_RMS_displacement);
+
+  if (e->verbose)
+    message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
+            clocks_getunit());
 }
 
 /**