Only dump the fixed costs file on a sensible repartition

examples/main.c

@@ -283,11 +283,11 @@ int main(int argc, char *argv[]) {
 
 #ifndef SWIFT_DEBUG_TASKS
   if (dump_tasks) {
-      if (myrank == 0) {
-          message("WARNING: complete task dumps are only created when "
-                  "configured with --enable-task-debugging.");
-          message("         Basic task statistics will be output.");
-      }
+    if (myrank == 0) {
+      message("WARNING: complete task dumps are only created when "
+              "configured with --enable-task-debugging.");
+      message("         Basic task statistics will be output.");
+    }
   }
 #endif
 
@@ -1037,7 +1037,9 @@ int main(int argc, char *argv[]) {
 #endif
 
       /* Generate the task statistics. */
-      task_dump_stats(&e, j + 1);
+      char dumpfile[40];
+      snprintf(dumpfile, 40, "thread_stats-step%d.dat", j + 1);
+      task_dump_stats(dumpfile, &e, /* header = */ 0, /* allranks = */ 1);
     }
 
 #ifdef SWIFT_DEBUG_THREADPOOL

src/Makefile.am

@@ -41,8 +41,8 @@ endif
 # List required headers
 include_HEADERS = space.h runner.h queue.h task.h lock.h cell.h part.h const.h \
     engine.h swift.h serial_io.h timers.h debug.h scheduler.h proxy.h parallel_io.h \
-    common_io.h single_io.h multipole.h map.h tools.h partition.h clocks.h parser.h \
-    physical_constants.h physical_constants_cgs.h potential.h version.h \
+    common_io.h single_io.h multipole.h map.h tools.h partition.h partition_fixed_costs.h \
+    clocks.h parser.h physical_constants.h physical_constants_cgs.h potential.h version.h \
     hydro_properties.h riemann.h threadpool.h cooling_io.h cooling.h cooling_struct.h \
     sourceterms.h sourceterms_struct.h statistics.h memswap.h cache.h runner_doiact_vec.h profiler.h \
     dump.h logger.h active.h timeline.h xmf.h gravity_properties.h gravity_derivatives.h \

src/engine.c

@@ -991,6 +991,11 @@ void engine_repartition(struct engine *e) {
    * bug that doesn't handle this case well. */
   if (e->nr_nodes == 1) return;
 
+  /* Generate the fixed costs include file. */
+  if (e->step > 3 && e->reparttype->trigger <= 1.f) {
+    task_dump_stats("partition_fixed_costs.h", e, /* header = */ 1, /* allranks = */ 1);
+  }
+
   /* Do the repartitioning. */
   partition_repartition(e->reparttype, e->nodeID, e->nr_nodes, e->s,
                         e->sched.tasks, e->sched.nr_tasks);

src/task.c

@@ -687,26 +687,26 @@ void task_dump_all(struct engine *e, int step) {
 
 /**
  * @brief Generate simple statistics about the times used by the tasks of
- *        all the engines and write these into two files, a human readable
- *        version and one intented for inclusion as the fixed costs for
- *        repartitioning.
+ *        all the engines and write these into two format, a human readable
+ *        version for debugging and one intented for inclusion as the fixed
+ *        costs for repartitioning.
  *
- * Dumps the human readable information to a file "thread_stats-stepn.dat"
- * where n is the given step value. When running under MPI all the tasks are
- * summed into this single file.
+ * Note that when running under MPI all the tasks can be summed into this single
+ * file. In the fuller, human readable file, the statistics included are the
+ * number of task of each type/subtype followed by the minimum, maximum, mean
+ * and total time, in millisec and then the fixed costs value.
  *
- * The fixed costs file will be called "thread_stats-stepn.h".
+ * If header is set, only the fixed costs value is written into the output
+ * file in a format that is suitable for inclusion in SWIFT (as
+ * partition_fixed_costs.h).
  *
+ * @param dumpfile name of the file for the output.
  * @param e the #engine
- * @param step the current step.
+ * @param header whether to write a header include file.
+ * @param allranks do the statistics over all ranks, if not just the current
+ *                 one, only used if header is false.
  */
-void task_dump_stats(struct engine *e, int step) {
-
-  char dumpfile[40];
-  snprintf(dumpfile, 40, "thread_stats-step%d.dat", step);
-
-  char costsfile[40];
-  snprintf(costsfile, 40, "thread_stats-step%d.h", step);
+void task_dump_stats(const char *dumpfile, struct engine *e, int header, int allranks) {
 
   /* Need arrays for sum, min and max across all types and subtypes. */
   double sum[task_type_count][task_subtype_count];
@@ -746,39 +746,43 @@ void task_dump_stats(struct engine *e, int step) {
     }
   }
 
-#ifdef WITH_MPI
-  /* Get these from all ranks for output from rank 0. Could wrap these into a
-   * single operation. */
-  size_t size = task_type_count * task_subtype_count;
-  int res = MPI_Reduce((engine_rank == 0 ? MPI_IN_PLACE : sum), sum, size,
-                       MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-  if (res != MPI_SUCCESS) mpi_error(res, "Failed to reduce task sums");
-
-  res = MPI_Reduce((engine_rank == 0 ? MPI_IN_PLACE : count), count, size,
-                   MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
-  if (res != MPI_SUCCESS) mpi_error(res, "Failed to reduce task counts");
-
-  res = MPI_Reduce((engine_rank == 0 ? MPI_IN_PLACE : min), min, size,
-                   MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
-  if (res != MPI_SUCCESS) mpi_error(res, "Failed to reduce task minima");
 
-  res = MPI_Reduce((engine_rank == 0 ? MPI_IN_PLACE : max), max, size,
-                   MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
-  if (res != MPI_SUCCESS) mpi_error(res, "Failed to reduce task maxima");
-
-  res = MPI_Reduce((engine_rank == 0 ? MPI_IN_PLACE : total), total, 1,
-                   MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
-  if (res != MPI_SUCCESS) mpi_error(res, "Failed to reduce task total time");
+#ifdef WITH_MPI
+  if (allranks || header) {
+    /* Get these from all ranks for output from rank 0. Could wrap these into a
+     * single operation. */
+    size_t size = task_type_count * task_subtype_count;
+    int res = MPI_Reduce((engine_rank == 0 ? MPI_IN_PLACE : sum), sum, size,
+                         MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+    if (res != MPI_SUCCESS) mpi_error(res, "Failed to reduce task sums");
+
+    res = MPI_Reduce((engine_rank == 0 ? MPI_IN_PLACE : count), count, size,
+                     MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
+    if (res != MPI_SUCCESS) mpi_error(res, "Failed to reduce task counts");
+
+    res = MPI_Reduce((engine_rank == 0 ? MPI_IN_PLACE : min), min, size,
+                     MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD);
+    if (res != MPI_SUCCESS) mpi_error(res, "Failed to reduce task minima");
+
+    res = MPI_Reduce((engine_rank == 0 ? MPI_IN_PLACE : max), max, size,
+                     MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
+    if (res != MPI_SUCCESS) mpi_error(res, "Failed to reduce task maxima");
+
+    res = MPI_Reduce((engine_rank == 0 ? MPI_IN_PLACE : total), total, 1,
+                     MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
+    if (res != MPI_SUCCESS) mpi_error(res, "Failed to reduce task total time");
+  }
 
-  if (engine_rank == 0) {
+  if (!allranks || (engine_rank == 0 && (allranks || header))) {
 #endif
 
     FILE *dfile = fopen(dumpfile, "w");
-    fprintf(dfile, "# task ntasks min max sum mean percent fixed_cost\n");
-
-    FILE *cfile = fopen(costsfile, "w");
-    fprintf(cfile, "/* use as src/partition_fixed_costs.h */\n");
-    fprintf(cfile, "#define HAVE_FIXED_COSTS 1\n");
+    if (header) {
+      fprintf(dfile, "/* use as src/partition_fixed_costs.h */\n");
+      fprintf(dfile, "#define HAVE_FIXED_COSTS 1\n");
+    } else {
+      fprintf(dfile, "# task ntasks min max sum mean percent fixed_cost\n");
+    }
 
     for (int j = 0; j < task_type_count; j++) {
       const char *taskID = taskID_names[j];
@@ -790,19 +794,21 @@ void task_dump_stats(struct engine *e, int step) {
           /* Fixed cost is in .1ns as we want to compare between runs in
            * some absolute units. */
           int fixed_cost = (int)(clocks_from_ticks(mean) * 10000.f);
-          fprintf(dfile,
-                  "%15s/%-10s %10d %14.4f %14.4f %14.4f %14.4f %14.4f %10d\n",
-                  taskID, subtaskID_names[k], count[j][k],
-                  clocks_from_ticks(min[j][k]), clocks_from_ticks(max[j][k]),
-                  clocks_from_ticks(sum[j][k]), clocks_from_ticks(mean),
-                  perc, fixed_cost);
-          fprintf(cfile, "repartition_costs[%d][%d] = %10d; /* %s/%s */\n", j,
-                  k, fixed_cost, taskID, subtaskID_names[k]);
+          if (header) {
+            fprintf(dfile, "repartition_costs[%d][%d] = %10d; /* %s/%s */\n", j,
+                    k, fixed_cost, taskID, subtaskID_names[k]);
+          } else {
+            fprintf(dfile,
+                    "%15s/%-10s %10d %14.4f %14.4f %14.4f %14.4f %14.4f %10d\n",
+                    taskID, subtaskID_names[k], count[j][k],
+                    clocks_from_ticks(min[j][k]), clocks_from_ticks(max[j][k]),
+                    clocks_from_ticks(sum[j][k]), clocks_from_ticks(mean),
+                    perc, fixed_cost);
+          }
         }
       }
     }
     fclose(dfile);
-    fclose(cfile);
 #ifdef WITH_MPI
   }
 #endif

src/task.h

@@ -199,7 +199,7 @@ int task_lock(struct task *t);
 void task_do_rewait(struct task *t);
 void task_print(const struct task *t);
 void task_dump_all(struct engine *e, int step);
-void task_dump_stats(struct engine *e, int step);
+void task_dump_stats(const char *dumpfile, struct engine *e, int header, int allranks);
 #ifdef WITH_MPI
 void task_create_mpi_comms(void);
 #endif