Commit b0f801dd authored by Peter W. Draper's avatar Peter W. Draper
Browse files

Only dump the fixed costs file on a sensible repartition

parent 4cd3ccbc
...@@ -1037,7 +1037,9 @@ int main(int argc, char *argv[]) { ...@@ -1037,7 +1037,9 @@ int main(int argc, char *argv[]) {
#endif #endif
/* Generate the task statistics. */ /* Generate the task statistics. */
task_dump_stats(&e, j + 1); char dumpfile[40];
snprintf(dumpfile, 40, "thread_stats-step%d.dat", j + 1);
task_dump_stats(dumpfile, &e, /* header = */ 0, /* allranks = */ 1);
} }
#ifdef SWIFT_DEBUG_THREADPOOL #ifdef SWIFT_DEBUG_THREADPOOL
......
...@@ -41,8 +41,8 @@ endif ...@@ -41,8 +41,8 @@ endif
# List required headers # List required headers
include_HEADERS = space.h runner.h queue.h task.h lock.h cell.h part.h const.h \ include_HEADERS = space.h runner.h queue.h task.h lock.h cell.h part.h const.h \
engine.h swift.h serial_io.h timers.h debug.h scheduler.h proxy.h parallel_io.h \ engine.h swift.h serial_io.h timers.h debug.h scheduler.h proxy.h parallel_io.h \
common_io.h single_io.h multipole.h map.h tools.h partition.h clocks.h parser.h \ common_io.h single_io.h multipole.h map.h tools.h partition.h partition_fixed_costs.h \
physical_constants.h physical_constants_cgs.h potential.h version.h \ clocks.h parser.h physical_constants.h physical_constants_cgs.h potential.h version.h \
hydro_properties.h riemann.h threadpool.h cooling_io.h cooling.h cooling_struct.h \ hydro_properties.h riemann.h threadpool.h cooling_io.h cooling.h cooling_struct.h \
sourceterms.h sourceterms_struct.h statistics.h memswap.h cache.h runner_doiact_vec.h profiler.h \ sourceterms.h sourceterms_struct.h statistics.h memswap.h cache.h runner_doiact_vec.h profiler.h \
dump.h logger.h active.h timeline.h xmf.h gravity_properties.h gravity_derivatives.h \ dump.h logger.h active.h timeline.h xmf.h gravity_properties.h gravity_derivatives.h \
......
...@@ -991,6 +991,11 @@ void engine_repartition(struct engine *e) { ...@@ -991,6 +991,11 @@ void engine_repartition(struct engine *e) {
* bug that doesn't handle this case well. */ * bug that doesn't handle this case well. */
if (e->nr_nodes == 1) return; if (e->nr_nodes == 1) return;
/* Generate the fixed costs include file. */
if (e->step > 3 && e->reparttype->trigger <= 1.f) {
task_dump_stats("partition_fixed_costs.h", e, /* header = */ 1, /* allranks = */ 1);
}
/* Do the repartitioning. */ /* Do the repartitioning. */
partition_repartition(e->reparttype, e->nodeID, e->nr_nodes, e->s, partition_repartition(e->reparttype, e->nodeID, e->nr_nodes, e->s,
e->sched.tasks, e->sched.nr_tasks); e->sched.tasks, e->sched.nr_tasks);
......
...@@ -687,26 +687,26 @@ void task_dump_all(struct engine *e, int step) { ...@@ -687,26 +687,26 @@ void task_dump_all(struct engine *e, int step) {
/** /**
* @brief Generate simple statistics about the times used by the tasks of * @brief Generate simple statistics about the times used by the tasks of
* all the engines and write these into two files, a human readable * all the engines and write these into two format, a human readable
* version and one intented for inclusion as the fixed costs for * version for debugging and one intented for inclusion as the fixed
* repartitioning. * costs for repartitioning.
* *
* Dumps the human readable information to a file "thread_stats-stepn.dat" * Note that when running under MPI all the tasks can be summed into this single
* where n is the given step value. When running under MPI all the tasks are * file. In the fuller, human readable file, the statistics included are the
* summed into this single file. * number of task of each type/subtype followed by the minimum, maximum, mean
* and total time, in millisec and then the fixed costs value.
* *
* The fixed costs file will be called "thread_stats-stepn.h". * If header is set, only the fixed costs value is written into the output
* file in a format that is suitable for inclusion in SWIFT (as
* partition_fixed_costs.h).
* *
* @param dumpfile name of the file for the output.
* @param e the #engine * @param e the #engine
* @param step the current step. * @param header whether to write a header include file.
* @param allranks do the statistics over all ranks, if not just the current
* one, only used if header is false.
*/ */
void task_dump_stats(struct engine *e, int step) { void task_dump_stats(const char *dumpfile, struct engine *e, int header, int allranks) {
char dumpfile[40];
snprintf(dumpfile, 40, "thread_stats-step%d.dat", step);
char costsfile[40];
snprintf(costsfile, 40, "thread_stats-step%d.h", step);
/* Need arrays for sum, min and max across all types and subtypes. */ /* Need arrays for sum, min and max across all types and subtypes. */
double sum[task_type_count][task_subtype_count]; double sum[task_type_count][task_subtype_count];
...@@ -746,7 +746,9 @@ void task_dump_stats(struct engine *e, int step) { ...@@ -746,7 +746,9 @@ void task_dump_stats(struct engine *e, int step) {
} }
} }
#ifdef WITH_MPI #ifdef WITH_MPI
if (allranks || header) {
/* Get these from all ranks for output from rank 0. Could wrap these into a /* Get these from all ranks for output from rank 0. Could wrap these into a
* single operation. */ * single operation. */
size_t size = task_type_count * task_subtype_count; size_t size = task_type_count * task_subtype_count;
...@@ -769,16 +771,18 @@ void task_dump_stats(struct engine *e, int step) { ...@@ -769,16 +771,18 @@ void task_dump_stats(struct engine *e, int step) {
res = MPI_Reduce((engine_rank == 0 ? MPI_IN_PLACE : total), total, 1, res = MPI_Reduce((engine_rank == 0 ? MPI_IN_PLACE : total), total, 1,
MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);
if (res != MPI_SUCCESS) mpi_error(res, "Failed to reduce task total time"); if (res != MPI_SUCCESS) mpi_error(res, "Failed to reduce task total time");
}
if (engine_rank == 0) { if (!allranks || (engine_rank == 0 && (allranks || header))) {
#endif #endif
FILE *dfile = fopen(dumpfile, "w"); FILE *dfile = fopen(dumpfile, "w");
if (header) {
fprintf(dfile, "/* use as src/partition_fixed_costs.h */\n");
fprintf(dfile, "#define HAVE_FIXED_COSTS 1\n");
} else {
fprintf(dfile, "# task ntasks min max sum mean percent fixed_cost\n"); fprintf(dfile, "# task ntasks min max sum mean percent fixed_cost\n");
}
FILE *cfile = fopen(costsfile, "w");
fprintf(cfile, "/* use as src/partition_fixed_costs.h */\n");
fprintf(cfile, "#define HAVE_FIXED_COSTS 1\n");
for (int j = 0; j < task_type_count; j++) { for (int j = 0; j < task_type_count; j++) {
const char *taskID = taskID_names[j]; const char *taskID = taskID_names[j];
...@@ -790,19 +794,21 @@ void task_dump_stats(struct engine *e, int step) { ...@@ -790,19 +794,21 @@ void task_dump_stats(struct engine *e, int step) {
/* Fixed cost is in .1ns as we want to compare between runs in /* Fixed cost is in .1ns as we want to compare between runs in
* some absolute units. */ * some absolute units. */
int fixed_cost = (int)(clocks_from_ticks(mean) * 10000.f); int fixed_cost = (int)(clocks_from_ticks(mean) * 10000.f);
if (header) {
fprintf(dfile, "repartition_costs[%d][%d] = %10d; /* %s/%s */\n", j,
k, fixed_cost, taskID, subtaskID_names[k]);
} else {
fprintf(dfile, fprintf(dfile,
"%15s/%-10s %10d %14.4f %14.4f %14.4f %14.4f %14.4f %10d\n", "%15s/%-10s %10d %14.4f %14.4f %14.4f %14.4f %14.4f %10d\n",
taskID, subtaskID_names[k], count[j][k], taskID, subtaskID_names[k], count[j][k],
clocks_from_ticks(min[j][k]), clocks_from_ticks(max[j][k]), clocks_from_ticks(min[j][k]), clocks_from_ticks(max[j][k]),
clocks_from_ticks(sum[j][k]), clocks_from_ticks(mean), clocks_from_ticks(sum[j][k]), clocks_from_ticks(mean),
perc, fixed_cost); perc, fixed_cost);
fprintf(cfile, "repartition_costs[%d][%d] = %10d; /* %s/%s */\n", j, }
k, fixed_cost, taskID, subtaskID_names[k]);
} }
} }
} }
fclose(dfile); fclose(dfile);
fclose(cfile);
#ifdef WITH_MPI #ifdef WITH_MPI
} }
#endif #endif
......
...@@ -199,7 +199,7 @@ int task_lock(struct task *t); ...@@ -199,7 +199,7 @@ int task_lock(struct task *t);
void task_do_rewait(struct task *t); void task_do_rewait(struct task *t);
void task_print(const struct task *t); void task_print(const struct task *t);
void task_dump_all(struct engine *e, int step); void task_dump_all(struct engine *e, int step);
void task_dump_stats(struct engine *e, int step); void task_dump_stats(const char *dumpfile, struct engine *e, int header, int allranks);
#ifdef WITH_MPI #ifdef WITH_MPI
void task_create_mpi_comms(void); void task_create_mpi_comms(void);
#endif #endif
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment