From 6c4dd72c630e3f5ae9fd0f298e9297ac34f59683 Mon Sep 17 00:00:00 2001 From: "Peter W. Draper" Date: Mon, 8 Jul 2019 13:10:26 +0100 Subject: [PATCH 1/5] Add an optional task dumper thread Polls for a file .dump and when found dumps the active tasks and memory (if configured) --- configure.ac | 13 ++++++ src/engine.c | 62 ++++++++++++++++++++++++++ src/task.c | 122 ++++++++++++++++++++++++++++++++++++++++++++++++++- src/task.h | 1 + 4 files changed, 197 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index c715fb34b..49f8dcbce 100644 --- a/configure.ac +++ b/configure.ac @@ -375,6 +375,7 @@ fi # Check whether we have any of the ARM v8.1 tick timers AX_ASM_ARM_PMCCNTR AX_ASM_ARM_CNTVCT + # See if we want memuse reporting. AC_ARG_ENABLE([memuse-reports], [AS_HELP_STRING([--enable-memuse-reports], @@ -387,6 +388,18 @@ if test "$enable_memuse_reports" = "yes"; then AC_DEFINE([SWIFT_MEMUSE_REPORTS],1,[Enable memory usage reports]) fi +# Check if we want to make the dumper thread active. +AC_ARG_ENABLE([dumper], + [AS_HELP_STRING([--enable-dumper], + [Dump active tasks and memory use (if configured)@<:@yes/no@:>@] + )], + [enable_dumper="$enableval"], + [enable_dumper="no"] +) +if test "$enable_dumper" = "yes"; then + AC_DEFINE([SWIFT_DUMPER_THREAD],1,[Enable dumper thread]) +fi + # Define HAVE_POSIX_MEMALIGN if it works. AX_FUNC_POSIX_MEMALIGN diff --git a/src/engine.c b/src/engine.c index 9e8ae1104..21a3ba29f 100644 --- a/src/engine.c +++ b/src/engine.c @@ -4863,6 +4863,62 @@ void engine_unpin(void) { #endif } +#ifdef SWIFT_DUMPER_THREAD +/** + * @brief dumper thread action, checks got the existence of the .dump file + * every 5 seconds and does the dump if found. + * + * @param p the #engine + */ +static void *engine_dumper_poll(void *p) { + struct engine *e = (struct engine *)p; + while (1) { + if (access(".dump", F_OK) == 0) { + + /* OK, do our work. */ + message("Dumping engine tasks in step: %d", e->step); + task_dump_active(e); + +#ifdef SWIFT_MEMUSE_REPORTS + /* Dump the currently logged memory. */ + message("Dumping memory use report"); + memuse_log_dump_error(e->nodeID); +#endif + + /* Add more interesting diagnostics. */ + + /* Delete the file. */ + unlink(".dump"); + message("Dumping completed"); + fflush(stdout); + } + + /* Take a breath. */ + sleep(5); + } + return NULL; +} +#endif /* SWIFT_DUMPER_THREAD */ + +#ifdef SWIFT_DUMPER_THREAD +/** + * @brief creates the dumper thread. + * + * This watches for the creation of a ".dump" file in the current directory + * and if found dumps the current state of the tasks and memory use (if also + * configured). + * + * @param e the #engine + * + */ +static void engine_dumper_init(struct engine *e) { + pthread_t dumper; + pthread_create(&dumper, NULL, &engine_dumper_poll, e); + + /* Thread does not exit, so nothing to do. */ +} +#endif /* SWIFT_DUMPER_THREAD */ + /** * @brief init an engine struct with the necessary properties for the * simulation. @@ -5701,6 +5757,12 @@ void engine_config(int restart, int fof, struct engine *e, free(buf); #endif +#ifdef SWIFT_DUMPER_THREAD + + /* Start the dumper thread.*/ + engine_dumper_init(e); +#endif + /* Wait for the runner threads to be in place. */ swift_barrier_wait(&e->wait_barrier); } diff --git a/src/task.c b/src/task.c index 9a0aa64fa..d22baae3b 100644 --- a/src/task.c +++ b/src/task.c @@ -856,7 +856,7 @@ void task_create_mpi_comms(void) { * * Dumps the information to a file "thread_info-stepn.dat" where n is the * given step value, or "thread_info_MPI-stepn.dat", if we are running - * under MPI. Note if running under MPIU all the ranks are dumped into this + * under MPI. Note if running under MPI all the ranks are dumped into this * one file, which has an additional field to identify the rank. * * @param e the #engine @@ -1089,3 +1089,123 @@ void task_dump_stats(const char *dumpfile, struct engine *e, int header, } #endif } + +/** + * @brief dump all the active tasks of all the known engines into a file. + * + * Dumps the information to a file "task_dump-stepn.dat" where n is the given + * step value, or "task_dump_MPI-stepn.dat", if we are running under MPI. Note + * if running under MPI all the ranks are dumped into this one file, which has + * an additional field to identify the rank. Very similar to task_dump_all() + * except for the additional fields used in task debugging and we record tasks + * that have not ran (i.e !skip, but toc == 0) and how many waits are still + * active. + * + * @param e the #engine + */ +void task_dump_active(struct engine *e) { + + /* Need this to convert ticks to seconds. */ + unsigned long long cpufreq = clocks_get_cpufreq(); + +#ifdef WITH_MPI + /* Make sure output file is empty, only on one rank. */ + char dumpfile[35]; + snprintf(dumpfile, sizeof(dumpfile), "task_dump_MPI-step%d.dat", e->step); + FILE *file_thread; + if (engine_rank == 0) { + file_thread = fopen(dumpfile, "w"); + fprintf(file_thread, + "# rank type subtype waits pair tic toc" + " ci.hydro.count cj.hydro.count ci.grav.count cj.grav.count" + " flags\n"); + fclose(file_thread); + } + MPI_Barrier(MPI_COMM_WORLD); + + for (int i = 0; i < e->nr_nodes; i++) { + + /* Rank 0 decides the index of the writing node, this happens + * one-by-one. */ + int kk = i; + MPI_Bcast(&kk, 1, MPI_INT, 0, MPI_COMM_WORLD); + + if (i == engine_rank) { + + /* Open file and position at end. */ + file_thread = fopen(dumpfile, "a"); + + /* Add some information to help with the plots and conversion of ticks to + * seconds. */ + fprintf( + file_thread, "%i none none -1 0 %lld %lld %lld %lld %lld 0 %lld\n", + engine_rank, (long long int)e->tic_step, (long long int)e->toc_step, + e->updates, e->g_updates, e->s_updates, cpufreq); + int count = 0; + for (int l = 0; l < e->sched.nr_tasks; l++) { + + /* Not implicit and not skipped. Note tasks that have not ran will + * have a toc of zero. */ + if (!e->sched.tasks[l].implicit && !e->sched.tasks[l].skip) { + fprintf( + file_thread, "%i %s %s %i %i %lli %lli %i %i %i %i %lli\n", + engine_rank, taskID_names[e->sched.tasks[l].type], + subtaskID_names[e->sched.tasks[l].subtype], + e->sched.tasks[l].wait, (e->sched.tasks[l].cj == NULL), + (long long int)e->sched.tasks[l].tic, + (long long int)e->sched.tasks[l].toc, + (e->sched.tasks[l].ci != NULL) ? e->sched.tasks[l].ci->hydro.count + : 0, + (e->sched.tasks[l].cj != NULL) ? e->sched.tasks[l].cj->hydro.count + : 0, + (e->sched.tasks[l].ci != NULL) ? e->sched.tasks[l].ci->grav.count + : 0, + (e->sched.tasks[l].cj != NULL) ? e->sched.tasks[l].cj->grav.count + : 0, + e->sched.tasks[l].flags); + } + count++; + } + fclose(file_thread); + } + + /* And we wait for all to synchronize. */ + MPI_Barrier(MPI_COMM_WORLD); + } + +#else + /* Non-MPI, so just a single engine's worth of tasks to dump. */ + char dumpfile[32]; + snprintf(dumpfile, sizeof(dumpfile), "task_dump-step%d.dat", e->step); + FILE *file_thread; + file_thread = fopen(dumpfile, "w"); + fprintf(file_thread, + "#type subtype waits pair tic toc ci.hydro.count cj.hydro.count " + "ci.grav.count cj.grav.count\n"); + + /* Add some information to help with the plots and conversion of ticks to + * seconds. */ + fprintf(file_thread, "none none -1 0, %lld %lld %lld %lld %lld %lld\n", + (unsigned long long)e->tic_step, (unsigned long long)e->toc_step, + e->updates, e->g_updates, e->s_updates, cpufreq); + for (int l = 0; l < e->sched.nr_tasks; l++) { + if (!e->sched.tasks[l].implicit && !e->sched.tasks[l].skip) { + fprintf( + file_thread, "%s %s %i %i %lli %lli %i %i %i %i\n", + taskID_names[e->sched.tasks[l].type], + subtaskID_names[e->sched.tasks[l].subtype], e->sched.tasks[l].wait, + (e->sched.tasks[l].cj == NULL), + (unsigned long long)e->sched.tasks[l].tic, + (unsigned long long)e->sched.tasks[l].toc, + (e->sched.tasks[l].ci == NULL) ? 0 + : e->sched.tasks[l].ci->hydro.count, + (e->sched.tasks[l].cj == NULL) ? 0 + : e->sched.tasks[l].cj->hydro.count, + (e->sched.tasks[l].ci == NULL) ? 0 : e->sched.tasks[l].ci->grav.count, + (e->sched.tasks[l].cj == NULL) ? 0 + : e->sched.tasks[l].cj->grav.count); + } + } + fclose(file_thread); +#endif // WITH_MPI +} diff --git a/src/task.h b/src/task.h index 363e12575..3b1c0f852 100644 --- a/src/task.h +++ b/src/task.h @@ -235,6 +235,7 @@ void task_print(const struct task *t); void task_dump_all(struct engine *e, int step); void task_dump_stats(const char *dumpfile, struct engine *e, int header, int allranks); +void task_dump_active(struct engine *e); void task_get_full_name(int type, int subtype, char *name); void task_get_group_name(int type, int subtype, char *cluster); -- GitLab From 3116c16f8a7a5fbf9be755ef9bcab3875240f46a Mon Sep 17 00:00:00 2001 From: "Peter W. Draper" Date: Wed, 10 Jul 2019 15:56:07 +0100 Subject: [PATCH 2/5] Also add rank of the pair, so we know where the message is going --- src/task.c | 67 ++++++++++++++++++++++++------------------------------ 1 file changed, 30 insertions(+), 37 deletions(-) diff --git a/src/task.c b/src/task.c index d22baae3b..28d3abb04 100644 --- a/src/task.c +++ b/src/task.c @@ -1116,7 +1116,7 @@ void task_dump_active(struct engine *e) { if (engine_rank == 0) { file_thread = fopen(dumpfile, "w"); fprintf(file_thread, - "# rank type subtype waits pair tic toc" + "# rank otherrank type subtype waits pair tic toc" " ci.hydro.count cj.hydro.count ci.grav.count cj.grav.count" " flags\n"); fclose(file_thread); @@ -1138,31 +1138,29 @@ void task_dump_active(struct engine *e) { /* Add some information to help with the plots and conversion of ticks to * seconds. */ fprintf( - file_thread, "%i none none -1 0 %lld %lld %lld %lld %lld 0 %lld\n", + file_thread, "%i 0 none none -1 0 %lld %lld %lld %lld %lld 0 %lld\n", engine_rank, (long long int)e->tic_step, (long long int)e->toc_step, e->updates, e->g_updates, e->s_updates, cpufreq); int count = 0; for (int l = 0; l < e->sched.nr_tasks; l++) { - - /* Not implicit and not skipped. Note tasks that have not ran will - * have a toc of zero. */ - if (!e->sched.tasks[l].implicit && !e->sched.tasks[l].skip) { - fprintf( - file_thread, "%i %s %s %i %i %lli %lli %i %i %i %i %lli\n", - engine_rank, taskID_names[e->sched.tasks[l].type], - subtaskID_names[e->sched.tasks[l].subtype], - e->sched.tasks[l].wait, (e->sched.tasks[l].cj == NULL), - (long long int)e->sched.tasks[l].tic, - (long long int)e->sched.tasks[l].toc, - (e->sched.tasks[l].ci != NULL) ? e->sched.tasks[l].ci->hydro.count - : 0, - (e->sched.tasks[l].cj != NULL) ? e->sched.tasks[l].cj->hydro.count - : 0, - (e->sched.tasks[l].ci != NULL) ? e->sched.tasks[l].ci->grav.count - : 0, - (e->sched.tasks[l].cj != NULL) ? e->sched.tasks[l].cj->grav.count - : 0, - e->sched.tasks[l].flags); + struct task *t = &e->sched.tasks[l]; + + /* Not implicit and not skipped. */ + if (!t->implicit && !t->skip) { + + /* Get destination rank of MPI requests. */ + int paired = (t->cj != NULL); + int otherrank = t->ci->nodeID; + if (paired) otherrank = t->cj->nodeID; + + fprintf(file_thread, "%i %i %s %s %i %i %lli %lli %i %i %i %i %lli\n", + engine_rank, otherrank, taskID_names[t->type], + subtaskID_names[t->subtype], t->wait, paired, + (long long int)t->tic, (long long int)t->toc, + (t->ci != NULL) ? t->ci->hydro.count : 0, + (t->cj != NULL) ? t->cj->hydro.count : 0, + (t->ci != NULL) ? t->ci->grav.count : 0, + (t->cj != NULL) ? t->cj->grav.count : 0, t->flags); } count++; } @@ -1189,21 +1187,16 @@ void task_dump_active(struct engine *e) { (unsigned long long)e->tic_step, (unsigned long long)e->toc_step, e->updates, e->g_updates, e->s_updates, cpufreq); for (int l = 0; l < e->sched.nr_tasks; l++) { - if (!e->sched.tasks[l].implicit && !e->sched.tasks[l].skip) { - fprintf( - file_thread, "%s %s %i %i %lli %lli %i %i %i %i\n", - taskID_names[e->sched.tasks[l].type], - subtaskID_names[e->sched.tasks[l].subtype], e->sched.tasks[l].wait, - (e->sched.tasks[l].cj == NULL), - (unsigned long long)e->sched.tasks[l].tic, - (unsigned long long)e->sched.tasks[l].toc, - (e->sched.tasks[l].ci == NULL) ? 0 - : e->sched.tasks[l].ci->hydro.count, - (e->sched.tasks[l].cj == NULL) ? 0 - : e->sched.tasks[l].cj->hydro.count, - (e->sched.tasks[l].ci == NULL) ? 0 : e->sched.tasks[l].ci->grav.count, - (e->sched.tasks[l].cj == NULL) ? 0 - : e->sched.tasks[l].cj->grav.count); + struct task *t = &e->sched.tasks[l]; + if (!t->implicit && !t->skip) { + fprintf(file_thread, "%s %s %i %i %lli %lli %i %i %i %i\n", + taskID_names[t->type], subtaskID_names[t->subtype], t->wait, + (t->cj == NULL), (unsigned long long)t->tic, + (unsigned long long)t->toc, + (t->ci == NULL) ? 0 : t->ci->hydro.count, + (t->cj == NULL) ? 0 : t->cj->hydro.count, + (t->ci == NULL) ? 0 : t->ci->grav.count, + (t->cj == NULL) ? 0 : t->cj->grav.count); } } fclose(file_thread); -- GitLab From 6248a70e1859153fa21cdebd5fe9f5b738a68caf Mon Sep 17 00:00:00 2001 From: "Peter W. Draper" Date: Tue, 13 Aug 2019 17:52:42 +0100 Subject: [PATCH 3/5] Add function to dump all the queues and add to the dumper thread --- src/engine.c | 1 + src/queue.c | 31 ++++++++++++++++++++++++++ src/queue.h | 2 ++ src/scheduler.c | 58 +++++++++++++++++++++++++++++++++++++++++++++++++ src/scheduler.h | 1 + 5 files changed, 93 insertions(+) diff --git a/src/engine.c b/src/engine.c index 21a3ba29f..056f19b2a 100644 --- a/src/engine.c +++ b/src/engine.c @@ -4886,6 +4886,7 @@ static void *engine_dumper_poll(void *p) { #endif /* Add more interesting diagnostics. */ + scheduler_dump_queues(e); /* Delete the file. */ unlink(".dump"); diff --git a/src/queue.c b/src/queue.c index 3a9919163..f6dbdcd0e 100644 --- a/src/queue.c +++ b/src/queue.c @@ -301,3 +301,34 @@ void queue_clean(struct queue *q) { free(q->tid); free(q->tid_incoming); } + + +/** + * @brief Dump a formatted list of tasks in the queue to the given file stream. + * + * @param nodeID the node id of this rank. + * @param index a number for this queue, added to the output. + * @param file the FILE stream, should opened for write. + * @param q The task #queue. + */ +void queue_dump(int nodeID, int index, FILE *file, struct queue *q) { + + swift_lock_type *qlock = &q->lock; + + /* Grab the queue lock. */ + if (lock_lock(qlock) != 0) error("Locking the qlock failed.\n"); + + /* Fill any tasks from the incoming DEQ. */ + queue_get_incoming(q); + + /* Loop over the queue entries. */ + for (int k = 0; k < q->count; k++) { + struct task *t = &q->tasks[q->tid[k]]; + + fprintf(file, "%d %d %d %s %s %d\n", nodeID, index, k, + taskID_names[t->type], subtaskID_names[t->subtype], t->wait); + } + + /* Release the task lock. */ + if (lock_unlock(qlock) != 0) error("Unlocking the qlock failed.\n"); +} diff --git a/src/queue.h b/src/queue.h index c85cf0cab..e3c7b6f7b 100644 --- a/src/queue.h +++ b/src/queue.h @@ -67,4 +67,6 @@ void queue_init(struct queue *q, struct task *tasks); void queue_insert(struct queue *q, struct task *t); void queue_clean(struct queue *q); +void queue_dump(int nodeID, int index, FILE *file, struct queue *q); + #endif /* SWIFT_QUEUE_H */ diff --git a/src/scheduler.c b/src/scheduler.c index 392c868e7..7bc0d4cb3 100644 --- a/src/scheduler.c +++ b/src/scheduler.c @@ -2265,3 +2265,61 @@ void scheduler_write_task_level(const struct scheduler *s) { fclose(f); free(count); } +/** + * @brief dump all the active queues of all the known schedulers into a file. + * + * @param e the #scheduler + */ +void scheduler_dump_queues(struct engine *e) { + + struct scheduler *s = &e->sched; + +#ifdef WITH_MPI + + /* Make sure output file is empty, only on one rank. */ + char dumpfile[35]; + snprintf(dumpfile, sizeof(dumpfile), "queue_dump_MPI-step%d.dat", e->step); + FILE *file_thread; + if (engine_rank == 0) { + file_thread = fopen(dumpfile, "w"); + fprintf(file_thread, "# rank queue index type subtype waits\n"); + fclose(file_thread); + } + MPI_Barrier(MPI_COMM_WORLD); + + for (int i = 0; i < e->nr_nodes; i++) { + + /* Rank 0 decides the index of the writing node, this happens + * one-by-one. */ + int kk = i; + MPI_Bcast(&kk, 1, MPI_INT, 0, MPI_COMM_WORLD); + + if (i == engine_rank) { + + /* Open file and position at end. */ + file_thread = fopen(dumpfile, "a"); + + for (int l = 0; l < s->nr_queues; l++) { + queue_dump(engine_rank, l, file_thread, &s->queues[l]); + } + fclose(file_thread); + } + + /* And we wait for all to synchronize. */ + MPI_Barrier(MPI_COMM_WORLD); + } + +#else + + /* Non-MPI, so just a single schedulers worth of queues to dump. */ + char dumpfile[32]; + snprintf(dumpfile, sizeof(dumpfile), "queue_dump-step%d.dat", e->step); + FILE *file_thread; + file_thread = fopen(dumpfile, "w"); + fprintf(file_thread, "# rank queue index type subtype waits\n"); + for (int l = 0; l < s->nr_queues; l++) { + queue_dump(engine_rank, l, file_thread, &s->queues[l]); + } + fclose(file_thread); +#endif // WITH_MPI +} diff --git a/src/scheduler.h b/src/scheduler.h index ac9bc754d..c12d2e980 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -201,5 +201,6 @@ void scheduler_clean(struct scheduler *s); void scheduler_free_tasks(struct scheduler *s); void scheduler_write_dependencies(struct scheduler *s, int verbose); void scheduler_write_task_level(const struct scheduler *s); +void scheduler_dump_queues(struct engine *e); #endif /* SWIFT_SCHEDULER_H */ -- GitLab From 9a5680fe19d79698c2fe5777641b4a22459657e4 Mon Sep 17 00:00:00 2001 From: "Peter W. Draper" Date: Wed, 14 Aug 2019 13:03:04 +0100 Subject: [PATCH 4/5] Make sure the .dump file is removed when starting, if not that can give funny effects --- src/engine.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/engine.c b/src/engine.c index bd4a83152..b84063145 100644 --- a/src/engine.c +++ b/src/engine.c @@ -33,8 +33,11 @@ #include #include #include +#include +#include #include + /* MPI headers. */ #ifdef WITH_MPI #include @@ -4920,9 +4923,13 @@ static void *engine_dumper_poll(void *p) { */ static void engine_dumper_init(struct engine *e) { pthread_t dumper; - pthread_create(&dumper, NULL, &engine_dumper_poll, e); - /* Thread does not exit, so nothing to do. */ + /* Make sure the .dump file is not present, that is bad when starting up. */ + struct stat buf; + if (stat(".dump", &buf) == 0) unlink(".dump"); + + /* Thread does not exit, so nothing to do but create it. */ + pthread_create(&dumper, NULL, &engine_dumper_poll, e); } #endif /* SWIFT_DUMPER_THREAD */ -- GitLab From 705f24b3774ac714304f13af17b0ad08078c147b Mon Sep 17 00:00:00 2001 From: "Peter W. Draper" Date: Fri, 16 Aug 2019 13:39:01 +0100 Subject: [PATCH 5/5] Output weight not wait! Cut down on cut-and-paste coding --- src/queue.c | 4 ++-- src/scheduler.c | 23 ++++++++++++----------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/queue.c b/src/queue.c index f6dbdcd0e..95ade9027 100644 --- a/src/queue.c +++ b/src/queue.c @@ -325,8 +325,8 @@ void queue_dump(int nodeID, int index, FILE *file, struct queue *q) { for (int k = 0; k < q->count; k++) { struct task *t = &q->tasks[q->tid[k]]; - fprintf(file, "%d %d %d %s %s %d\n", nodeID, index, k, - taskID_names[t->type], subtaskID_names[t->subtype], t->wait); + fprintf(file, "%d %d %d %s %s %.2f\n", nodeID, index, k, + taskID_names[t->type], subtaskID_names[t->subtype], t->weight); } /* Release the task lock. */ diff --git a/src/scheduler.c b/src/scheduler.c index 7cbfbe486..230b7abc3 100644 --- a/src/scheduler.c +++ b/src/scheduler.c @@ -2305,17 +2305,23 @@ void scheduler_dump_queues(struct engine *e) { struct scheduler *s = &e->sched; -#ifdef WITH_MPI - - /* Make sure output file is empty, only on one rank. */ + /* Open the file and write the header. */ char dumpfile[35]; +#ifdef WITH_MPI snprintf(dumpfile, sizeof(dumpfile), "queue_dump_MPI-step%d.dat", e->step); - FILE *file_thread; +#else + snprintf(dumpfile, sizeof(dumpfile), "queue_dump-step%d.dat", e->step); +#endif + FILE *file_thread = NULL; if (engine_rank == 0) { file_thread = fopen(dumpfile, "w"); - fprintf(file_thread, "# rank queue index type subtype waits\n"); - fclose(file_thread); + fprintf(file_thread, "# rank queue index type subtype weight\n"); } + +#ifdef WITH_MPI + + /* Make sure output file is closed and empty, then we reopen on each rank. */ + if (engine_rank == 0) fclose(file_thread); MPI_Barrier(MPI_COMM_WORLD); for (int i = 0; i < e->nr_nodes; i++) { @@ -2343,11 +2349,6 @@ void scheduler_dump_queues(struct engine *e) { #else /* Non-MPI, so just a single schedulers worth of queues to dump. */ - char dumpfile[32]; - snprintf(dumpfile, sizeof(dumpfile), "queue_dump-step%d.dat", e->step); - FILE *file_thread; - file_thread = fopen(dumpfile, "w"); - fprintf(file_thread, "# rank queue index type subtype waits\n"); for (int l = 0; l < s->nr_queues; l++) { queue_dump(engine_rank, l, file_thread, &s->queues[l]); } -- GitLab