diff --git a/src/engine.c b/src/engine.c index 21a3ba29fe8988c349ce6b02406b953fa006755e..056f19b2a99d4df1edb59dfe9132e2995fce3eca 100644 --- a/src/engine.c +++ b/src/engine.c @@ -4886,6 +4886,7 @@ static void *engine_dumper_poll(void *p) { #endif /* Add more interesting diagnostics. */ + scheduler_dump_queues(e); /* Delete the file. */ unlink(".dump"); diff --git a/src/queue.c b/src/queue.c index 3a9919163a4fb9d146c055e58859a175858c17eb..f6dbdcd0e0975e0cbd11161704d26408a9dfa356 100644 --- a/src/queue.c +++ b/src/queue.c @@ -301,3 +301,34 @@ void queue_clean(struct queue *q) { free(q->tid); free(q->tid_incoming); } + + +/** + * @brief Dump a formatted list of tasks in the queue to the given file stream. + * + * @param nodeID the node id of this rank. + * @param index a number for this queue, added to the output. + * @param file the FILE stream, should opened for write. + * @param q The task #queue. + */ +void queue_dump(int nodeID, int index, FILE *file, struct queue *q) { + + swift_lock_type *qlock = &q->lock; + + /* Grab the queue lock. */ + if (lock_lock(qlock) != 0) error("Locking the qlock failed.\n"); + + /* Fill any tasks from the incoming DEQ. */ + queue_get_incoming(q); + + /* Loop over the queue entries. */ + for (int k = 0; k < q->count; k++) { + struct task *t = &q->tasks[q->tid[k]]; + + fprintf(file, "%d %d %d %s %s %d\n", nodeID, index, k, + taskID_names[t->type], subtaskID_names[t->subtype], t->wait); + } + + /* Release the task lock. */ + if (lock_unlock(qlock) != 0) error("Unlocking the qlock failed.\n"); +} diff --git a/src/queue.h b/src/queue.h index c85cf0cabe30a03d163e2564fdc216c19495761a..e3c7b6f7b2059df14c14813b34946df2a1935c74 100644 --- a/src/queue.h +++ b/src/queue.h @@ -67,4 +67,6 @@ void queue_init(struct queue *q, struct task *tasks); void queue_insert(struct queue *q, struct task *t); void queue_clean(struct queue *q); +void queue_dump(int nodeID, int index, FILE *file, struct queue *q); + #endif /* SWIFT_QUEUE_H */ diff --git a/src/scheduler.c b/src/scheduler.c index 392c868e7e269bee630860ba94d5e87e4f1ac0b8..7bc0d4cb38d395d03ff45579adf2bef917282945 100644 --- a/src/scheduler.c +++ b/src/scheduler.c @@ -2265,3 +2265,61 @@ void scheduler_write_task_level(const struct scheduler *s) { fclose(f); free(count); } +/** + * @brief dump all the active queues of all the known schedulers into a file. + * + * @param e the #scheduler + */ +void scheduler_dump_queues(struct engine *e) { + + struct scheduler *s = &e->sched; + +#ifdef WITH_MPI + + /* Make sure output file is empty, only on one rank. */ + char dumpfile[35]; + snprintf(dumpfile, sizeof(dumpfile), "queue_dump_MPI-step%d.dat", e->step); + FILE *file_thread; + if (engine_rank == 0) { + file_thread = fopen(dumpfile, "w"); + fprintf(file_thread, "# rank queue index type subtype waits\n"); + fclose(file_thread); + } + MPI_Barrier(MPI_COMM_WORLD); + + for (int i = 0; i < e->nr_nodes; i++) { + + /* Rank 0 decides the index of the writing node, this happens + * one-by-one. */ + int kk = i; + MPI_Bcast(&kk, 1, MPI_INT, 0, MPI_COMM_WORLD); + + if (i == engine_rank) { + + /* Open file and position at end. */ + file_thread = fopen(dumpfile, "a"); + + for (int l = 0; l < s->nr_queues; l++) { + queue_dump(engine_rank, l, file_thread, &s->queues[l]); + } + fclose(file_thread); + } + + /* And we wait for all to synchronize. */ + MPI_Barrier(MPI_COMM_WORLD); + } + +#else + + /* Non-MPI, so just a single schedulers worth of queues to dump. */ + char dumpfile[32]; + snprintf(dumpfile, sizeof(dumpfile), "queue_dump-step%d.dat", e->step); + FILE *file_thread; + file_thread = fopen(dumpfile, "w"); + fprintf(file_thread, "# rank queue index type subtype waits\n"); + for (int l = 0; l < s->nr_queues; l++) { + queue_dump(engine_rank, l, file_thread, &s->queues[l]); + } + fclose(file_thread); +#endif // WITH_MPI +} diff --git a/src/scheduler.h b/src/scheduler.h index ac9bc754db1ea03f91c0ce522dc325c18d25ec09..c12d2e98061b1d68a671d68ade9b9d879dfab651 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -201,5 +201,6 @@ void scheduler_clean(struct scheduler *s); void scheduler_free_tasks(struct scheduler *s); void scheduler_write_dependencies(struct scheduler *s, int verbose); void scheduler_write_task_level(const struct scheduler *s); +void scheduler_dump_queues(struct engine *e); #endif /* SWIFT_SCHEDULER_H */