diff --git a/.gitignore b/.gitignore
index 44d98c6ff5fa70ab2d5bf53752c30b40b69f6272..5615f036ef0f3a51db7c156afe69b8511e015d4c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,7 +29,7 @@ examples/*/*.h5
examples/*/*.png
examples/*/*.mp4
examples/*/*.txt
-examples/*/*.dot
+examples/*/dependency_graph_*.csv
examples/*/restart/*
examples/*/used_parameters.yml
examples/*/unused_parameters.yml
diff --git a/examples/main.c b/examples/main.c
index aca3cf8b1f85c19d7522259f34094271ec9998c9..b571eebcbd48af899ec9b0a56b753a3cbd333986 100644
--- a/examples/main.c
+++ b/examples/main.c
@@ -477,8 +477,7 @@ int main(int argc, char *argv[]) {
}
/* Check that we can write the structure finding catalogues by testing if the
- * output
- * directory exists and is searchable and writable. */
+ * output directory exists and is searchable and writable. */
if (with_structure_finding) {
char stfbasename[PARSER_MAX_LINE_SIZE];
parser_get_param_string(params, "StructureFinding:basename", stfbasename);
diff --git a/src/engine.c b/src/engine.c
index 2734fe6a1d572f19c6763931c314535170cc8b8c..5edbed475a822a50503e3766ac702cdf0db7b201 100644
--- a/src/engine.c
+++ b/src/engine.c
@@ -2798,7 +2798,7 @@ void engine_init_particles(struct engine *e, int flag_entropy_ICs,
gravity_exact_force_compute(e->s, e);
#endif
- if (e->nodeID == 0) scheduler_write_dependencies(&e->sched, e->verbose);
+ scheduler_write_dependencies(&e->sched, e->verbose);
if (e->nodeID == 0) scheduler_write_task_level(&e->sched);
/* Run the 0th time-step */
@@ -4671,8 +4671,7 @@ void engine_config(int restart, struct engine *e, struct swift_params *params,
/* Expected average for tasks per cell. If set to zero we use a heuristic
* guess based on the numbers of cells and how many tasks per cell we expect.
* On restart this number cannot be estimated (no cells yet), so we recover
- * from the end of the dumped run. Can be changed on restart.
- */
+ * from the end of the dumped run. Can be changed on restart. */
e->tasks_per_cell =
parser_get_opt_param_int(params, "Scheduler:tasks_per_cell", 0);
int maxtasks = 0;
diff --git a/src/scheduler.c b/src/scheduler.c
index 70705e3ee48fc3a038bd3ecc835b77017588f24f..bf976e39f33afa2a92ec3a68202ec17ff70c2dc9 100644
--- a/src/scheduler.c
+++ b/src/scheduler.c
@@ -29,6 +29,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <sys/stat.h>
/* MPI headers. */
#ifdef WITH_MPI
@@ -114,31 +115,217 @@ void scheduler_addunlock(struct scheduler *s, struct task *ta,
}
/**
- * @brief generate the dependency name for the tasks
+ * @brief compute the number of similar dependencies
*
- * @param ta_type The #task type.
- * @param ta_subtype The #task type.
- * @param ta_name (return) The formatted string
+ * @param s The #scheduler
+ * @param ta The #task
+ * @param tb The dependent #task
+ *
+ * @return Number of dependencies
*/
-void scheduler_task_dependency_name(int ta_type, int ta_subtype,
- char *ta_name) {
-
- /* Check input */
- if ((ta_type < 0) || (ta_type >= task_type_count))
- error("Unknown task type %i", ta_type);
-
- if ((ta_subtype < 0) || (ta_subtype >= task_subtype_count))
- error("Unknown task subtype %i with type %s", ta_subtype,
- taskID_names[ta_type]);
-
- /* construct line */
- if (ta_subtype == task_subtype_none)
- sprintf(ta_name, "%s", taskID_names[ta_type]);
- else
- sprintf(ta_name, "\"%s %s\"", taskID_names[ta_type],
- subtaskID_names[ta_subtype]);
+int scheduler_get_number_relation(const struct scheduler *s,
+ const struct task *ta,
+ const struct task *tb) {
+
+ int count = 0;
+
+ /* loop over all tasks */
+ for (int i = 0; i < s->nr_tasks; i++) {
+ const struct task *ta_tmp = &s->tasks[i];
+
+ /* and their dependencies */
+ for (int j = 0; j < ta->nr_unlock_tasks; j++) {
+ const struct task *tb_tmp = ta->unlock_tasks[j];
+
+ if (ta->type == ta_tmp->type && ta->subtype == ta_tmp->subtype &&
+ tb->type == tb_tmp->type && tb->subtype == tb_tmp->subtype) {
+ count += 1;
+ }
+ }
+ }
+ return count;
+}
+
+/* Conservative number of dependencies per task type */
+#define MAX_NUMBER_DEP 128
+
+/**
+ * @brief Informations about all the task dependencies of
+ * a single task.
+ */
+struct task_dependency {
+ /* Main task */
+ /* ID of the task */
+ int type_in;
+
+ /* ID of the subtask */
+ int subtype_in;
+
+ /* Is the task implicit */
+ int implicit_in;
+
+ /* Dependent task */
+ /* ID of the dependent task */
+ int type_out[MAX_NUMBER_DEP];
+
+ /* ID of the dependent subtask */
+ int subtype_out[MAX_NUMBER_DEP];
+
+ /* Is the dependent task implicit */
+ int implicit_out[MAX_NUMBER_DEP];
+
+ /* Statistics */
+ /* number of link between the two task type */
+ int number_link[MAX_NUMBER_DEP];
+
+ /* number of ranks having this relation */
+ int number_rank[MAX_NUMBER_DEP];
+};
+
+#ifdef WITH_MPI
+/**
+ * @brief Define the #task_dependency for MPI
+ *
+ * @param tstype The #MPI_Datatype to initialize
+ */
+void task_dependency_define(MPI_Datatype *tstype) {
+
+ /* Define the variables */
+ const int count = 8;
+ int blocklens[count];
+ MPI_Datatype types[count];
+ MPI_Aint disps[count];
+
+ /* all the type are int */
+ for (int i = 0; i < count; i++) {
+ types[i] = MPI_INT;
+ }
+
+ /* Task in */
+ disps[0] = offsetof(struct task_dependency, type_in);
+ blocklens[0] = 1;
+ disps[1] = offsetof(struct task_dependency, subtype_in);
+ blocklens[1] = 1;
+ disps[2] = offsetof(struct task_dependency, implicit_in);
+ blocklens[2] = 1;
+
+ /* Task out */
+ disps[3] = offsetof(struct task_dependency, type_out);
+ blocklens[3] = MAX_NUMBER_DEP;
+ disps[4] = offsetof(struct task_dependency, subtype_out);
+ blocklens[4] = MAX_NUMBER_DEP;
+ disps[5] = offsetof(struct task_dependency, implicit_out);
+ blocklens[5] = MAX_NUMBER_DEP;
+
+ /* statistics */
+ disps[6] = offsetof(struct task_dependency, number_link);
+ blocklens[6] = MAX_NUMBER_DEP;
+ disps[7] = offsetof(struct task_dependency, number_rank);
+ blocklens[7] = MAX_NUMBER_DEP;
+
+ /* define it for MPI */
+ MPI_Type_create_struct(count, blocklens, disps, types, tstype);
+ MPI_Type_commit(tstype);
+}
+
+/**
+ * @brief Sum operator of #task_dependency for MPI
+ *
+ * @param in_p The #task_dependency to add
+ * @param out_p The #task_dependency where in_p is added
+ * @param len The length of the arrays
+ * @param type The MPI datatype
+ */
+void task_dependency_sum(void *in_p, void *out_p, int *len,
+ MPI_Datatype *type) {
+
+ /* change pointer type */
+ struct task_dependency *in = (struct task_dependency *)in_p;
+ struct task_dependency *out = (struct task_dependency *)out_p;
+
+ /* Loop over all the current objects */
+ for (int i = 0; i < *len; i++) {
+
+ /* loop over all the object set in invals */
+ for (int j = 0; j < MAX_NUMBER_DEP; j++) {
+
+ /* Have we reached the end of the links? */
+ if (in[i].number_link[j] == -1) {
+ break;
+ }
+
+ /* get a few variables */
+ int tb_type = in[i].type_out[j];
+ int tb_subtype = in[i].subtype_out[j];
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Check tasks */
+ if (tb_type >= task_type_count) {
+ error("Unknown task type %i", tb_type);
+ }
+
+ if (tb_subtype >= task_subtype_count) {
+ error("Unknown subtask type %i", tb_subtype);
+ }
+#endif
+
+ /* find the corresponding id */
+ int k = 0;
+ while (k < MAX_NUMBER_DEP) {
+ /* have we reached the end of the links? */
+ if (out[i].number_link[k] == -1) {
+ /* reset the counter in order to be safe */
+ out[i].number_link[k] = 0;
+ out[i].number_rank[k] = 0;
+
+ /* set the relation */
+ out[i].type_in = in[i].type_in;
+ out[i].subtype_in = in[i].subtype_in;
+ out[i].implicit_in = in[i].implicit_in;
+
+ out[i].type_out[k] = in[i].type_out[j];
+ out[i].subtype_out[k] = in[i].subtype_out[j];
+ out[i].implicit_out[k] = in[i].implicit_out[j];
+ break;
+ }
+
+ /* do we have the same relation? */
+ if (out[i].type_out[k] == tb_type &&
+ out[i].subtype_out[k] == tb_subtype) {
+ break;
+ }
+
+ k++;
+ }
+
+ /* Check if we are still in the memory */
+ if (k == MAX_NUMBER_DEP) {
+ error("Not enough memory, please increase MAX_NUMBER_DEP");
+ }
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Check if correct relation */
+ if (out[i].type_in != in[i].type_in ||
+ out[i].subtype_in != in[i].subtype_in ||
+ out[i].implicit_in != in[i].implicit_in ||
+ out[i].type_out[k] != in[i].type_out[j] ||
+ out[i].subtype_out[k] != in[i].subtype_out[j] ||
+ out[i].implicit_out[k] != in[i].implicit_out[j]) {
+ error("Tasks do not correspond");
+ }
+#endif
+
+ /* sum the contributions */
+ out[i].number_link[k] += in[i].number_link[j];
+ out[i].number_rank[k] += in[i].number_rank[j];
+ }
+ }
+
+ return;
}
+#endif // WITH_MPI
+
/**
* @brief Write a dot file with the task dependencies.
*
@@ -152,253 +339,178 @@ void scheduler_write_dependencies(struct scheduler *s, int verbose) {
const ticks tic = getticks();
- /* Conservative number of dependencies per task type */
- const int max_nber_dep = 128;
-
/* Number of possible relations between tasks */
- const int nber_relation =
- 2 * task_type_count * task_subtype_count * max_nber_dep;
+ const int nber_tasks = task_type_count * task_subtype_count;
- /* To get the table of max_nber_dep for a task:
- * ind = (ta * task_subtype_count + sa) * max_nber_dep * 2
+ /* To get the table for a task:
+ * ind = (ta * task_subtype_count + sa)
* where ta is the value of task_type and sa is the value of
* task_subtype */
- int *table = (int *)malloc(nber_relation * sizeof(int));
- if (table == NULL)
- error("Error allocating memory for task-dependency graph (table).");
-
- int *count_rel = (int *)malloc(nber_relation * sizeof(int) / 2);
- if (count_rel == NULL)
- error("Error allocating memory for task-dependency graph (count_rel).");
-
- /* Reset everything */
- for (int i = 0; i < nber_relation; i++) table[i] = -1;
- for (int i = 0; i < nber_relation / 2; i++) count_rel[i] = 0;
+ struct task_dependency *task_dep = (struct task_dependency *)malloc(
+ nber_tasks * sizeof(struct task_dependency));
- /* Create file */
- char filename[200] = "dependency_graph.dot";
- FILE *f = fopen(filename, "w");
- if (f == NULL) error("Error opening dependency graph file.");
+ if (task_dep == NULL)
+ error("Error allocating memory for task-dependency graph (table).");
- /* Write header */
- fprintf(f, "digraph task_dep {\n");
- fprintf(f, "label=\"Task dependencies for SWIFT %s\";\n", git_revision());
- fprintf(f, "\t compound=true;\n");
- fprintf(f, "\t ratio=0.66;\n");
- fprintf(f, "\t node[nodesep=0.15];\n");
+ /* Reset counter */
+ for (int i = 0; i < nber_tasks; i++) {
+ for (int j = 0; j < MAX_NUMBER_DEP; j++) {
+ /* Use number_link as indicator of the existance of a relation */
+ task_dep[i].number_link[j] = -1;
+ }
+ }
/* loop over all tasks */
for (int i = 0; i < s->nr_tasks; i++) {
const struct task *ta = &s->tasks[i];
+ /* Current index */
+ const int ind = ta->type * task_subtype_count + ta->subtype;
+
+ struct task_dependency *cur = &task_dep[ind];
+
+ /* Set ta */
+ cur->type_in = ta->type;
+ cur->subtype_in = ta->subtype;
+ cur->implicit_in = ta->implicit;
+
/* and their dependencies */
for (int j = 0; j < ta->nr_unlock_tasks; j++) {
const struct task *tb = ta->unlock_tasks[j];
- /* check if dependency already written */
- int written = 0;
-
- /* Current index */
- int ind = ta->type * task_subtype_count + ta->subtype;
- ind *= 2 * max_nber_dep;
-
int k = 0;
- int *cur = &table[ind];
- while (k < max_nber_dep) {
+ while (k < MAX_NUMBER_DEP) {
/* not written yet */
- if (cur[0] == -1) {
- cur[0] = tb->type;
- cur[1] = tb->subtype;
+ if (cur->number_link[k] == -1) {
+ /* set tb */
+ cur->type_out[k] = tb->type;
+ cur->subtype_out[k] = tb->subtype;
+ cur->implicit_out[k] = tb->implicit;
+
+ /* statistics */
+ const int count = scheduler_get_number_relation(s, ta, tb);
+ cur->number_link[k] = count;
+ cur->number_rank[k] = 1;
+
break;
}
/* already written */
- if (cur[0] == tb->type && cur[1] == tb->subtype) {
- written = 1;
+ if (cur->type_out[k] == tb->type &&
+ cur->subtype_out[k] == tb->subtype) {
break;
}
k += 1;
- cur = &cur[2];
}
- /* max_nber_dep is too small */
- if (k == max_nber_dep)
- error("Not enough memory, please increase max_nber_dep");
+ /* MAX_NUMBER_DEP is too small */
+ if (k == MAX_NUMBER_DEP)
+ error("Not enough memory, please increase MAX_NUMBER_DEP");
+ }
+ }
- /* Increase counter of relation */
- count_rel[ind / 2 + k] += 1;
+#ifdef WITH_MPI
+ /* create MPI operator */
+ MPI_Datatype data_type;
+ task_dependency_define(&data_type);
- /* Not written yet => write it */
- if (!written) {
+ MPI_Op sum;
+ MPI_Op_create(task_dependency_sum, /* commute */ 1, &sum);
- /* text to write */
- char ta_name[200];
- char tb_name[200];
+ /* create recv buffer */
+ struct task_dependency *recv = NULL;
- /* construct line */
- scheduler_task_dependency_name(ta->type, ta->subtype, ta_name);
- scheduler_task_dependency_name(tb->type, tb->subtype, tb_name);
-
- /* Change colour of implicit tasks */
- if (ta->implicit)
- fprintf(f, "\t %s [style = filled];\n\t %s [color = lightgrey];\n",
- ta_name, ta_name);
- if (tb->implicit)
- fprintf(f, "\t %s [style = filled];\n\t %s [color = lightgrey];\n",
- tb_name, tb_name);
-
- /* Change shape of MPI communications */
- if (ta->type == task_type_send || ta->type == task_type_recv)
- fprintf(f, "\t \"%s %s\" [shape = diamond];\n",
- taskID_names[ta->type], subtaskID_names[ta->subtype]);
- if (tb->type == task_type_send || tb->type == task_type_recv)
- fprintf(f, "\t \"%s %s\" [shape = diamond];\n",
- taskID_names[tb->type], subtaskID_names[tb->subtype]);
+ if (s->nodeID == 0) {
+ recv = (struct task_dependency *)malloc(nber_tasks *
+ sizeof(struct task_dependency));
+
+ /* reset counter */
+ for (int i = 0; i < nber_tasks; i++) {
+ for (int j = 0; j < MAX_NUMBER_DEP; j++) {
+ /* Use number_link as indicator of the existance of a relation */
+ recv[i].number_link[j] = -1;
}
}
}
- int density_cluster[4] = {0};
- int gradient_cluster[4] = {0};
- int force_cluster[4] = {0};
- int limiter_cluster[4] = {0};
- int gravity_cluster[5] = {0};
- int stars_density_cluster[4] = {0};
-
- /* Check whether we need to construct a group of tasks */
- for (int type = 0; type < task_type_count; ++type) {
-
- for (int subtype = 0; subtype < task_subtype_count; ++subtype) {
-
- const int ind = 2 * (type * task_subtype_count + subtype) * max_nber_dep;
-
- /* Does this task/sub-task exist? */
- if (table[ind] != -1) {
-
- for (int k = 0; k < 4; ++k) {
- if (type == task_type_self + k && subtype == task_subtype_density)
- density_cluster[k] = 1;
- if (type == task_type_self + k && subtype == task_subtype_gradient)
- gradient_cluster[k] = 1;
- if (type == task_type_self + k && subtype == task_subtype_force)
- force_cluster[k] = 1;
- if (type == task_type_self + k && subtype == task_subtype_limiter)
- limiter_cluster[k] = 1;
- if (type == task_type_self + k && subtype == task_subtype_grav)
- gravity_cluster[k] = 1;
- if (type == task_type_self + k &&
- subtype == task_subtype_stars_density)
- stars_density_cluster[k] = 1;
- }
- if (type == task_type_grav_mesh) gravity_cluster[2] = 1;
- if (type == task_type_grav_long_range) gravity_cluster[3] = 1;
- if (type == task_type_grav_mm) gravity_cluster[4] = 1;
- }
- }
+ /* Do the reduction */
+ int test =
+ MPI_Reduce(task_dep, recv, nber_tasks, data_type, sum, 0, MPI_COMM_WORLD);
+ if (test != MPI_SUCCESS) error("MPI reduce failed");
+
+ /* free some memory */
+ if (s->nodeID == 0) {
+ free(task_dep);
+ task_dep = recv;
}
+#endif
- /* Make a cluster for the density tasks */
- fprintf(f, "\t subgraph cluster0{\n");
- fprintf(f, "\t\t label=\"\";\n");
- for (int k = 0; k < 4; ++k)
- if (density_cluster[k])
- fprintf(f, "\t\t \"%s %s\";\n", taskID_names[task_type_self + k],
- subtaskID_names[task_subtype_density]);
- fprintf(f, "\t};\n");
-
- /* Make a cluster for the force tasks */
- fprintf(f, "\t subgraph cluster1{\n");
- fprintf(f, "\t\t label=\"\";\n");
- for (int k = 0; k < 4; ++k)
- if (force_cluster[k])
- fprintf(f, "\t\t \"%s %s\";\n", taskID_names[task_type_self + k],
- subtaskID_names[task_subtype_force]);
- fprintf(f, "\t};\n");
-
- /* Make a cluster for the gradient tasks */
- fprintf(f, "\t subgraph cluster2{\n");
- fprintf(f, "\t\t label=\"\";\n");
- for (int k = 0; k < 4; ++k)
- if (gradient_cluster[k])
- fprintf(f, "\t\t \"%s %s\";\n", taskID_names[task_type_self + k],
- subtaskID_names[task_subtype_gradient]);
- fprintf(f, "\t};\n");
-
- /* Make a cluster for the limiter tasks */
- fprintf(f, "\t subgraph cluster2{\n");
- fprintf(f, "\t\t label=\"\";\n");
- for (int k = 0; k < 4; ++k)
- if (limiter_cluster[k])
- fprintf(f, "\t\t \"%s %s\";\n", taskID_names[task_type_self + k],
- subtaskID_names[task_subtype_limiter]);
- fprintf(f, "\t};\n");
-
- /* Make a cluster for the gravity tasks */
- fprintf(f, "\t subgraph cluster4{\n");
- fprintf(f, "\t\t label=\"\";\n");
- for (int k = 0; k < 2; ++k)
- if (gravity_cluster[k])
- fprintf(f, "\t\t \"%s %s\";\n", taskID_names[task_type_self + k],
- subtaskID_names[task_subtype_grav]);
- if (gravity_cluster[2])
- fprintf(f, "\t\t %s;\n", taskID_names[task_type_grav_mesh]);
- if (gravity_cluster[3])
- fprintf(f, "\t\t %s;\n", taskID_names[task_type_grav_long_range]);
- if (gravity_cluster[4])
- fprintf(f, "\t\t %s;\n", taskID_names[task_type_grav_mm]);
- fprintf(f, "\t};\n");
-
- /* Make a cluster for the density tasks */
- fprintf(f, "\t subgraph cluster4{\n");
- fprintf(f, "\t\t label=\"\";\n");
- for (int k = 0; k < 4; ++k)
- if (stars_density_cluster[k])
- fprintf(f, "\t\t \"%s %s\";\n", taskID_names[task_type_self + k],
- subtaskID_names[task_subtype_stars_density]);
- fprintf(f, "\t};\n");
-
- /* Write down the number of relation */
- for (int ta_type = 0; ta_type < task_type_count; ta_type++) {
-
- for (int ta_subtype = 0; ta_subtype < task_subtype_count; ta_subtype++) {
-
- /* Get task indice */
- const int ind =
- (ta_type * task_subtype_count + ta_subtype) * max_nber_dep;
-
- /* Loop over dependencies */
- for (int k = 0; k < max_nber_dep; k++) {
-
- if (count_rel[ind + k] == 0) continue;
-
- /* Get task type */
- const int i = 2 * (ind + k);
- int tb_type = table[i];
- int tb_subtype = table[i + 1];
-
- /* Get names */
+ if (s->nodeID == 0) {
+ /* Create file */
+ char *filename = "dependency_graph.csv";
+ FILE *f = fopen(filename, "w");
+ if (f == NULL) error("Error opening dependency graph file.");
+
+ /* Write header */
+ fprintf(f, "# %s\n", git_revision());
+ fprintf(
+ f,
+ "task_in,task_out,implicit_in,implicit_out,mpi_in,mpi_out,cluster_in,"
+ "cluster_out,number_link,number_rank\n");
+
+ for (int i = 0; i < nber_tasks; i++) {
+ for (int j = 0; j < MAX_NUMBER_DEP; j++) {
+ /* Does this link exists */
+ if (task_dep[i].number_link[j] == -1) {
+ continue;
+ }
+
+ /* Define a few variables */
+ const int ta_type = task_dep[i].type_in;
+ const int ta_subtype = task_dep[i].subtype_in;
+ const int ta_implicit = task_dep[i].implicit_in;
+
+ const int tb_type = task_dep[i].type_out[j];
+ const int tb_subtype = task_dep[i].subtype_out[j];
+ const int tb_implicit = task_dep[i].implicit_out[j];
+
+ const int count = task_dep[i].number_link[j];
+ const int number_rank = task_dep[i].number_rank[j];
+
+ /* text to write */
char ta_name[200];
char tb_name[200];
- scheduler_task_dependency_name(ta_type, ta_subtype, ta_name);
- scheduler_task_dependency_name(tb_type, tb_subtype, tb_name);
+ /* construct line */
+ task_get_full_name(ta_type, ta_subtype, ta_name);
+ task_get_full_name(tb_type, tb_subtype, tb_name);
+
+ /* Check if MPI */
+ int ta_mpi = 0;
+ if (ta_type == task_type_send || ta_type == task_type_recv) ta_mpi = 1;
+
+ int tb_mpi = 0;
+ if (tb_type == task_type_send || tb_type == task_type_recv) tb_mpi = 1;
- /* Write to the fle */
- fprintf(f, "\t %s->%s[label=%i];\n", ta_name, tb_name,
- count_rel[ind + k]);
+ /* Get group name */
+ char ta_cluster[20];
+ char tb_cluster[20];
+ task_get_group_name(ta_type, ta_subtype, ta_cluster);
+ task_get_group_name(tb_type, tb_subtype, tb_cluster);
+
+ fprintf(f, "%s,%s,%d,%d,%d,%d,%s,%s,%d,%d\n", ta_name, tb_name,
+ ta_implicit, tb_implicit, ta_mpi, tb_mpi, ta_cluster,
+ tb_cluster, count, number_rank);
}
}
+ /* Close the file */
+ fclose(f);
}
- /* Close the file */
- fprintf(f, "}");
- fclose(f);
-
/* Be clean */
- free(table);
- free(count_rel);
+ free(task_dep);
if (verbose)
message("Printing task graph took %.3f %s.",
@@ -1960,9 +2072,6 @@ void scheduler_reweight(struct scheduler *s, int verbose) {
case task_type_timestep:
cost = wscale * count_i + wscale * gcount_i;
break;
- case task_type_timestep_limiter:
- cost = wscale * count_i;
- break;
case task_type_send:
#if defined(WITH_MPI) && (defined(HAVE_PARMETIS) || defined(HAVE_METIS))
partcost = 0;
@@ -2172,8 +2281,7 @@ void scheduler_enqueue(struct scheduler *s, struct task *t) {
subtaskMPI_comms[t->subtype], &t->req);
} else if (t->subtype == task_subtype_xv ||
t->subtype == task_subtype_rho ||
- t->subtype == task_subtype_gradient ||
- t->subtype == task_subtype_limiter) {
+ t->subtype == task_subtype_gradient) {
err = MPI_Irecv(t->ci->hydro.parts, t->ci->hydro.count, part_mpi_type,
t->ci->nodeID, t->flags, subtaskMPI_comms[t->subtype],
&t->req);
@@ -2224,8 +2332,7 @@ void scheduler_enqueue(struct scheduler *s, struct task *t) {
subtaskMPI_comms[t->subtype], &t->req);
} else if (t->subtype == task_subtype_xv ||
t->subtype == task_subtype_rho ||
- t->subtype == task_subtype_gradient ||
- t->subtype == task_subtype_limiter) {
+ t->subtype == task_subtype_gradient) {
if ((t->ci->hydro.count * sizeof(struct part)) > s->mpi_message_limit)
err = MPI_Isend(t->ci->hydro.parts, t->ci->hydro.count,
part_mpi_type, t->cj->nodeID, t->flags,
diff --git a/src/task.c b/src/task.c
index f27b5fd413c1c216afb7d777244b0cee88159e6e..4aed4db3abf183143f3836651b6a37bc6b8e2fdc 100644
--- a/src/task.c
+++ b/src/task.c
@@ -628,6 +628,71 @@ void task_print(const struct task *t) {
t->nr_unlock_tasks, t->skip);
}
+/**
+ * @brief Get the group name of a task.
+ *
+ * This is used to group tasks with similar actions in the task dependency
+ * graph.
+ *
+ * @param type The #task type.
+ * @param subtype The #subtask type.
+ * @param cluster (return) The group name (should be allocated)
+ */
+void task_get_group_name(int type, int subtype, char *cluster) {
+
+ if (type == task_type_grav_long_range || type == task_type_grav_mm ||
+ type == task_type_grav_mesh) {
+
+ strcpy(cluster, "Gravity");
+ return;
+ }
+
+ switch (subtype) {
+ case task_subtype_density:
+ strcpy(cluster, "Density");
+ break;
+ case task_subtype_gradient:
+ strcpy(cluster, "Gradient");
+ break;
+ case task_subtype_force:
+ strcpy(cluster, "Force");
+ break;
+ case task_subtype_grav:
+ strcpy(cluster, "Gravity");
+ break;
+ case task_subtype_stars_density:
+ strcpy(cluster, "Stars");
+ break;
+ default:
+ strcpy(cluster, "None");
+ break;
+ }
+}
+
+/**
+ * @brief Generate the full name of a #task.
+ *
+ * @param type The #task type.
+ * @param subtype The #task type.
+ * @param name (return) The formatted string
+ */
+void task_get_full_name(int type, int subtype, char *name) {
+
+#ifdef SWIFT_DEBUG_CHECKS
+ /* Check input */
+ if (type >= task_type_count) error("Unknown task type %i", type);
+
+ if (subtype >= task_subtype_count)
+ error("Unknown task subtype %i with type %s", subtype, taskID_names[type]);
+#endif
+
+ /* Full task name */
+ if (subtype == task_subtype_none)
+ sprintf(name, "%s", taskID_names[type]);
+ else
+ sprintf(name, "%s_%s", taskID_names[type], subtaskID_names[subtype]);
+}
+
#ifdef WITH_MPI
/**
* @brief Create global communicators for each of the subtasks.
diff --git a/src/task.h b/src/task.h
index 83c71a73e23da8d25c99bf038045cde12aa64c89..100ac225bd5956e8d59d6a197c1257cb3e796ebb 100644
--- a/src/task.h
+++ b/src/task.h
@@ -205,6 +205,9 @@ float task_overlap(const struct task *ta, const struct task *tb);
int task_lock(struct task *t);
void task_do_rewait(struct task *t);
void task_print(const struct task *t);
+void task_get_full_name(int type, int subtype, char *name);
+void task_get_group_name(int type, int subtype, char *cluster);
+
#ifdef WITH_MPI
void task_create_mpi_comms(void);
#endif
diff --git a/tools/plot_task_dependencies.py b/tools/plot_task_dependencies.py
new file mode 100644
index 0000000000000000000000000000000000000000..fd08f34487daee222786b7c4806200b670b31a7d
--- /dev/null
+++ b/tools/plot_task_dependencies.py
@@ -0,0 +1,359 @@
+#!/usr/bin/env python3
+"""
+This file generates a graphviz file that represents the SWIFT tasks
+ dependencies.
+
+Example: ./plot_task_dependencies.py dependency_graph_*.csv
+"""
+import sys
+from pandas import read_csv
+import numpy as np
+from subprocess import call
+
+
+def getGitVersion(f, git):
+ """
+ Read the git version from the file
+
+ Parameters
+ ----------
+
+ f: str
+ Filename
+
+ git: str
+ Git version of previous file
+
+ Returns
+ -------
+
+ new_git: str
+ Git version of current file
+ """
+ # read comment in csv file
+ with open(f, "r") as f:
+ line = f.readline()
+
+ # check if really a comment
+ if line[0] != "#":
+ return None
+
+ # remove trailing characters
+ new_git = line[2:].rstrip()
+
+ # check if previous and current are the same
+ if git is not None and git != new_git:
+ raise Exception("Files were not produced by the same version")
+
+ return new_git
+
+
+def appendSingleData(data0, datai):
+ """
+ Append two DataFrame together
+
+ Parameters
+ ----------
+
+ data0: DataFrame
+ One of the dataframe
+
+ datai: DataFrame
+ The second dataframe
+
+ Returns
+ -------
+
+ data0: DataFrame
+ The updated dataframe
+ """
+
+ # loop over all rows in datai
+ for i, row in datai.iterrows():
+ # get data
+ ta = datai["task_in"][i]
+ tb = datai["task_out"][i]
+ ind = np.logical_and(data0["task_in"] == ta,
+ data0["task_out"] == tb)
+
+ # check number of ta->tb
+ N = np.sum(ind)
+ if N > 1:
+ raise Exception("Same dependency written multiple times %s->%s" %
+ (ta, tb))
+ # if not present in data0
+ if N == 0:
+ data0.append(row)
+ else:
+ # otherwise just update the number of link
+ ind = ind[ind].index[0]
+ tmp = data0["number_link"][ind] + datai["number_link"][i]
+ data0.at[ind, "number_link"] = tmp
+
+ return data0
+
+
+def appendData(data):
+ """
+ Append all the dataframe together
+
+ Parameters
+ ----------
+
+ data: list
+ List containing all the dataframe to append together
+
+ Returns
+ -------
+
+ data: DataFrame
+ The complete dataframe
+ """
+ N = len(data)
+ if N == 1:
+ return data[0]
+
+ # add number link to data[0]
+ for i in range(N-1):
+ i += 1
+ data[0] = appendSingleData(data[0], data[i])
+
+ return data[0]
+
+
+def writeTask(f, name, implicit, mpi):
+ """
+ Write the special task (e.g. implicit and mpi)
+
+ Parameters
+ ----------
+
+ f: File
+ File where to write the data
+
+ name: str
+ Task name
+
+ implicit: int
+ Is the task implicit
+
+ mpi: int
+ Is the task MPI related
+ """
+ # do we need to do something?
+ if not implicit and not mpi:
+ return
+
+ # generate text
+ txt = "\t " + name + "["
+ if implicit:
+ txt += "style=filled, color=lightgrey"
+ if mpi:
+ txt += ","
+ if mpi:
+ txt += "shape=diamond"
+ txt += "];\n"
+
+ # write it
+ f.write(txt)
+
+
+def writeHeader(f, data, git):
+ """
+ Write the header and the special tasks
+
+ Parameters
+ ----------
+
+ f: File
+ File where to write the data
+
+ data: DataFrame
+ The dataframe to write
+
+ git: str
+ The git version
+ """
+ # write header
+ f.write("digraph task_dep {\n")
+ f.write("\t # Header\n")
+ f.write('\t label="Task dependencies for SWIFT %s";\n' % git)
+ f.write("\t compound=true;\n")
+ f.write("\t ratio=0.66;\n")
+ f.write("\t node[nodesep=0.15];\n")
+
+ f.write("\n")
+
+ # write the special task
+ f.write("\t # Special tasks\n")
+ N = len(data)
+ written = []
+ # do task in
+ for i in range(N):
+ ta = data["task_in"][i]
+ if ta in written:
+ continue
+
+ written.append(ta)
+ writeTask(f, ta, data["implicit_in"][i], data["mpi_in"][i])
+
+ # do task out
+ for i in range(N):
+ tb = data["task_out"][i]
+ if tb in written:
+ continue
+
+ written.append(tb)
+ writeTask(f, tb, data["implicit_out"][i], data["mpi_out"][i])
+
+ f.write("\n")
+
+
+def writeCluster(f, tasks, cluster):
+ """
+ Write a single cluster
+
+ Parameters
+ ----------
+
+ f: File
+ File where to write the data
+
+ tasks: list
+ List of all tasks in the cluster
+
+ cluster: str
+ Cluster name
+ """
+ f.write("\t subgraph cluster%s {\n" % cluster)
+ f.write('\t\t label="";\n')
+ for t in tasks:
+ f.write("\t\t %s;\n" % t)
+ f.write("\t };\n\n")
+
+
+def writeClusters(f, data):
+ """
+ Write all the clusters
+
+ Parameters
+ ----------
+
+ f: File
+ File where to write the data
+
+ data: DataFrame
+ The dataframe to write
+ """
+ f.write("\t # Clusters\n")
+ # get list of all the clusters
+ clusters = data[["cluster_in", "cluster_out"]]
+ clusters = np.unique(clusters)
+
+ cluster_in = data["cluster_in"]
+ cluster_out = data["cluster_out"]
+ # loop over all clusters
+ for cluster in clusters:
+ # is it a cluster?
+ if cluster == "None":
+ continue
+
+ # get all the task in current cluster
+ ta = data["task_in"][cluster_in == cluster]
+ tb = data["task_out"][cluster_out == cluster]
+
+ # make them unique
+ tasks = np.append(ta, tb)
+ tasks = np.unique(tasks)
+
+ # write current cluster
+ writeCluster(f, tasks, cluster)
+
+ f.write("\n")
+
+
+def writeDependencies(f, data):
+ """
+ Write all the dependencies between tasks
+
+ Parameters
+ ----------
+
+ f: File
+ File where to write the data
+
+ data: DataFrame
+ The dataframe to write
+
+ """
+ f.write("\t # Dependencies\n")
+ N = len(data)
+ written = []
+ max_rank = data["number_rank"].max()
+ for i in range(N):
+ # get data
+ ta = data["task_in"][i]
+ tb = data["task_out"][i]
+ number_link = data["number_link"][i]
+
+ # check if already done
+ name = "%s_%s" % (ta, tb)
+ if name in written:
+ raise Exception("Found two same task dependencies")
+
+ written.append(name)
+
+ # write relation
+ arrow = ""
+ if data["number_rank"][i] != max_rank:
+ arrow = ",style=dashed"
+ f.write("\t %s->%s[label=%i%s]\n" %
+ (ta, tb, number_link, arrow))
+
+
+def writeFooter(f):
+ """
+ Write the footer
+
+ Parameters
+ ----------
+
+ f: File
+ File where to write the data
+ """
+ f.write("}")
+
+
+if __name__ == "__main__":
+ # get input
+ filenames = sys.argv[1:]
+ if len(filenames) < 1:
+ raise Exception("You should provide at least a file name")
+
+ # output
+ dot_output = "dependency_graph.dot"
+ png_output = "dependency_graph.png"
+
+ # read files
+ data = []
+ git = None
+ for f in filenames:
+ tmp = read_csv(f, delimiter=",", comment="#")
+ git = getGitVersion(f, git)
+ data.append(tmp)
+
+ data = appendData(data)
+
+ # write output
+ with open(dot_output, "w") as f:
+ writeHeader(f, data, git)
+
+ writeClusters(f, data)
+
+ writeDependencies(f, data)
+
+ writeFooter(f)
+
+ call(["dot", "-Tpng", dot_output, "-o", png_output])
+
+ print("You will find the graph in %s" % png_output)
diff --git a/tools/plot_task_dependencies.sh b/tools/plot_task_dependencies.sh
deleted file mode 100755
index 77784d8a9cdd3720621c9ad35c4cfbdaf0167ff1..0000000000000000000000000000000000000000
--- a/tools/plot_task_dependencies.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/bash
-
-# Creates a graphic from the task graph file dependency_graph.dot.
-# Requires the graphviz command "dot".
-
-if [ ! -e dependency_graph.dot ]; then
- echo "Missing task-graph output 'dependency_graph.dot'! Cannot generate figure."
-else
- dot -Tpng dependency_graph.dot -o task_graph.png
- echo "Output written to task_graph.png"
-fi
-
-exit