add a root task for all send/recv tasks.

src/engine.c

@@ -589,6 +589,9 @@ void engine_addtasks_send(struct engine *e, struct cell *ci, struct cell *cj) {
     struct task *t_rho = scheduler_addtask(s, task_type_send, task_subtype_none,
                                            2 * ci->tag + 1, 0, ci, cj, 0);
 
+    /* The first send should depend on the engine's send_root. */
+    scheduler_addunlock(s, e->send_root, t_xv);
+
     /* The send_rho task depends on the cell's ghost task. */
     scheduler_addunlock(s, ci->super->ghost, t_rho);
 
@@ -633,6 +636,9 @@ void engine_addtasks_recv(struct engine *e, struct cell *c, struct task *t_xv,
                                           2 * c->tag, 0, c, NULL, 0);
     t_rho = c->recv_rho = scheduler_addtask(
         s, task_type_recv, task_subtype_none, 2 * c->tag + 1, 0, c, NULL, 0);
+
+    /* The first recv should depend on the engine's recv_root. */
+    scheduler_addunlock(s, e->recv_root, t_xv);
   }
 
   /* Add dependencies. */
@@ -1414,6 +1420,12 @@ void engine_maketasks(struct engine *e) {
   /* Add the communication tasks if MPI is being used. */
   if ((e->policy & engine_policy_mpi) == engine_policy_mpi) {
 
+    /* Create root tasks for send/recv. */
+    e->send_root = scheduler_addtask(&e->sched, task_type_comm_root,
+                                     task_subtype_none, 0, 0, NULL, NULL, 0);
+    e->recv_root = scheduler_addtask(&e->sched, task_type_comm_root,
+                                     task_subtype_none, 0, 0, NULL, NULL, 0);
+
     /* Loop over the proxies. */
     for (int pid = 0; pid < e->nr_proxies; pid++) {
 

src/engine.h

@@ -162,6 +162,9 @@ struct engine {
   struct link *links;
   int nr_links, size_links;
 
+  /* Root task for all send and recv tasks. */
+  struct task *send_root, *recv_root;
+
   /* Are we talkative ? */
   int verbose;
 };

src/runner.c

@@ -1212,6 +1212,8 @@ void *runner_main(void *data) {
           scheduler_do_rewait((struct task *)t->ci, (struct task *)t->cj,
                               t->flags, t->rank);
           break;
+        case task_type_comm_root:
+          break;
         default:
           error("Unknown task type.");
       }

src/scheduler.c

@@ -1066,7 +1066,6 @@ void scheduler_enqueue(struct scheduler *s, struct task *t) {
   if (t->implicit) {
     for (int j = 0; j < t->nr_unlock_tasks; j++) {
       struct task *t2 = t->unlock_tasks[j];
-
       if (atomic_dec(&t2->wait) == 1) scheduler_enqueue(s, t2);
     }
   }

src/task.c

@@ -46,7 +46,7 @@ const char *taskID_names[task_type_count] = {
     "none",      "sort",       "self",       "pair",    "sub",
     "init",      "ghost",      "drift",      "kick",    "send",
     "recv",      "grav_pp",    "grav_mm",    "grav_up", "grav_down",
-    "part_sort", "gpart_sort", "split_cell", "rewait"};
+    "part_sort", "gpart_sort", "split_cell", "rewait",  "comm_root"};
 
 const char *subtaskID_names[task_type_count] = {"none",  "density",
                                                 "force", "grav"};

src/task.h

@@ -49,6 +49,7 @@ enum task_types {
   task_type_gpart_sort,
   task_type_split_cell,
   task_type_rewait,
+  task_type_comm_root,
   task_type_count
 };