Merge branch 'RT_MPI_fix' into 'master'

Re-organised the creation of the RT-specific time-step tasks on foreign cells to avoid duplicates

See merge request !1685

src/engine_maketasks.c

@@ -556,8 +556,6 @@ void engine_addtasks_send_black_holes(struct engine *e, struct cell *ci,
  * created.
  * @param t_rt_transport The recv_rt_transport #task, if it has already been
  * created.
- * @param t_rt_advance_cell_time The rt_advance_cell_time #task, if it has
- * already been created
  * @param t_rt_sorts The rt_sort #task, if it has already been created.
  * @param tend The top-level time-step communication #task.
  * @param with_feedback Are we running with stellar feedback?
@@ -570,8 +568,8 @@ void engine_addtasks_recv_hydro(
     struct engine *e, struct cell *c, struct task *t_xv, struct task *t_rho,
     struct task *t_gradient, struct task *t_prep1, struct task *t_limiter,
     struct task *t_unpack_limiter, struct task *t_rt_gradient,
-    struct task *t_rt_transport, struct task *t_rt_advance_cell_time,
-    struct task *t_rt_sorts, struct task *const tend, const int with_feedback,
+    struct task *t_rt_transport, struct task *t_rt_sorts,
+    struct task *const tend, const int with_feedback,
     const int with_black_holes, const int with_limiter, const int with_sync,
     const int with_rt) {
 
@@ -632,20 +630,34 @@ void engine_addtasks_recv_hydro(
       if (c->super == NULL)
         error("trying to add rt_advance_cell_time above super level...");
 #endif
-      t_rt_advance_cell_time =
-          scheduler_addtask(s, task_type_rt_advance_cell_time,
-                            task_subtype_none, 0, 0, c->super, NULL);
 
-      c->super->rt.rt_advance_cell_time = t_rt_advance_cell_time;
       /* Create the RT collect times task at the top level, if it hasn't
        * already. */
-      if (c->top->rt.rt_collect_times == NULL)
+      if (c->top->rt.rt_collect_times == NULL) {
         c->top->rt.rt_collect_times =
             scheduler_addtask(s, task_type_rt_collect_times, task_subtype_none,
                               0, 0, c->top, NULL);
-      /* Don't run collect times before you run advance cell time */
-      scheduler_addunlock(s, t_rt_advance_cell_time,
-                          c->top->rt.rt_collect_times);
+      }
+
+      /* Create the RT advance times task at the super level, if it hasn't
+       * already. Also set all the dependencies */
+      if (c->super->rt.rt_advance_cell_time == NULL) {
+
+        c->super->rt.rt_advance_cell_time =
+            scheduler_addtask(s, task_type_rt_advance_cell_time,
+                              task_subtype_none, 0, 0, c->super, NULL);
+
+        /* Don't run collect times before you run advance cell time */
+        scheduler_addunlock(s, c->super->rt.rt_advance_cell_time,
+                            c->top->rt.rt_collect_times);
+
+        /* In normal steps, tend mustn't run before rt_advance_cell_time or the
+         * cell's ti_rt_end_min will be updated wrongly. In sub-cycles, we don't
+         * have the tend tasks, so there's no worry about that. (Them missing is
+         * the reason we need the rt_advanced_cell_time to complete the
+         * sub-cycles in the first place) */
+        scheduler_addunlock(s, c->super->rt.rt_advance_cell_time, tend);
+      }
 
       /* Make sure we sort after receiving RT data. The hydro sorts may or may
        * not be active. Blocking them with dependencies deadlocks with MPI. So
@@ -672,8 +684,8 @@ void engine_addtasks_recv_hydro(
       scheduler_addunlock(s, t_rt_sorts, t_rt_transport);
       /* If one or both recv tasks are active, make sure the
        * rt_advance_cell_time tasks doesn't run before them */
-      scheduler_addunlock(s, t_rt_gradient, t_rt_advance_cell_time);
-      scheduler_addunlock(s, t_rt_transport, t_rt_advance_cell_time);
+      scheduler_addunlock(s, t_rt_gradient, c->super->rt.rt_advance_cell_time);
+      scheduler_addunlock(s, t_rt_transport, c->super->rt.rt_advance_cell_time);
       /* Make sure the gradient recv don't run before the xv is finished.
        * This can occur when a cell itself is inactive for both hydro and
        * RT, but needs to be sent over for some other cell's pair task.
@@ -686,13 +698,6 @@ void engine_addtasks_recv_hydro(
 #ifdef EXTRA_HYDRO_LOOP
       scheduler_addunlock(s, t_gradient, t_rt_gradient);
 #endif
-
-      /* In normal steps, tend mustn't run before rt_advance_cell_time or the
-       * cell's ti_rt_end_min will be updated wrongly. In sub-cycles, we don't
-       * have the tend tasks, so there's no worry about that. (Them missing is
-       * the reason we need the rt_advanced_cell_time to complete the sub-cycles
-       * in the first place) */
-      scheduler_addunlock(s, t_rt_advance_cell_time, tend);
     }
   }
 
@@ -804,7 +809,7 @@ void engine_addtasks_recv_hydro(
          * sure we don't receive data before we're done with all the work. */
         scheduler_addunlock(s, l->t, tend);
         /* advance cell time mustn't run before transport is done */
-        scheduler_addunlock(s, l->t, t_rt_advance_cell_time);
+        scheduler_addunlock(s, l->t, c->super->rt.rt_advance_cell_time);
       }
     }
   }
@@ -815,9 +820,8 @@ void engine_addtasks_recv_hydro(
       if (c->progeny[k] != NULL)
         engine_addtasks_recv_hydro(
             e, c->progeny[k], t_xv, t_rho, t_gradient, t_prep1, t_limiter,
-            t_unpack_limiter, t_rt_gradient, t_rt_transport,
-            t_rt_advance_cell_time, t_rt_sorts, tend, with_feedback,
-            with_black_holes, with_limiter, with_sync, with_rt);
+            t_unpack_limiter, t_rt_gradient, t_rt_transport, t_rt_sorts, tend,
+            with_feedback, with_black_holes, with_limiter, with_sync, with_rt);
 
 #else
   error("SWIFT was not compiled with MPI support.");
@@ -4305,8 +4309,8 @@ void engine_addtasks_recv_mapper(void *map_data, int num_elements,
           e, ci, /*t_xv=*/NULL, /*t_rho=*/NULL, /*t_gradient=*/NULL,
           /*t_prep1=*/NULL, /*t_limiter=*/NULL, /*t_unpack_limiter=*/NULL,
           /*t_rt_gradient=*/NULL, /*t_rt_transport=*/NULL,
-          /*t_rt_advance_cell_time=*/NULL, /*t_rt_sorts=*/NULL, tend,
-          with_feedback, with_black_holes, with_limiter, with_sync, with_rt);
+          /*t_rt_sorts=*/NULL, tend, with_feedback, with_black_holes,
+          with_limiter, with_sync, with_rt);
     }
 
     /* Add the recv tasks for the cells in the proxy that have a stars