Add another communication sub-type for the gparts. Send the gparts when these tasks get enqueued.

This follows suit with the addition of task sub-types for the communications. I have added one to ship the gparts between nodes.

This also allows to send/recv the gparts instead of the parts at enqueuing time. Note that the gpart communications are currently unused since I never pushed the support for gravity over MPI to the master.

It also allows to only loop over the relevant bits we received and not over the whole particle zoo to construct the time information on the receiving side.

This merges into the new_timeline_mpi branch and not in master as it is yet another MPI bug/feature fix.

src/runner.c

@@ -1208,20 +1208,18 @@ void runner_do_end_force(struct runner *r, struct cell *c, int timer) {
 }
 
 /**
- * @brief Construct the cell properties from the received particles
+ * @brief Construct the cell properties from the received #part.
  *
  * @param r The runner thread.
  * @param c The cell.
  * @param timer Are we timing this ?
  */
-void runner_do_recv_cell(struct runner *r, struct cell *c, int timer) {
+void runner_do_recv_part(struct runner *r, struct cell *c, int timer) {
 
 #ifdef WITH_MPI
 
   const struct part *restrict parts = c->parts;
-  const struct gpart *restrict gparts = c->gparts;
   const size_t nr_parts = c->count;
-  const size_t nr_gparts = c->gcount;
   const integertime_t ti_current = r->e->ti_current;
 
   TIMER_TIC;
@@ -1246,34 +1244,98 @@ void runner_do_recv_cell(struct runner *r, struct cell *c, int timer) {
         error("Received un-drifted particle !");
 #endif
     }
+  }
+
+  /* Otherwise, recurse and collect. */
+  else {
+    for (int k = 0; k < 8; k++) {
+      if (c->progeny[k] != NULL) {
+        runner_do_recv_part(r, c->progeny[k], 0);
+        ti_end_min = min(ti_end_min, c->progeny[k]->ti_end_min);
+        ti_end_max = max(ti_end_max, c->progeny[k]->ti_end_max);
+        h_max = max(h_max, c->progeny[k]->h_max);
+      }
+    }
+  }
+
+#ifdef SWIFT_DEBUG_CHECKS
+  if (ti_end_min < ti_current)
+    error(
+        "Received a cell at an incorrect time c->ti_end_min=%lld, "
+        "e->ti_current=%lld.",
+        ti_end_min, ti_current);
+#endif
+
+  /* ... and store. */
+  c->ti_end_min = ti_end_min;
+  c->ti_end_max = ti_end_max;
+  c->ti_old = ti_current;
+  c->h_max = h_max;
+
+  if (timer) TIMER_TOC(timer_dorecv_part);
+
+#else
+  error("SWIFT was not compiled with MPI support.");
+#endif
+}
+
+/**
+ * @brief Construct the cell properties from the received #gpart.
+ *
+ * @param r The runner thread.
+ * @param c The cell.
+ * @param timer Are we timing this ?
+ */
+void runner_do_recv_gpart(struct runner *r, struct cell *c, int timer) {
+
+#ifdef WITH_MPI
+
+  const struct gpart *restrict gparts = c->gparts;
+  const size_t nr_gparts = c->gcount;
+  const integertime_t ti_current = r->e->ti_current;
+
+  TIMER_TIC;
+
+  integertime_t ti_end_min = max_nr_timesteps;
+  integertime_t ti_end_max = 0;
+
+  /* If this cell is a leaf, collect the particle data. */
+  if (!c->split) {
+
+    /* Collect everything... */
     for (size_t k = 0; k < nr_gparts; k++) {
       const integertime_t ti_end =
           get_integer_time_end(ti_current, gparts[k].time_bin);
       ti_end_min = min(ti_end_min, ti_end);
       ti_end_max = max(ti_end_max, ti_end);
     }
-
   }
 
   /* Otherwise, recurse and collect. */
   else {
     for (int k = 0; k < 8; k++) {
       if (c->progeny[k] != NULL) {
-        runner_do_recv_cell(r, c->progeny[k], 0);
+        runner_do_recv_gpart(r, c->progeny[k], 0);
         ti_end_min = min(ti_end_min, c->progeny[k]->ti_end_min);
         ti_end_max = max(ti_end_max, c->progeny[k]->ti_end_max);
-        h_max = max(h_max, c->progeny[k]->h_max);
       }
     }
   }
 
+#ifdef SWIFT_DEBUG_CHECKS
+  if (ti_end_min < ti_current)
+    error(
+        "Received a cell at an incorrect time c->ti_end_min=%lld, "
+        "e->ti_current=%lld.",
+        ti_end_min, ti_current);
+#endif
+
   /* ... and store. */
   c->ti_end_min = ti_end_min;
   c->ti_end_max = ti_end_max;
   c->ti_old = ti_current;
-  c->h_max = h_max;
 
-  if (timer) TIMER_TOC(timer_dorecv_cell);
+  if (timer) TIMER_TOC(timer_dorecv_gpart);
 
 #else
   error("SWIFT was not compiled with MPI support.");
@@ -1473,8 +1535,11 @@ void *runner_main(void *data) {
           if (t->subtype == task_subtype_tend) {
             cell_unpack_ti_ends(ci, t->buff);
             free(t->buff);
-          } else {
-            runner_do_recv_cell(r, ci, 1);
+          } else if (t->subtype == task_subtype_xv ||
+                     t->subtype == task_subtype_rho) {
+            runner_do_recv_part(r, ci, 1);
+          } else if (t->subtype == task_subtype_gpart) {
+            runner_do_recv_gpart(r, ci, 1);
           }
           break;
 #endif