Change the reduction in io_write_cell_offsets() to make sure all ranks have the full information.

src/common_io.c

@@ -689,91 +689,36 @@ void io_write_cell_offsets(hid_t h_grp, const int cdim[3], const double dim[3],
 #ifdef WITH_MPI
   /* Now, reduce all the arrays. Note that we use a bit-wise OR here. This
      is safe as we made sure only local cells have non-zero values. */
-  if (nodeID == 0) {
-    MPI_Reduce(MPI_IN_PLACE, count_part, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
-               0, MPI_COMM_WORLD);
-  } else {
-    MPI_Reduce(count_part, NULL, nr_cells, MPI_LONG_LONG_INT, MPI_BOR, 0,
-               MPI_COMM_WORLD);
-  }
-  if (nodeID == 0) {
-    MPI_Reduce(MPI_IN_PLACE, count_gpart, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
-               0, MPI_COMM_WORLD);
-  } else {
-    MPI_Reduce(count_gpart, NULL, nr_cells, MPI_LONG_LONG_INT, MPI_BOR, 0,
-               MPI_COMM_WORLD);
-  }
-  if (nodeID == 0) {
-    MPI_Reduce(MPI_IN_PLACE, count_background_gpart, nr_cells,
-               MPI_LONG_LONG_INT, MPI_BOR, 0, MPI_COMM_WORLD);
-  } else {
-    MPI_Reduce(count_background_gpart, NULL, nr_cells, MPI_LONG_LONG_INT,
-               MPI_BOR, 0, MPI_COMM_WORLD);
-  }
-  if (nodeID == 0) {
-    MPI_Reduce(MPI_IN_PLACE, count_spart, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
-               0, MPI_COMM_WORLD);
-  } else {
-    MPI_Reduce(count_spart, NULL, nr_cells, MPI_LONG_LONG_INT, MPI_BOR, 0,
-               MPI_COMM_WORLD);
-  }
-  if (nodeID == 0) {
-    MPI_Reduce(MPI_IN_PLACE, count_bpart, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
-               0, MPI_COMM_WORLD);
-  } else {
-    MPI_Reduce(count_bpart, NULL, nr_cells, MPI_LONG_LONG_INT, MPI_BOR, 0,
-               MPI_COMM_WORLD);
-  }
-
-  if (nodeID == 0) {
-    MPI_Reduce(MPI_IN_PLACE, offset_part, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
-               0, MPI_COMM_WORLD);
-  } else {
-    MPI_Reduce(offset_part, NULL, nr_cells, MPI_LONG_LONG_INT, MPI_BOR, 0,
-               MPI_COMM_WORLD);
-  }
-  if (nodeID == 0) {
-    MPI_Reduce(MPI_IN_PLACE, offset_gpart, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
-               0, MPI_COMM_WORLD);
-  } else {
-    MPI_Reduce(offset_gpart, NULL, nr_cells, MPI_LONG_LONG_INT, MPI_BOR, 0,
-               MPI_COMM_WORLD);
-  }
-  if (nodeID == 0) {
-    MPI_Reduce(MPI_IN_PLACE, offset_background_gpart, nr_cells,
-               MPI_LONG_LONG_INT, MPI_BOR, 0, MPI_COMM_WORLD);
-  } else {
-    MPI_Reduce(offset_background_gpart, NULL, nr_cells, MPI_LONG_LONG_INT,
-               MPI_BOR, 0, MPI_COMM_WORLD);
-  }
-  if (nodeID == 0) {
-    MPI_Reduce(MPI_IN_PLACE, offset_spart, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
-               0, MPI_COMM_WORLD);
-  } else {
-    MPI_Reduce(offset_spart, NULL, nr_cells, MPI_LONG_LONG_INT, MPI_BOR, 0,
-               MPI_COMM_WORLD);
-  }
-  if (nodeID == 0) {
-    MPI_Reduce(MPI_IN_PLACE, offset_bpart, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
-               0, MPI_COMM_WORLD);
-  } else {
-    MPI_Reduce(offset_bpart, NULL, nr_cells, MPI_LONG_LONG_INT, MPI_BOR, 0,
-               MPI_COMM_WORLD);
-  }
+  MPI_Allreduce(MPI_IN_PLACE, count_part, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
+                0, MPI_COMM_WORLD);
+  MPI_Allreduce(MPI_IN_PLACE, count_gpart, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
+                0, MPI_COMM_WORLD);
+  MPI_Allreduce(MPI_IN_PLACE, count_background_gpart, nr_cells,
+                MPI_LONG_LONG_INT, MPI_BOR, 0, MPI_COMM_WORLD);
+  MPI_Allreduce(MPI_IN_PLACE, count_spart, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
+                0, MPI_COMM_WORLD);
+  MPI_Allreduce(MPI_IN_PLACE, count_bpart, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
+                0, MPI_COMM_WORLD);
+
+  MPI_Allreduce(MPI_IN_PLACE, offset_part, nr_cells, MPI_LONG_LONG_INT, MPI_BOR,
+                0, MPI_COMM_WORLD);
+  MPI_Allreduce(MPI_IN_PLACE, offset_gpart, nr_cells, MPI_LONG_LONG_INT,
+                MPI_BOR, 0, MPI_COMM_WORLD);
+  MPI_Allreduce(MPI_IN_PLACE, offset_background_gpart, nr_cells,
+                MPI_LONG_LONG_INT, MPI_BOR, 0, MPI_COMM_WORLD);
+  MPI_Allreduce(MPI_IN_PLACE, offset_spart, nr_cells, MPI_LONG_LONG_INT,
+                MPI_BOR, 0, MPI_COMM_WORLD);
+  MPI_Allreduce(MPI_IN_PLACE, offset_bpart, nr_cells, MPI_LONG_LONG_INT,
+                MPI_BOR, 0, MPI_COMM_WORLD);
 
   /* For the centres we use a sum as MPI does not like bit-wise operations
      on floating point numbers */
-  if (nodeID == 0) {
-    MPI_Reduce(MPI_IN_PLACE, centres, 3 * nr_cells, MPI_DOUBLE, MPI_SUM, 0,
-               MPI_COMM_WORLD);
-  } else {
-    MPI_Reduce(centres, NULL, 3 * nr_cells, MPI_DOUBLE, MPI_SUM, 0,
-               MPI_COMM_WORLD);
-  }
+  MPI_Allreduce(MPI_IN_PLACE, centres, 3 * nr_cells, MPI_DOUBLE, MPI_SUM, 0,
+                MPI_COMM_WORLD);
 #endif
 
   /* When writing a single file, only rank 0 writes the meta-data */
-  if (!distributed && nodeID == 0) {
+  if ((distributed) || (!distributed && nodeID == 0)) {
 
     /* Unit conversion if necessary */
     const double factor = units_conversion_factor(

src/distributed_io.c

@@ -437,8 +437,8 @@ void write_output_distributed(struct engine* e, const char* baseName,
   /* Write the location of the particles in the arrays */
   io_write_cell_offsets(h_grp, e->s->cdim, e->s->dim, e->s->pos_dithering,
                         e->s->cells_top, e->s->nr_cells, e->s->width, mpi_rank,
-                        /*distributed=*/1, N_total, global_offsets, internal_units,
-                        snapshot_units);
+                        /*distributed=*/1, N_total, global_offsets,
+                        internal_units, snapshot_units);
   H5Gclose(h_grp);
 
   /* Tell the user if a conversion will be needed */