diff --git a/examples/main.c b/examples/main.c
index 6fc5b433719822558d531f4ed2691e7127139a79..6bbb2ec4da939286734ff01b21eafb1e88716f88 100644
--- a/examples/main.c
+++ b/examples/main.c
@@ -470,8 +470,8 @@ int main(int argc, char *argv[]) {
#ifdef WITH_MPI
if (with_mpole_reconstruction && nr_nodes > 1)
error("Cannot reconstruct m-poles every step over MPI (yet).");
- if (with_star_formation)
- error("Can't run with star formation over MPI (yet)");
+ if (with_star_formation && with_feedback)
+ error("Can't run with star formation and feedback over MPI (yet)");
if (with_limiter) error("Can't run with time-step limiter over MPI (yet)");
#endif
diff --git a/src/cell.c b/src/cell.c
index f8722eccc55f662ef20d7762292b9798c5e0a157..a44cac0009d82664fa2d9879b482c795c69ab099 100644
--- a/src/cell.c
+++ b/src/cell.c
@@ -4816,6 +4816,17 @@ void cell_reorder_extra_parts(struct cell *c, const ptrdiff_t parts_offset) {
parts[i].gpart->id_or_neg_offset = -(i + parts_offset);
}
}
+
+#ifdef SWIFT_DEBUG_CHECKS
+ for (int i = 0; i < c->hydro.count_total; ++i) {
+ if (parts[i].time_bin == time_bin_not_created && i < c->hydro.count) {
+ error("Extra particle before the end of the regular array");
+ }
+ if (parts[i].time_bin != time_bin_not_created && i >= c->hydro.count) {
+ error("Regular particle after the end of the regular array");
+ }
+ }
+#endif
}
/**
@@ -4862,6 +4873,17 @@ void cell_reorder_extra_sparts(struct cell *c, const ptrdiff_t sparts_offset) {
#endif
}
}
+
+#ifdef SWIFT_DEBUG_CHECKS
+ for (int i = 0; i < c->stars.count_total; ++i) {
+ if (sparts[i].time_bin == time_bin_not_created && i < c->stars.count) {
+ error("Extra particle before the end of the regular array");
+ }
+ if (sparts[i].time_bin != time_bin_not_created && i >= c->stars.count) {
+ error("Regular particle after the end of the regular array");
+ }
+ }
+#endif
}
/**
@@ -4907,6 +4929,17 @@ void cell_reorder_extra_gparts(struct cell *c, struct part *parts,
}
}
}
+
+#ifdef SWIFT_DEBUG_CHECKS
+ for (int i = 0; i < c->grav.count_total; ++i) {
+ if (gparts[i].time_bin == time_bin_not_created && i < c->grav.count) {
+ error("Extra particle before the end of the regular array");
+ }
+ if (gparts[i].time_bin != time_bin_not_created && i >= c->grav.count) {
+ error("Regular particle after the end of the regular array");
+ }
+ }
+#endif
}
/**
diff --git a/src/space.c b/src/space.c
index 4b568cf3e877f8504c3e7e166ae324b0f49bdefb..00badc4d90c3031df65c65a965a7a1a99d5f3fa8 100644
--- a/src/space.c
+++ b/src/space.c
@@ -680,14 +680,23 @@ void space_allocate_extras(struct space *s, int verbose) {
size_t size_gparts = s->size_gparts;
size_t size_sparts = s->size_sparts;
- int local_cells = 0;
- for (int i = 0; i < s->nr_cells; ++i)
- if (s->cells_top[i].nodeID == local_nodeID) local_cells++;
+ int *local_cells = (int *)malloc(sizeof(int) * s->nr_cells);
+ if (local_cells == NULL)
+ error("Failed to allocate list of local top-level cells");
+
+ /* List the local cells */
+ int nr_local_cells = 0;
+ for (int i = 0; i < s->nr_cells; ++i) {
+ if (s->cells_top[i].nodeID == local_nodeID) {
+ local_cells[nr_local_cells] = i;
+ ++nr_local_cells;
+ }
+ }
/* Number of extra particles we want for each type */
- const size_t expected_num_extra_parts = local_cells * space_extra_parts;
- const size_t expected_num_extra_gparts = local_cells * space_extra_gparts;
- const size_t expected_num_extra_sparts = local_cells * space_extra_sparts;
+ const size_t expected_num_extra_parts = nr_local_cells * space_extra_parts;
+ const size_t expected_num_extra_gparts = nr_local_cells * space_extra_gparts;
+ const size_t expected_num_extra_sparts = nr_local_cells * space_extra_sparts;
if (verbose) {
message("Currently have %zd/%zd/%zd real particles.", nr_actual_parts,
@@ -755,11 +764,10 @@ void space_allocate_extras(struct space *s, int verbose) {
s->gparts[i].id_or_neg_offset = -1;
}
- /* Put the spare particles in their correct cell */
-#ifdef WITH_MPI
- error("Need to do this correctly over MPI for only the local cells.");
-#endif
- int count_in_cell = 0, current_cell = 0;
+ /* Put the spare particles in their correct cell */
+ int local_cell_id = 0;
+ int current_cell = local_cells[local_cell_id];
+ int count_in_cell = 0;
size_t count_extra_gparts = 0;
for (size_t i = 0; i < nr_actual_gparts + expected_num_extra_gparts; ++i) {
@@ -781,7 +789,11 @@ void space_allocate_extras(struct space *s, int verbose) {
/* Once we have reached the number of extra gpart per cell, we move to the
* next */
if (count_in_cell == space_extra_gparts) {
- ++current_cell;
+ ++local_cell_id;
+
+ if (local_cell_id == nr_local_cells) break;
+
+ current_cell = local_cells[local_cell_id];
count_in_cell = 0;
}
}
@@ -844,11 +856,10 @@ void space_allocate_extras(struct space *s, int verbose) {
s->parts[i].id = -1;
}
- /* Put the spare particles in their correct cell */
-#ifdef WITH_MPI
- error("Need to do this correctly over MPI for only the local cells.");
-#endif
- int count_in_cell = 0, current_cell = 0;
+ /* Put the spare particles in their correct cell */
+ int local_cell_id = 0;
+ int current_cell = local_cells[local_cell_id];
+ int count_in_cell = 0;
size_t count_extra_parts = 0;
for (size_t i = 0; i < nr_actual_parts + expected_num_extra_parts; ++i) {
@@ -870,7 +881,11 @@ void space_allocate_extras(struct space *s, int verbose) {
/* Once we have reached the number of extra part per cell, we move to the
* next */
if (count_in_cell == space_extra_parts) {
- ++current_cell;
+ ++local_cell_id;
+
+ if (local_cell_id == nr_local_cells) break;
+
+ current_cell = local_cells[local_cell_id];
count_in_cell = 0;
}
}
@@ -923,11 +938,10 @@ void space_allocate_extras(struct space *s, int verbose) {
s->sparts[i].id = -42;
}
- /* Put the spare particles in their correct cell */
-#ifdef WITH_MPI
- error("Need to do this correctly over MPI for only the local cells.");
-#endif
- int count_in_cell = 0, current_cell = 0;
+ /* Put the spare particles in their correct cell */
+ int local_cell_id = 0;
+ int current_cell = local_cells[local_cell_id];
+ int count_in_cell = 0;
size_t count_extra_sparts = 0;
for (size_t i = 0; i < nr_actual_sparts + expected_num_extra_sparts; ++i) {
@@ -949,7 +963,11 @@ void space_allocate_extras(struct space *s, int verbose) {
/* Once we have reached the number of extra spart per cell, we move to the
* next */
if (count_in_cell == space_extra_sparts) {
- ++current_cell;
+ ++local_cell_id;
+
+ if (local_cell_id == nr_local_cells) break;
+
+ current_cell = local_cells[local_cell_id];
count_in_cell = 0;
}
}
@@ -1630,12 +1648,12 @@ void space_split(struct space *s, int verbose) {
void space_reorder_extra_parts_mapper(void *map_data, int num_cells,
void *extra_data) {
-
- struct cell *cells_top = (struct cell *)map_data;
+ int *local_cells = (int *)map_data;
struct space *s = (struct space *)extra_data;
+ struct cell *cells_top = s->cells_top;
for (int ind = 0; ind < num_cells; ind++) {
- struct cell *c = &cells_top[ind];
+ struct cell *c = &cells_top[local_cells[ind]];
cell_reorder_extra_parts(c, c->hydro.parts - s->parts);
}
}
@@ -1643,11 +1661,12 @@ void space_reorder_extra_parts_mapper(void *map_data, int num_cells,
void space_reorder_extra_gparts_mapper(void *map_data, int num_cells,
void *extra_data) {
- struct cell *cells_top = (struct cell *)map_data;
+ int *local_cells = (int *)map_data;
struct space *s = (struct space *)extra_data;
+ struct cell *cells_top = s->cells_top;
for (int ind = 0; ind < num_cells; ind++) {
- struct cell *c = &cells_top[ind];
+ struct cell *c = &cells_top[local_cells[ind]];
cell_reorder_extra_gparts(c, s->parts, s->sparts);
}
}
@@ -1655,11 +1674,12 @@ void space_reorder_extra_gparts_mapper(void *map_data, int num_cells,
void space_reorder_extra_sparts_mapper(void *map_data, int num_cells,
void *extra_data) {
- struct cell *cells_top = (struct cell *)map_data;
+ int *local_cells = (int *)map_data;
struct space *s = (struct space *)extra_data;
+ struct cell *cells_top = s->cells_top;
for (int ind = 0; ind < num_cells; ind++) {
- struct cell *c = &cells_top[ind];
+ struct cell *c = &cells_top[local_cells[ind]];
cell_reorder_extra_sparts(c, c->stars.parts - s->sparts);
}
}
@@ -1676,25 +1696,20 @@ void space_reorder_extra_sparts_mapper(void *map_data, int num_cells,
*/
void space_reorder_extras(struct space *s, int verbose) {
-#ifdef WITH_MPI
- if (space_extra_parts || space_extra_gparts || space_extra_sparts)
- error("Need an MPI-proof version of this.");
-#endif
-
/* Re-order the gas particles */
if (space_extra_parts)
threadpool_map(&s->e->threadpool, space_reorder_extra_parts_mapper,
- s->cells_top, s->nr_cells, sizeof(struct cell), 0, s);
+ s->local_cells_top, s->nr_local_cells, sizeof(int), 0, s);
/* Re-order the gravity particles */
if (space_extra_gparts)
threadpool_map(&s->e->threadpool, space_reorder_extra_gparts_mapper,
- s->cells_top, s->nr_cells, sizeof(struct cell), 0, s);
+ s->local_cells_top, s->nr_local_cells, sizeof(int), 0, s);
/* Re-order the star particles */
if (space_extra_sparts)
threadpool_map(&s->e->threadpool, space_reorder_extra_sparts_mapper,
- s->cells_top, s->nr_cells, sizeof(struct cell), 0, s);
+ s->local_cells_top, s->nr_local_cells, sizeof(int), 0, s);
}
/**