diff --git a/examples/main.c b/examples/main.c
index 95784341ab3ad363b90c5a33bed9ffc0885499ce..ad80c61b18217c44d5b436b3cc892ba7e3a4fa6c 100644
--- a/examples/main.c
+++ b/examples/main.c
@@ -583,9 +583,10 @@ int main(int argc, char *argv[]) {
#ifdef WITH_MPI
if (with_mpole_reconstruction && nr_nodes > 1)
error("Cannot reconstruct m-poles every step over MPI (yet).");
-#ifdef WITH_LOGGER
- error("Can't run with the particle logger over MPI (yet)");
-#endif
+ if (with_timestep_limiter)
+ error("Can't run with time-step limiter over MPI (yet)");
+ if (with_timestep_sync)
+ error("Can't run with time-step synchronization over MPI (yet)");
#endif
/* Temporary early aborts for modes not supported with hand-vec. */
@@ -1205,7 +1206,7 @@ int main(int argc, char *argv[]) {
#ifdef WITH_MPI
/* Split the space. */
engine_split(&e, &initial_partition);
- engine_redistribute(&e);
+ engine_redistribute(&e, /* initial_redistribute */ 1);
#endif
/* Initialise the particles */
@@ -1418,6 +1419,9 @@ int main(int argc, char *argv[]) {
#ifdef WITH_LOGGER
logger_log_all(e.logger, &e);
+ /* Write a final index file */
+ engine_dump_index(&e);
+
/* Write a sentinel timestamp */
logger_log_timestamp(e.logger, e.ti_current, e.time,
&e.logger->timestamp_offset);
diff --git a/src/engine.c b/src/engine.c
index f6f10841e3752f1328565f81b5e27d658cd3c565..d1af00128e02b605056c94ab9e784540243e34fc 100644
--- a/src/engine.c
+++ b/src/engine.c
@@ -223,7 +223,7 @@ void engine_repartition(struct engine *e) {
Finally, the space, tasks, and proxies need to be rebuilt. */
/* Redistribute the particles between the nodes. */
- engine_redistribute(e);
+ engine_redistribute(e, /* initial_redistribute */ 0);
/* Make the proxies. */
engine_makeproxies(e);
diff --git a/src/engine.h b/src/engine.h
index 7bf3ae264120e0481b8e71087b22cf9753f756d1..6d2d234dba2eedc15ebca3d70f09daa73c0187b8 100644
--- a/src/engine.h
+++ b/src/engine.h
@@ -544,7 +544,7 @@ void engine_rebuild(struct engine *e, int redistributed, int clean_h_values);
void engine_repartition(struct engine *e);
void engine_repartition_trigger(struct engine *e);
void engine_makeproxies(struct engine *e);
-void engine_redistribute(struct engine *e);
+void engine_redistribute(struct engine *e, int initial_redistribute);
void engine_print_policy(struct engine *e);
int engine_is_done(struct engine *e);
void engine_pin(void);
diff --git a/src/engine_redistribute.c b/src/engine_redistribute.c
index 05ea63dfff4b910f1dac72dd50ca6ddc6c9fbe4a..20382f99ef2239ba2d267bc74edb17f395cb4d5c 100644
--- a/src/engine_redistribute.c
+++ b/src/engine_redistribute.c
@@ -514,8 +514,9 @@ static void engine_redistribute_relink_mapper(void *map_data, int num_elements,
*
*
* @param e The #engine.
+ * @param initial_redistribute Is it the initial redistribute (just after reading the particles).
*/
-void engine_redistribute(struct engine *e) {
+void engine_redistribute(struct engine *e, int initial_redistribute) {
#ifdef WITH_MPI
@@ -977,11 +978,15 @@ void engine_redistribute(struct engine *e) {
nr_bparts_new += b_counts[k * nr_nodes + nodeID];
#ifdef WITH_LOGGER
- /* Log the particles before sending them out */
- logger_log_before_communcations(s->parts, nr_parts_new, counts,
- s->gparts, nr_gparts_new, g_counts,
- s->sparts, nr_sparts_new, s_counts,
- s->bparts, nr_bparts_new, b_counts);
+ if (!initial_redistribute) {
+ /* Log the particles before sending them out */
+ const int sending = 1;
+ logger_log_repartition(e->logger, nr_nodes, sending, s->parts,
+ s->xparts, nr_parts_new, counts,
+ s->gparts, nr_gparts_new, g_counts,
+ s->sparts, nr_sparts_new, s_counts,
+ s->bparts, nr_bparts_new, b_counts);
+ }
#endif
/* Now exchange the particles, type by type to keep the memory required
@@ -1037,11 +1042,15 @@ void engine_redistribute(struct engine *e) {
stuff we just received */
#ifdef WITH_LOGGER
- /* Log the received particles */
- logger_log_after_communcations(s->parts, s->nr_parts, counts,
- s->gparts, s->nr_gparts, g_counts,
- s->sparts, s->nr_sparts, s_counts,
- s->bparts, s->nr_bparts, b_counts);
+ if (!initial_redistribute) {
+ /* Log the received particles */
+ const int receiving = 0;
+ logger_log_repartition(e->logger, nr_nodes, receiving, s->parts,
+ s->xparts, nr_parts_new, counts,
+ s->gparts, nr_gparts_new, g_counts,
+ s->sparts, nr_sparts_new, s_counts,
+ s->bparts, nr_bparts_new, b_counts);
+ }
#endif
/* Restore the part<->gpart and spart<->gpart links.
diff --git a/src/logger.c b/src/logger.c
index 9eb52a9da492968596f7481297371857a63a1613..600e66365b847b4a39f5c09543e219128624c293 100644
--- a/src/logger.c
+++ b/src/logger.c
@@ -849,6 +849,9 @@ int logger_read_timestamp(unsigned long long int *t, double *time,
/**
* @brief Log all the particles leaving the current rank.
*
+ * @param log The #logger_writer.
+ * @param nr_nodes Number of nodes used in the simulation.
+ * @param sneding Are we sending the particles (or receiving)?
* @param parts The list of #part.
* @param nr_parts The number of parts.
* @param count The number of parts in each ranks.
@@ -863,37 +866,93 @@ int logger_read_timestamp(unsigned long long int *t, double *time,
* @param b_counts The number of bparts in each ranks.
*
*/
-void logger_log_before_communcations(
- struct part *parts, size_t nr_parts, int *counts,
+void logger_log_repartition(
+ struct logger_writer *log, int nr_nodes, int sending, struct part *parts,
+ struct xpart *xparts, size_t nr_parts, int *counts,
struct gpart *gparts, size_t nr_gparts, int *g_counts,
struct spart *sparts, size_t nr_sparts, int *s_counts,
struct bpart *bparts, size_t nr_bparts, int *b_counts) {
- error("TODO");
-}
-/**
- * @brief Log all the particles arriving in the current rank.
- *
- * @param parts The list of #part.
- * @param nr_parts The number of parts.
- * @param count The number of parts in each ranks.
- * @param gparts The list of #gpart.
- * @param nr_gparts The number of gparts.
- * @param gcount The number of gparts in each ranks.
- * @param sparts The list of #spart.
- * @param nr_sparts The number of sparts.
- * @param s_counts The number of sparts in each ranks.
- * @param bparts The list of #bpart.
- * @param nr_bparts The number of bparts.
- * @param b_counts The number of bparts in each ranks.
- *
- */
-void logger_log_after_communcations(
- struct part *parts, size_t nr_parts, int *counts,
- struct gpart *gparts, size_t nr_gparts, int *g_counts,
- struct spart *sparts, size_t nr_sparts, int *s_counts,
- struct bpart *bparts, size_t nr_bparts, int *b_counts) {
- error("TODO");
+ size_t part_offset = 0;
+ size_t spart_offset = 0;
+ size_t gpart_offset = 0;
+ size_t bpart_offset = 0;
+
+ for(int i = 0; i < nr_nodes; i++) {
+ const size_t c_ind = sending ? engine_rank * nr_nodes + i:
+ i * nr_nodes + engine_rank;
+
+ /* No need to log the local particles. */
+ if (i == engine_rank) {
+ part_offset += counts[c_ind];
+ spart_offset += s_counts[c_ind];
+ gpart_offset += g_counts[c_ind];
+ bpart_offset += b_counts[c_ind];
+ continue;
+ }
+
+ const int flag = logger_generate_flag(
+ logger_flag_mpi | logger_flag_delete, i);
+
+ const unsigned int mask_hydro =
+ logger_mask_data[logger_x].mask | logger_mask_data[logger_v].mask |
+ logger_mask_data[logger_a].mask | logger_mask_data[logger_u].mask |
+ logger_mask_data[logger_h].mask | logger_mask_data[logger_rho].mask |
+ logger_mask_data[logger_consts].mask |
+ logger_mask_data[logger_special_flags].mask;
+
+ /* Log the hydro parts. */
+ for(int j = 0; j < counts[c_ind]; j++) {
+ size_t ind = part_offset + j;
+ message("%i: %lli", sending, parts[ind].id);
+ logger_log_part(log, &parts[ind], mask_hydro,
+ &xparts[ind].logger_data.last_offset,
+ flag);
+ xparts[ind].logger_data.steps_since_last_output = 0;
+ }
+
+ const unsigned int mask_stars = logger_mask_data[logger_x].mask |
+ logger_mask_data[logger_v].mask |
+ logger_mask_data[logger_consts].mask |
+ logger_mask_data[logger_special_flags].mask;
+
+ /* Log the stellar parts. */
+ for(int j = 0; j < s_counts[c_ind]; j++) {
+ size_t ind = spart_offset + j;
+ logger_log_spart(log, &sparts[ind], mask_stars,
+ &sparts[ind].logger_data.last_offset,
+ flag);
+ sparts[ind].logger_data.steps_since_last_output = 0;
+ }
+
+ const unsigned int mask_grav =
+ logger_mask_data[logger_x].mask | logger_mask_data[logger_v].mask |
+ logger_mask_data[logger_a].mask | logger_mask_data[logger_consts].mask |
+ logger_mask_data[logger_special_flags].mask;
+
+ /* Log the gparts */
+ for(int j = 0; j < g_counts[c_ind]; j++) {
+ size_t ind = gpart_offset + j;
+ /* Log only the dark matter */
+ if (gparts[ind].type != swift_type_dark_matter) continue;
+
+ logger_log_gpart(log, &gparts[ind], mask_grav,
+ &gparts[ind].logger_data.last_offset,
+ flag);
+ gparts[ind].logger_data.steps_since_last_output = 0;
+ }
+
+ /* Log the bparts */
+ if (b_counts[c_ind] > 0) {
+ error("TODO");
+ }
+
+ /* Update the counters */
+ part_offset += counts[c_ind];
+ spart_offset += s_counts[c_ind];
+ gpart_offset += g_counts[c_ind];
+ bpart_offset += b_counts[c_ind];
+ }
}
/**
diff --git a/src/logger.h b/src/logger.h
index c7a76cf8bd27a3404803e258ed13c98e07c4e229..f58d2deaae44d8046da0283b978c5c96c94d6fed 100644
--- a/src/logger.h
+++ b/src/logger.h
@@ -197,14 +197,9 @@ INLINE static int logger_generate_flag(enum logger_special_flags flag, int data)
}
#ifdef WITH_MPI
-void logger_log_before_communcations(
- struct part *parts, size_t nr_parts, int *counts,
- struct gpart *gparts, size_t nr_gparts, int *g_counts,
- struct spart *sparts, size_t nr_sparts, int *s_counts,
- struct bpart *bparts, size_t nr_bparts, int *b_counts);
-
-void logger_log_after_communcations(
- struct part *parts, size_t nr_parts, int *counts,
+void logger_log_repartition(
+ struct logger_writer *log, int nr_nodes, int sending, struct part *parts,
+ struct xpart *xparts, size_t nr_parts, int *counts,
struct gpart *gparts, size_t nr_gparts, int *g_counts,
struct spart *sparts, size_t nr_sparts, int *s_counts,
struct bpart *bparts, size_t nr_bparts, int *b_counts);
diff --git a/src/space.c b/src/space.c
index 9d652042fd4361ea1fd76e09eeef4556c772c887..2da60826ecb8d871402dbca6d089cf6e3bbd35bf 100644
--- a/src/space.c
+++ b/src/space.c
@@ -654,7 +654,7 @@ void space_regrid(struct space *s, int verbose) {
}
/* Re-distribute the particles to their new nodes. */
- engine_redistribute(s->e);
+ engine_redistribute(s->e, /* initial_redistribute */ 0);
/* Make the proxies. */
engine_makeproxies(s->e);
@@ -672,7 +672,7 @@ void space_regrid(struct space *s, int verbose) {
partition_restore_celllist(s, s->e->reparttype);
/* Now re-distribute the particles, should just add to cells? */
- engine_redistribute(s->e);
+ engine_redistribute(s->e, /* initial_redistribute */ 0);
/* Make the proxies. */
engine_makeproxies(s->e);