Logger: second review

examples/main.c

@@ -254,7 +254,7 @@ int main(int argc, char *argv[]) {
                  "frequency (Hz) to be used for time measurements.",
                  NULL, 0, 0),
       OPT_BOOLEAN(0, "logger", &with_logger,
-                  "Run with the logger.", NULL, 0, 0),
+                  "Run with the particle logger.", NULL, 0, 0),
       OPT_INTEGER('n', "steps", &nsteps,
                   "Execute a fixed number of time steps. When unset use the "
                   "time_end parameter to stop.",

src/engine.c

@@ -457,7 +457,7 @@ void engine_exchange_strays(struct engine *e, const size_t offset_parts,
 
 #ifdef WITH_LOGGER
     if (e->policy & engine_policy_logger) {
-      const int logger_flag = logger_generate_flag(
+      const int logger_flag = logger_generate_flag_data(
         logger_flag_mpi_exit, node_id);
 
       /* Log the particle when leaving a rank. */
@@ -506,7 +506,7 @@ void engine_exchange_strays(struct engine *e, const size_t offset_parts,
 
 #ifdef WITH_LOGGER
     if (e->policy & engine_policy_logger) {
-      const int logger_flag = logger_generate_flag(
+      const int logger_flag = logger_generate_flag_data(
         logger_flag_mpi_exit, node_id);
 
       /* Log the particle when leaving a rank. */
@@ -555,7 +555,7 @@ void engine_exchange_strays(struct engine *e, const size_t offset_parts,
 
 #ifdef WITH_LOGGER
     if (e->policy & engine_policy_logger) {
-      error("TODO");
+      error("Not yet implemented.");
     }
 #endif
   }
@@ -593,7 +593,7 @@ void engine_exchange_strays(struct engine *e, const size_t offset_parts,
     if ((e->policy & engine_policy_logger) &&
         s->gparts[offset_gparts + k].type == swift_type_dark_matter) {
 
-      const int logger_flag = logger_generate_flag(
+      const int logger_flag = logger_generate_flag_data(
          logger_flag_mpi_exit, node_id);
 
       /* Log the particle when leaving a rank. */
@@ -826,7 +826,7 @@ void engine_exchange_strays(struct engine *e, const size_t offset_parts,
 
 #ifdef WITH_LOGGER
       if (e->policy & engine_policy_logger) {
-        const int flag = logger_generate_flag(logger_flag_mpi_enter,
+        const int flag = logger_generate_flag_data(logger_flag_mpi_enter,
                                               prox->nodeID);
 
         struct part *parts = &s->parts[offset_parts + count_parts];
@@ -839,37 +839,23 @@ void engine_exchange_strays(struct engine *e, const size_t offset_parts,
           logger_masks_all_part |
           logger_mask_data[logger_special_flags].mask;
 
-        for(int i = 0; i < prox->nr_parts_in; i++) {
-          logger_log_part(e->logger, &parts[i], mask_hydro,
-                          &xparts[i].logger_data.last_offset,
-                          flag);
-          /* Reset the counter */
-          xparts[i].logger_data.steps_since_last_output = 0;
-        }
+        logger_log_parts(e->logger, parts, xparts,
+                         mask_hydro, prox->nr_parts_in, flag);
 
         /* Log the stellar particles */
         const unsigned int mask_stars = logger_masks_all_spart |
           logger_mask_data[logger_special_flags].mask;
-        for(int i = 0; i < prox->nr_sparts_in; i++) {
-          logger_log_spart(e->logger, &sparts[i], mask_stars,
-                           &sparts[i].logger_data.last_offset,
-                           flag);
-          sparts[i].logger_data.steps_since_last_output = 0;
-        }
+
+        logger_log_sparts(e->logger, sparts, mask_stars,
+                          prox->nr_sparts_in, flag);
 
         /* Log the gparts */
         const unsigned int mask_grav =
           logger_masks_all_gpart |
           logger_mask_data[logger_special_flags].mask;
-        for(int i = 0; i < prox->nr_gparts_in; i++) {
-          /* Log only the dark matter */
-          if (gparts[i].type != swift_type_dark_matter) continue;
-
-          logger_log_gpart(e->logger, &gparts[i], mask_grav,
-                           &gparts[i].logger_data.last_offset,
-                           flag);
-          gparts[i].logger_data.steps_since_last_output = 0;
-        }
+
+        logger_log_gparts(e->logger, gparts, mask_grav,
+                          prox->nr_gparts_in, flag);
 
         /* Log the bparts */
         if (prox->nr_bparts_in > 0) {

src/engine_redistribute.c

@@ -979,12 +979,60 @@ void engine_redistribute(struct engine *e) {
 #ifdef WITH_LOGGER
   if (e->policy & engine_policy_logger) {
     /* Log the particles before sending them out */
-    const int sending = 1;
-    logger_log_repartition(e->logger, nr_nodes, sending, s->parts,
-                           s->xparts, nr_parts_new, counts,
-                           s->gparts, nr_gparts_new, g_counts,
-                           s->sparts, nr_sparts_new, s_counts,
-                           s->bparts, nr_bparts_new, b_counts);
+    size_t part_offset = 0;
+    size_t spart_offset = 0;
+    size_t gpart_offset = 0;
+    size_t bpart_offset = 0;
+
+    for(int i = 0; i < nr_nodes; i++) {
+      const size_t c_ind = engine_rank * nr_nodes + i;
+
+      /* No need to log the local particles. */
+      if (i == engine_rank) {
+        part_offset += counts[c_ind];
+        spart_offset += s_counts[c_ind];
+        gpart_offset += g_counts[c_ind];
+        bpart_offset += b_counts[c_ind];
+        continue;
+      }
+      const int flag = logger_generate_flag_data(
+        logger_flag_mpi_exit, i);
+
+      const unsigned int mask_hydro =
+        logger_masks_all_part |
+        logger_mask_data[logger_special_flags].mask;
+
+      /* Log the hydro parts. */
+      logger_log_parts(e->logger, &parts[part_offset],
+                       &xparts[part_offset], mask_hydro,
+                       counts[c_ind], flag);
+
+      const unsigned int mask_stars = logger_masks_all_spart |
+        logger_mask_data[logger_special_flags].mask;
+
+      /* Log the stellar parts. */
+      logger_log_sparts(e->logger, &sparts[spart_offset], mask_stars,
+                        s_counts[c_ind], flag);
+
+      const unsigned int mask_grav =
+        logger_masks_all_gpart |
+        logger_mask_data[logger_special_flags].mask;
+
+      /* Log the gparts */
+      logger_log_gparts(e->logger, &gparts[gpart_offset], mask_grav,
+                        g_counts[c_ind], flag);
+
+      /* Log the bparts */
+      if (b_counts[c_ind] > 0) {
+        error("TODO");
+      }
+
+      /* Update the counters */
+      part_offset += counts[c_ind];
+      spart_offset += s_counts[c_ind];
+      gpart_offset += g_counts[c_ind];
+      bpart_offset += b_counts[c_ind];
+    }
   }
 #endif
 
@@ -1042,13 +1090,61 @@ void engine_redistribute(struct engine *e) {
 
 #ifdef WITH_LOGGER
   if (e->policy & engine_policy_logger) {
-    /* Log the received particles */
-    const int sending = 0;
-    logger_log_repartition(e->logger, nr_nodes, sending, s->parts,
-                           s->xparts, nr_parts_new, counts,
-                           s->gparts, nr_gparts_new, g_counts,
-                           s->sparts, nr_sparts_new, s_counts,
-                           s->bparts, nr_bparts_new, b_counts);
+    size_t part_offset = 0;
+    size_t spart_offset = 0;
+    size_t gpart_offset = 0;
+    size_t bpart_offset = 0;
+
+    for(int i = 0; i < nr_nodes; i++) {
+      const size_t c_ind = i * nr_nodes + engine_rank;
+
+      /* No need to log the local particles. */
+      if (i == engine_rank) {
+        part_offset += counts[c_ind];
+        spart_offset += s_counts[c_ind];
+        gpart_offset += g_counts[c_ind];
+        bpart_offset += b_counts[c_ind];
+        continue;
+      }
+
+      const int flag = logger_generate_flag_data(
+        logger_flag_mpi_enter, i);
+
+      const unsigned int mask_hydro =
+        logger_masks_all_part |
+        logger_mask_data[logger_special_flags].mask;
+
+      /* Log the hydro parts. */
+      logger_log_parts(e->logger, &parts[part_offset],
+                       &xparts[part_offset], mask_hydro,
+                       counts[c_ind], flag);
+
+      const unsigned int mask_stars = logger_masks_all_spart |
+        logger_mask_data[logger_special_flags].mask;
+
+      /* Log the stellar parts. */
+      logger_log_sparts(e->logger, &sparts[spart_offset], mask_stars,
+                        s_counts[c_ind], flag);
+
+      const unsigned int mask_grav =
+        logger_masks_all_gpart |
+        logger_mask_data[logger_special_flags].mask;
+
+      /* Log the gparts */
+      logger_log_gparts(e->logger, &gparts[gpart_offset], mask_grav,
+                        g_counts[c_ind], flag);
+
+      /* Log the bparts */
+      if (b_counts[c_ind] > 0) {
+        error("TODO");
+      }
+
+      /* Update the counters */
+      part_offset += counts[c_ind];
+      spart_offset += s_counts[c_ind];
+      gpart_offset += g_counts[c_ind];
+      bpart_offset += b_counts[c_ind];
+    }
   }
 #endif
 

src/logger.c

@@ -210,30 +210,24 @@ void logger_log_all(struct logger_writer *log, const struct engine *e) {
 }
 
 /**
- * @brief Dump a #part to the log.
+ * @brief Copy the particle fields into a given buffer.
  *
- * @param log The #logger_writer
- * @param p The #part to dump.
- * @param mask The mask of the data to dump.
- * @param offset Pointer to the offset of the previous log of this particle;
- * (return) offset of this log.
- * @param special_flags The value of the special flag.
+ * @param p The #part to copy.
+ * @param mask The mask for the fields to write.
+ * @param offset The offset to the previous log.
+ * @param offset_new The offset of the current record.
+ * @param buff The buffer to use when writing.
+ * @param special_flags The data for the special flags.
  */
-void logger_log_part(struct logger_writer *log, const struct part *p,
-                     unsigned int mask, size_t *offset,
-                     const int special_flags) {
+void logger_copy_part_fields(
+    const struct part *p, unsigned int mask,
+    size_t *offset, size_t offset_new, char *buff,
+    const int special_flags) {
 
   /* Make sure we're not writing a timestamp. */
   if (mask & logger_mask_data[logger_timestamp].mask)
     error("You should not log particles as timestamps.");
 
-  /* Start by computing the size of the message. */
-  const int size = logger_compute_chunk_size(mask);
-
-  /* Allocate a chunk of memory in the dump of the right size. */
-  size_t offset_new;
-  char *buff = (char *)dump_get(&log->dump, size, &offset_new);
-
   /* Write the header. */
   buff = logger_write_chunk_header(buff, &mask, offset, offset_new);
 
@@ -293,41 +287,94 @@ void logger_log_part(struct logger_writer *log, const struct part *p,
     buff += logger_mask_data[logger_special_flags].size;
   }
 
-  /* Update the log message offset. */
-  *offset = offset_new;
 }
 
 /**
- * @brief Dump a #spart to the log.
+ * @brief Dump a #part to the log.
  *
  * @param log The #logger_writer
- * @param sp The #spart to dump.
+ * @param p The #part to dump.
  * @param mask The mask of the data to dump.
  * @param offset Pointer to the offset of the previous log of this particle;
- * @param special_flags The value of the special flag.
  * (return) offset of this log.
+ * @param special_flags The value of the special flag.
  */
-void logger_log_spart(struct logger_writer *log, const struct spart *sp,
-                      unsigned int mask, size_t *offset,
+void logger_log_part(struct logger_writer *log, const struct part *p,
+                     unsigned int mask, size_t *offset,
+                     const int special_flags) {
+
+  /* Start by computing the size of the message. */
+  const int size = logger_compute_chunk_size(mask);
+
+  /* Allocate a chunk of memory in the dump of the right size. */
+  size_t offset_new;
+  char *buff = (char *)dump_get(&log->dump, size, &offset_new);
+
+  /* Copy everything into the buffer */
+  logger_copy_part_fields(p, mask, offset, offset_new, buff,
+                          special_flags);
+
+  /* Update the log message offset. */
+  *offset = offset_new;
+}
+
+/**
+ * @brief Dump a group of #part to the log.
+ *
+ * @param log The #logger_writer
+ * @param sp The #part to dump.
+ * @param mask The mask of the data to dump.
+ * @param count The number of particle to dump.
+ * @param special_flags The value of the special flags.
+ */
+void logger_log_parts(struct logger_writer *log, const struct part *p,
+                      struct xpart *xp,  unsigned int mask, int count,
                       const int special_flags) {
+  /* Start by computing the size of the message. */
+  const int size = logger_compute_chunk_size(mask);
+
+  /* Allocate a chunk of memory in the dump of the right size. */
+  size_t offset_new;
+  char *buff = (char *)dump_get(&log->dump, count * size, &offset_new);
+
+  for(int i = 0; i < count; i++) {
+    /* Copy everything into the buffer */
+    logger_copy_part_fields(&p[i], mask, &xp[i].logger_data.last_offset,
+                            offset_new, buff, special_flags);
+
+    /* Update the pointers */
+    xp[i].logger_data.last_offset = offset_new;
+    xp[i].logger_data.steps_since_last_output = 0;
+    buff += size;
+    offset_new += size;
+  }
+}
+
+/**
+ * @brief Copy the particle fields into a given buffer.
+ *
+ * @param sp The #spart to copy.
+ * @param mask The mask for the fields to write.
+ * @param offset The offset to the previous log.
+ * @param offset_new The offset of the current record.
+ * @param buff The buffer to use when writing.
+ * @param special_flags The data for the special flags.
+ */
+void logger_copy_spart_fields(
+    const struct spart *sp, unsigned int mask,
+    size_t *offset, size_t offset_new, char *buff,
+    const int special_flags) {
 
   /* Make sure we're not writing a timestamp. */
   if (mask & logger_mask_data[logger_timestamp].mask)
     error("You should not log particles as timestamps.");
 
-  /* Make sure we're not looging fields not supported by gparts. */
+  /* Make sure we're not looging fields not supported by sparts. */
   if (mask &
       (logger_mask_data[logger_u].mask | logger_mask_data[logger_rho].mask |
        logger_mask_data[logger_a].mask))
     error("Can't log SPH quantities for sparts.");
 
-  /* Start by computing the size of the message. */
-  const int size = logger_compute_chunk_size(mask);
-
-  /* Allocate a chunk of memory in the dump of the right size. */
-  size_t offset_new;
-  char *buff = (char *)dump_get(&log->dump, size, &offset_new);
-
   /* Write the header. */
   buff = logger_write_chunk_header(buff, &mask, offset, offset_new);
 
@@ -358,27 +405,87 @@ void logger_log_spart(struct logger_writer *log, const struct spart *sp,
     memcpy(buff, &special_flags, logger_mask_data[logger_special_flags].size);
     buff += logger_mask_data[logger_special_flags].size;
   }
-
-  /* Update the log message offset. */
-  *offset = offset_new;
 }
 
 /**
- * @brief Dump a #gpart to the log.
+ * @brief Dump a #spart to the log.
  *
  * @param log The #logger_writer
- * @param p The #gpart to dump.
+ * @param sp The #spart to dump.
  * @param mask The mask of the data to dump.
  * @param offset Pointer to the offset of the previous log of this particle;
- * @param special_flags The value of the special flags.
+ * @param special_flags The value of the special flag.
  * (return) offset of this log.
  */
-void logger_log_gpart(struct logger_writer *log, const struct gpart *p,
+void logger_log_spart(struct logger_writer *log, const struct spart *sp,
                       unsigned int mask, size_t *offset,
                       const int special_flags) {
 
+
+  /* Start by computing the size of the message. */
+  const int size = logger_compute_chunk_size(mask);
+
+  /* Allocate a chunk of memory in the dump of the right size. */
+  size_t offset_new;
+  char *buff = (char *)dump_get(&log->dump, size, &offset_new);
+
+  /* Copy the fields into the buffer. */
+  logger_copy_spart_fields(sp, mask, offset, offset_new, buff,
+                           special_flags);
+
+  /* Update the log message offset. */
+  *offset = offset_new;
+}
+
+/**
+ * @brief Dump a group of #spart to the log.
+ *
+ * @param log The #logger_writer
+ * @param sp The #spart to dump.
+ * @param mask The mask of the data to dump.
+ * @param count The number of particle to dump.
+ * @param special_flags The value of the special flags.
+ */
+void logger_log_sparts(struct logger_writer *log, struct spart *sp,
+                       unsigned int mask, int count,
+                       const int special_flags) {
+  /* Start by computing the size of the message. */
+  const int size = logger_compute_chunk_size(mask);
+
+  /* Allocate a chunk of memory in the dump of the right size. */
+  size_t offset_new;
+  char *buff = (char *)dump_get(&log->dump, count * size, &offset_new);
+
+  for(int i = 0; i < count; i++) {
+    /* Copy everything into the buffer */
+    logger_copy_spart_fields(&sp[i], mask, &sp[i].logger_data.last_offset,
+                             offset_new, buff, special_flags);
+
+    /* Update the pointers */
+    sp[i].logger_data.last_offset = offset_new;
+    sp[i].logger_data.steps_since_last_output = 0;
+    buff += size;
+    offset_new += size;
+  }
+}
+
+/**
+ * @brief Copy the particle fields into a given buffer.
+ *
+ * @param gp The #gpart to copy.
+ * @param mask The mask for the fields to write.
+ * @param offset The offset to the previous log.
+ * @param offset_new The offset of the current record.
+ * @param buff The buffer to use when writing.
+ * @param special_flags The data of the special flag.
+ */
+void logger_copy_gpart_fields(
+    const struct gpart *gp, unsigned int mask,
+    size_t *offset, size_t offset_new, char *buff,
+    const int special_flags) {
+
 #ifdef SWIFT_DEBUG_CHECKS
-  if (p->id_or_neg_offset < 0) {
+  if (gp->id_or_neg_offset < 0) {
     error("Cannot log a gpart attached to another particle");
   }
 #endif
@@ -392,40 +499,33 @@ void logger_log_gpart(struct logger_writer *log, const struct gpart *p,
       (logger_mask_data[logger_u].mask | logger_mask_data[logger_rho].mask))
     error("Can't log SPH quantities for gparts.");
 
-  /* Start by computing the size of the message. */
-  const int size = logger_compute_chunk_size(mask);
-
-  /* Allocate a chunk of memory in the dump of the right size. */
-  size_t offset_new;
-  char *buff = (char *)dump_get(&log->dump, size, &offset_new);
-
   /* Write the header. */
   buff = logger_write_chunk_header(buff, &mask, offset, offset_new);
 
   /* Particle position as three doubles. */
   if (mask & logger_mask_data[logger_x].mask) {
-    memcpy(buff, p->x, logger_mask_data[logger_x].size);
+    memcpy(buff, gp->x, logger_mask_data[logger_x].size);
     buff += logger_mask_data[logger_x].size;
   }
 
   /* Particle velocity as three floats. */
   if (mask & logger_mask_data[logger_v].mask) {
-    memcpy(buff, p->v_full, logger_mask_data[logger_v].size);
+    memcpy(buff, gp->v_full, logger_mask_data[logger_v].size);
     buff += logger_mask_data[logger_v].size;
   }
 
   /* Particle accelleration as three floats. */
   if (mask & logger_mask_data[logger_a].mask) {
-    memcpy(buff, p->a_grav, logger_mask_data[logger_a].size);
+    memcpy(buff, gp->a_grav, logger_mask_data[logger_a].size);
     buff += logger_mask_data[logger_a].size;
   }
 
   /* Particle constants, which is a bit more complicated. */
   if (mask & logger_mask_data[logger_consts].mask) {
     // TODO make it dependent of logger_mask_data.
-    memcpy(buff, &p->mass, sizeof(float));
+    memcpy(buff, &gp->mass, sizeof(float));
     buff += sizeof(float);
-    const int64_t id = p->id_or_neg_offset;
+    const int64_t id = gp->id_or_neg_offset;
     memcpy(buff, &id, sizeof(int64_t));
     buff += sizeof(int64_t);
   }
@@ -436,10 +536,72 @@ void logger_log_gpart(struct logger_writer *log, const struct gpart *p,
     buff += logger_mask_data[logger_special_flags].size;
   }
 
+}
+
+/**
+ * @brief Dump a #gpart to the log.
+ *
+ * @param log The #logger_writer
+ * @param p The #gpart to dump.
+ * @param mask The mask of the data to dump.
+ * @param offset Pointer to the offset of the previous log of this particle;
+ * (return) offset of this log.
+ * @param special_flags The value of the special flags.
+ */
+void logger_log_gpart(struct logger_writer *log, const struct gpart *p,
+                      unsigned int mask, size_t *offset,
+                      const int special_flags) {
+
+  /* Start by computing the size of the message. */
+  const int size = logger_compute_chunk_size(mask);
+
+  /* Allocate a chunk of memory in the dump of the right size. */
+  size_t offset_new;
+  char *buff = (char *)dump_get(&log->dump, size, &offset_new);
+
+  /* Copy everything into the buffer */
+  logger_copy_gpart_fields(p, mask, offset, offset_new, buff,
+                           special_flags);
+
   /* Update the log message offset. */
   *offset = offset_new;
 }
 
+/**
+ * @brief Dump a group of #gpart to the log.
+ *
+ * @param log The #logger_writer
+ * @param p The #gpart to dump.
+ * @param mask The mask of the data to dump.
+ * @param count The number of particle to dump.
+ * @param special_flags The value of the special flags.
+ */
+void logger_log_gparts(struct logger_writer *log, struct gpart *p,
+                       unsigned int mask, int count,
+                       const int special_flags) {
+  /* Start by computing the size of the message. */
+  const int size = logger_compute_chunk_size(mask);
+
+  /* Allocate a chunk of memory in the dump of the right size. */
+  size_t offset_new;
+  char *buff = (char *)dump_get(&log->dump, count * size, &offset_new);
+
+  for(int i = 0; i < count; i++) {
+    /* Log only the dark matter */
+    if (p[i].type != swift_type_dark_matter) continue;
+
+    /* Copy everything into the buffer */
+    logger_copy_gpart_fields(&p[i], mask, &p[i].logger_data.last_offset,
+                             offset_new, buff, special_flags);
+
+    /* Update the pointers */
+    p[i].logger_data.last_offset = offset_new;
+    p[i].logger_data.steps_since_last_output = 0;
+    buff += size;
+    offset_new += size;
+  }
+}
+
 /**
  * @brief write a timestamp
  *
@@ -832,112 +994,6 @@ int logger_read_timestamp(unsigned long long int *t, double *time,
 }
 
 
-#ifdef WITH_MPI
-/**
- * @brief Log all the particles leaving the current rank.
- *
- * @param log The #logger_writer.
- * @param nr_nodes Number of nodes used in the simulation.
- * @param sneding Are we sending the particles (or receiving)?
- * @param parts The list of #part.
- * @param nr_parts The number of parts.
- * @param count The number of parts in each ranks.
- * @param gparts The list of #gpart.
- * @param nr_gparts The number of gparts.
- * @param gcount The number of gparts in each ranks.
- * @param sparts The list of #spart.
- * @param nr_sparts The number of sparts.
- * @param s_counts The number of sparts in each ranks.
- * @param bparts The list of #bpart.
- * @param nr_bparts The number of bparts.
- * @param b_counts The number of bparts in each ranks.
- *
- */
-void logger_log_repartition(