Implemented a faster generation of index files

src/csds_definitions.h

@@ -28,7 +28,7 @@
 
 /* Size of the strings. */
 #define CSDS_STRING_SIZE 200
-#define CSDS_FORMAT_STRING "CSDS"
+#define CSDS_FORMAT_STRING "SWIFT_CSDS"
 
 /* Defines the special flags */
 #define CSDS_SPECIAL_FLAGS_NAME "SpecialFlags"

src/csds_header.c

@@ -116,7 +116,7 @@ void header_read(struct header *h, struct csds_logfile *log) {
 
   /* read the file format. */
   char file_format[CSDS_STRING_SIZE];
-  map = csds_loader_io_read_data(map, CSDS_STRING_SIZE, &file_format);
+  map = csds_loader_io_read_data(map, 20, &file_format);
   if (strcmp(file_format, CSDS_FORMAT_STRING))
     error_python("Wrong file format (%s).", file_format);
 

src/csds_reader_generate_index.c

@@ -480,6 +480,10 @@ size_t csds_reader_update_state_to_next_index(
   /* Record the initial time */
   const ticks tic = getticks();
 
+  /* Get the mask for the IDs */
+  const struct field_information *field_id =
+    header_get_field_from_name(h, "ParticleIDs", csds_type_dark_matter);
+
   /* Warn the OS that we will read in a sequential way */
   CSDS_ADVICE_SEQUENTIAL(log->log);
 
@@ -488,7 +492,7 @@ size_t csds_reader_update_state_to_next_index(
     /* Print status */
     if (reader->verbose > 0) {
       step += 1;
-      if (step % 100 == 0) {
+      if (step % 1000 == 0) {
         step = 0;
 
         /* Get the percentage */
@@ -511,10 +515,6 @@ size_t csds_reader_update_state_to_next_index(
     }
     mask_type mask = 0;
     size_t h_offset = 0;
-    int part_type = -1;  // only available if the record is flagged.
-    int data = 0;
-
-    /* Get the mask */
     csds_loader_io_read_mask(log->log.map + offset, &mask, &h_offset);
 
     /* Go to the next record */
@@ -523,13 +523,33 @@ size_t csds_reader_update_state_to_next_index(
     offset += size_record_header;
 
     /* Check if we have a particle with a flag */
-    if (mask & CSDS_TIMESTAMP_MASK || !(mask & CSDS_SPECIAL_FLAGS_MASK)) {
+    if (mask & CSDS_TIMESTAMP_MASK) {
+      continue;
+    }
+
+    /* Read the ID */
+    int64_t id = 0;
+    csds_particle_read_field(old_offset, &id, field_id,
+                             /* derivative */ 0, &mask, &h_offset,
+                             h->fields[csds_type_dark_matter],
+                             h->number_fields[csds_type_dark_matter],
+                             reader->log.log.map);
+    struct index_data item = {id, old_offset};
+
+    if (!(mask & CSDS_SPECIAL_FLAGS_MASK)) {
+      void *p = csds_hashmap_set(current_state[csds_type_dark_matter], &item);
+      if (p == NULL)
+        error("Found a new particle without mask");
       continue;
     }
 
     /* Get the special flag */
+    int data = 0;
+    int part_type = -1;
     enum csds_special_flags flag = csds_particle_read_special_flag(
         old_offset, &mask, &h_offset, &data, &part_type, log->log.map);
+    if (part_type != csds_type_dark_matter)
+      error("Wrong type");
 
 #ifdef CSDS_DEBUG_CHECKS
     if (flag == csds_flag_none) {
@@ -543,17 +563,6 @@ size_t csds_reader_update_state_to_next_index(
       error("Found a special flag without a particle type");
     }
 
-    /* Get the mask for the IDs */
-    const struct field_information *field_id =
-      header_get_field_from_name(h, "ParticleIDs", (enum part_type) part_type);
-
-    /* Read the ID */
-    int64_t id = 0;
-    csds_particle_read_field(old_offset, &id, field_id,
-                             /* derivative */ 0, &mask, &h_offset,
-                             h->fields[part_type], h->number_fields[part_type],
-                             reader->log.log.map);
-
     /* Add the particle to the arrays */
     if (flag == csds_flag_change_type || flag == csds_flag_mpi_exit ||
         flag == csds_flag_delete) {
@@ -563,7 +572,6 @@ size_t csds_reader_update_state_to_next_index(
       };
     } else if (flag == csds_flag_create || flag == csds_flag_mpi_enter) {
       index_writer_log(&parts_created[part_type], id, old_offset);
-      struct index_data item = {id, old_offset};
       void *p = (void *)csds_hashmap_set(current_state[part_type], &item);
       if (p != NULL) {
         error("Already found a particle with the same ID");
@@ -579,77 +587,10 @@ size_t csds_reader_update_state_to_next_index(
     printf("\n");
   }
 
-  /* Print the time */
-  if (reader->verbose > 0 || reader->verbose == CSDS_VERBOSE_TIMERS)
-    message("Finding new/removed particles took %.3f %s",
-            clocks_from_ticks(getticks() - tic), clocks_getunit());
-
-  /* Initialize the total number of particles for the progress bar */
-  size_t total_number_particles = 0;
-  for (int type = 0; type < csds_type_count; type++) {
-    total_number_particles += csds_hashmap_count(current_state[type]);
-  }
-
-  /* Record the time */
-  const ticks tic2 = getticks();
-  size_t current_number = 0;
-  int local_counter = 0;
-  const int local_update_limit = 1000;
-
-  /* Update the offsets of current_state
-   * No need to update the others as they contain
-   * data about when particles are removed/created*/
-#pragma omp parallel for firstprivate(local_counter)
-  for (int type = 0; type < csds_type_count; type++) {
-    /* Loop over the buckets ~ particles */
-    size_t n_buckets = csds_hashmap_get_number_buckets(current_state[type]);
-    for (size_t i = 0; i < n_buckets; i++) {
-      struct index_data *index_data =
-          csds_hashmap_get_from_position(current_state[type], i);
-
-      /* Did we get an item? */
-      if (index_data == NULL) {
-        continue;
-      }
-
-      /* Update the offset */
-      index_data->offset = csds_reader_get_last_offset_before(
-          reader, index_data, time_record.offset);
-
-      /* Are we done or should we print something? */
-      if (!(reader->verbose > 0)) continue;
-
-      /* Update the counters */
-      local_counter++;
-      if (local_counter < local_update_limit) continue;
-
-        /* Add the local counter to the global one */
-#pragma omp atomic
-      current_number += local_counter;
-      local_counter = 0;
-
-      /* Only the master is printing */
-      int current_thread = csds_get_thread_num();
-      if (current_thread != 0) continue;
-
-      float ratio = (float)current_number / (float)total_number_particles;
-
-      /* Compute the remaining time */
-      const int delta_time = clocks_diff_ticks(getticks(), tic2) / 1000.0;
-      const int remaining_time = delta_time * (1. - ratio) / ratio;
-
-      /* Print the message */
-      tools_print_progress(100 * ratio, remaining_time, "Updating offsets");
-    }
-  }
-
-  /* Cleanup the output */
-  if (reader->verbose > 0) printf("\n");
-
   /* Print the time */
   if (reader->verbose > 0 || reader->verbose == CSDS_VERBOSE_TIMERS)
     message("Updating particles took %.3f %s",
-            clocks_from_ticks(getticks() - tic2), clocks_getunit());
+            clocks_from_ticks(getticks() - tic), clocks_getunit());
 
   return offset;
 }