Make use of a temporary threadpool in the reading routines to speed-up the...

Make use of a temporary threadpool in the reading routines to speed-up the duplication of parts into gparts.

examples/main.c

@@ -505,18 +505,20 @@ int main(int argc, char *argv[]) {
   read_ic_parallel(ICfileName, &us, dim, &parts, &gparts, &sparts, &Ngas,
                    &Ngpart, &Nspart, &periodic, &flag_entropy_ICs, with_hydro,
                    (with_external_gravity || with_self_gravity), with_stars,
-                   myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL, dry_run);
+                   myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL, nr_threads,
+                   dry_run);
 #else
   read_ic_serial(ICfileName, &us, dim, &parts, &gparts, &sparts, &Ngas, &Ngpart,
                  &Nspart, &periodic, &flag_entropy_ICs, with_hydro,
                  (with_external_gravity || with_self_gravity), with_stars,
-                 myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL, dry_run);
+                 myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL, nr_threads,
+                 dry_run);
 #endif
 #else
   read_ic_single(ICfileName, &us, dim, &parts, &gparts, &sparts, &Ngas, &Ngpart,
                  &Nspart, &periodic, &flag_entropy_ICs, with_hydro,
                  (with_external_gravity || with_self_gravity), with_stars,
-                 dry_run);
+                 nr_threads, dry_run);
 #endif
   if (myrank == 0) {
     clocks_gettime(&toc);

src/common_io.c

@@ -610,6 +610,23 @@ void io_copy_temp_buffer(void* temp, const struct engine* e,
   }
 }
 
+void io_prepare_dm_gparts_mapper(void* restrict data, int Ndm, void* dummy) {
+
+  struct gpart* restrict gparts = (struct gpart*)data;
+
+  /* Let's give all these gparts a negative id */
+  for (int i = 0; i < Ndm; ++i) {
+
+    /* 0 or negative ids are not allowed */
+    if (gparts[i].id_or_neg_offset <= 0)
+      error("0 or negative ID for DM particle %i: ID=%lld", i,
+            gparts[i].id_or_neg_offset);
+
+    /* Set gpart type */
+    gparts[i].type = swift_type_dark_matter;
+  }
+}
+
 /**
  * @brief Prepare the DM particles (in gparts) read in for the addition of the
  * other particle types
@@ -617,21 +634,15 @@ void io_copy_temp_buffer(void* temp, const struct engine* e,
  * This function assumes that the DM particles are all at the start of the
  * gparts array
  *
+ * @param tp The current #threadpool.
  * @param gparts The array of #gpart freshly read in.
  * @param Ndm The number of DM particles read in.
  */
-void io_prepare_dm_gparts(struct gpart* const gparts, size_t Ndm) {
-
-  /* Let's give all these gparts a negative id */
-  for (size_t i = 0; i < Ndm; ++i) {
-    /* 0 or negative ids are not allowed */
-    if (gparts[i].id_or_neg_offset <= 0)
-      error("0 or negative ID for DM particle %zu: ID=%lld", i,
-            gparts[i].id_or_neg_offset);
+void io_prepare_dm_gparts(struct threadpool* tp, struct gpart* const gparts,
+                          size_t Ndm) {
 
-    /* Set gpart type */
-    gparts[i].type = swift_type_dark_matter;
-  }
+  threadpool_map(tp, io_prepare_dm_gparts_mapper, gparts, Ndm,
+                 sizeof(struct gpart), 0, NULL);
 }
 
 /**

src/common_io.h

@@ -35,6 +35,7 @@
 /* Avoid cyclic inclusion problems */
 struct io_props;
 struct engine;
+struct threadpool;
 
 #if defined(HAVE_HDF5)
 
@@ -86,7 +87,8 @@ void io_copy_temp_buffer(void* temp, const struct engine* e,
 
 void io_collect_dm_gparts(const struct gpart* const gparts, size_t Ntot,
                           struct gpart* const dmparts, size_t Ndm);
-void io_prepare_dm_gparts(struct gpart* const gparts, size_t Ndm);
+void io_prepare_dm_gparts(struct threadpool* tp, struct gpart* const gparts,
+                          size_t Ndm);
 void io_duplicate_hydro_gparts(struct part* const parts,
                                struct gpart* const gparts, size_t Ngas,
                                size_t Ndm);

src/parallel_io.c

@@ -539,7 +539,7 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units,
                       size_t* Nstars, int* periodic, int* flag_entropy,
                       int with_hydro, int with_gravity, int with_stars,
                       int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info,
-                      int dry_run) {
+                      int n_threads, int dry_run) {
 
   hid_t h_file = 0, h_grp = 0;
   /* GADGET has only cubic boxes (in cosmological mode) */
@@ -759,16 +759,24 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units,
     H5Gclose(h_grp);
   }
 
-  /* Prepare the DM particles */
-  if (!dry_run && with_gravity) io_prepare_dm_gparts(*gparts, Ndm);
+  if (!dry_run && with_gravity) {
 
-  /* Duplicate the hydro particles into gparts */
-  if (!dry_run && with_gravity && with_hydro)
-    io_duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm);
+    /* Let's initialise a bit of thread parallelism here */
+    struct threadpool tp;
+    threadpool_init(&tp, 4);
 
-  /* Duplicate the star particles into gparts */
-  if (!dry_run && with_gravity && with_stars)
-    io_duplicate_star_gparts(*sparts, *gparts, *Nstars, Ndm + *Ngas);
+    /* Prepare the DM particles */
+    io_prepare_dm_gparts(*gparts, Ndm);
+
+    /* Duplicate the hydro particles into gparts */
+    if (with_hydro) io_duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm);
+
+    /* Duplicate the star particles into gparts */
+    if (with_stars)
+      io_duplicate_star_gparts(*sparts, *gparts, *Nstars, Ndm + *Ngas);
+
+    threadpool_clean(&tp);
+  }
 
   /* message("Done Reading particles..."); */
 

src/parallel_io.h

@@ -40,7 +40,7 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units,
                       size_t* Nsparts, int* periodic, int* flag_entropy,
                       int with_hydro, int with_gravity, int with_stars,
                       int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info,
-                      int dry_run);
+                      int nr_threads, int dry_run);
 
 void write_output_parallel(struct engine* e, const char* baseName,
                            const struct unit_system* internal_units,

src/serial_io.c

@@ -396,7 +396,7 @@ void read_ic_serial(char* fileName, const struct unit_system* internal_units,
                     size_t* Nstars, int* periodic, int* flag_entropy,
                     int with_hydro, int with_gravity, int with_stars,
                     int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info,
-                    int dry_run) {
+                    int n_threads, int dry_run) {
 
   hid_t h_file = 0, h_grp = 0;
   /* GADGET has only cubic boxes (in cosmological mode) */
@@ -647,16 +647,25 @@ void read_ic_serial(char* fileName, const struct unit_system* internal_units,
     MPI_Barrier(comm);
   }
 
-  /* Prepare the DM particles */
-  if (!dry_run && with_gravity) io_prepare_dm_gparts(*gparts, Ndm);
+  /* Duplicate the parts for gravity */
+  if (!dry_run && with_gravity) {
 
-  /* Duplicate the hydro particles into gparts */
-  if (!dry_run && with_gravity && with_hydro)
-    io_duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm);
+    /* Let's initialise a bit of thread parallelism here */
+    struct threadpool tp;
+    threadpool_init(&tp, n_threads);
 
-  /* Duplicate the star particles into gparts */
-  if (!dry_run && with_gravity && with_stars)
-    io_duplicate_star_gparts(*sparts, *gparts, *Nstars, Ndm + *Ngas);
+    /* Prepare the DM particles */
+    io_prepare_dm_gparts(&tp, *gparts, Ndm);
+
+    /* Duplicate the hydro particles into gparts */
+    if (with_hydro) io_duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm);
+
+    /* Duplicate the star particles into gparts */
+    if (with_stars)
+      io_duplicate_star_gparts(*sparts, *gparts, *Nstars, Ndm + *Ngas);
+
+    threadpool_clean(&tp);
+  }
 
   /* message("Done Reading particles..."); */
 

src/serial_io.h

@@ -40,7 +40,7 @@ void read_ic_serial(char* fileName, const struct unit_system* internal_units,
                     size_t* Nstars, int* periodic, int* flag_entropy,
                     int with_hydro, int with_gravity, int with_stars,
                     int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info,
-                    int dry_run);
+                    int nr_threads, int dry_run);
 
 void write_output_serial(struct engine* e, const char* baseName,
                          const struct unit_system* internal_units,

src/single_io.c

@@ -310,7 +310,7 @@ void read_ic_single(char* fileName, const struct unit_system* internal_units,
                     struct spart** sparts, size_t* Ngas, size_t* Ngparts,
                     size_t* Nstars, int* periodic, int* flag_entropy,
                     int with_hydro, int with_gravity, int with_stars,
-                    int dry_run) {
+                    int n_threads, int dry_run) {
 
   hid_t h_file = 0, h_grp = 0;
   /* GADGET has only cubic boxes (in cosmological mode) */
@@ -515,16 +515,25 @@ void read_ic_single(char* fileName, const struct unit_system* internal_units,
     H5Gclose(h_grp);
   }
 
-  /* Prepare the DM particles */
-  if (!dry_run && with_gravity) io_prepare_dm_gparts(*gparts, Ndm);
+  /* Duplicate the parts for gravity */
+  if (!dry_run && with_gravity) {
 
-  /* Duplicate the hydro particles into gparts */
-  if (!dry_run && with_gravity && with_hydro)
-    io_duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm);
+    /* Let's initialise a bit of thread parallelism here */
+    struct threadpool tp;
+    threadpool_init(&tp, n_threads);
 
-  /* Duplicate the star particles into gparts */
-  if (!dry_run && with_gravity && with_stars)
-    io_duplicate_star_gparts(*sparts, *gparts, *Nstars, Ndm + *Ngas);
+    /* Prepare the DM particles */
+    io_prepare_dm_gparts(&tp, *gparts, Ndm);
+
+    /* Duplicate the hydro particles into gparts */
+    if (with_hydro) io_duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm);
+
+    /* Duplicate the star particles into gparts */
+    if (with_stars)
+      io_duplicate_star_gparts(*sparts, *gparts, *Nstars, Ndm + *Ngas);
+
+    threadpool_clean(&tp);
+  }
 
   /* message("Done Reading particles..."); */
 

src/single_io.h

@@ -34,7 +34,7 @@ void read_ic_single(char* fileName, const struct unit_system* internal_units,
                     struct spart** sparts, size_t* Ngas, size_t* Ndm,
                     size_t* Nstars, int* periodic, int* flag_entropy,
                     int with_hydro, int with_gravity, int with_stars,
-                    int dry_run);
+                    int nr_threads, int dry_run);
 
 void write_output_single(struct engine* e, const char* baseName,
                          const struct unit_system* internal_units,