Merge branch 'master' of gitlab.cosma.dur.ac.uk:swift/swiftsim

examples/EAGLE_6/eagle_6.yml

@@ -8,7 +8,6 @@ InternalUnitSystem:
 
 Scheduler:
   max_top_level_cells: 8
-  tasks_per_cell: 50
   
 # Parameters governing the time integration
 TimeIntegration:

examples/main.c

@@ -505,18 +505,20 @@ int main(int argc, char *argv[]) {
   read_ic_parallel(ICfileName, &us, dim, &parts, &gparts, &sparts, &Ngas,
                    &Ngpart, &Nspart, &periodic, &flag_entropy_ICs, with_hydro,
                    (with_external_gravity || with_self_gravity), with_stars,
-                   myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL, dry_run);
+                   myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL, nr_threads,
+                   dry_run);
 #else
   read_ic_serial(ICfileName, &us, dim, &parts, &gparts, &sparts, &Ngas, &Ngpart,
                  &Nspart, &periodic, &flag_entropy_ICs, with_hydro,
                  (with_external_gravity || with_self_gravity), with_stars,
-                 myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL, dry_run);
+                 myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL, nr_threads,
+                 dry_run);
 #endif
 #else
   read_ic_single(ICfileName, &us, dim, &parts, &gparts, &sparts, &Ngas, &Ngpart,
                  &Nspart, &periodic, &flag_entropy_ICs, with_hydro,
                  (with_external_gravity || with_self_gravity), with_stars,
-                 dry_run);
+                 nr_threads, dry_run);
 #endif
   if (myrank == 0) {
     clocks_gettime(&toc);

src/common_io.c

@@ -406,6 +406,8 @@ void io_write_code_description(hid_t h_file) {
   H5Gclose(h_grpcode);
 }
 
+#endif /* HAVE_HDF5 */
+
 /**
  * @brief Mapper function to copy #part or #gpart fields into a buffer.
  */
@@ -608,39 +610,16 @@ void io_copy_temp_buffer(void* temp, const struct engine* e,
   }
 }
 
-#endif /* HAVE_HDF5 */
+void io_prepare_dm_gparts_mapper(void* restrict data, int Ndm, void* dummy) {
 
-/* ------------------------------------------------------------------------------------------------
- * This part writes the XMF file descriptor enabling a visualisation through
- * ParaView
- * ------------------------------------------------------------------------------------------------
- */
-
-/**
- * @brief Prepares the XMF file for the new entry
- *
- * Creates a temporary file on the disk in order to copy the right lines.
- *
- * @todo Use a proper XML library to avoid stupid copies.
- */
-
-/**
- * @brief Prepare the DM particles (in gparts) read in for the addition of the
- * other particle types
- *
- * This function assumes that the DM particles are all at the start of the
- * gparts array
- *
- * @param gparts The array of #gpart freshly read in.
- * @param Ndm The number of DM particles read in.
- */
-void io_prepare_dm_gparts(struct gpart* const gparts, size_t Ndm) {
+  struct gpart* restrict gparts = (struct gpart*)data;
 
   /* Let's give all these gparts a negative id */
-  for (size_t i = 0; i < Ndm; ++i) {
+  for (int i = 0; i < Ndm; ++i) {
+
     /* 0 or negative ids are not allowed */
     if (gparts[i].id_or_neg_offset <= 0)
-      error("0 or negative ID for DM particle %zu: ID=%lld", i,
+      error("0 or negative ID for DM particle %i: ID=%lld", i,
             gparts[i].id_or_neg_offset);
 
     /* Set gpart type */
@@ -649,22 +628,43 @@ void io_prepare_dm_gparts(struct gpart* const gparts, size_t Ndm) {
 }
 
 /**
- * @brief Copy every #part into the corresponding #gpart and link them.
+ * @brief Prepare the DM particles (in gparts) read in for the addition of the
+ * other particle types
  *
  * This function assumes that the DM particles are all at the start of the
- * gparts array and adds the hydro particles afterwards
+ * gparts array
  *
- * @param parts The array of #part freshly read in.
- * @param gparts The array of #gpart freshly read in with all the DM particles
- * at the start
- * @param Ngas The number of gas particles read in.
+ * @param tp The current #threadpool.
+ * @param gparts The array of #gpart freshly read in.
  * @param Ndm The number of DM particles read in.
  */
-void io_duplicate_hydro_gparts(struct part* const parts,
-                               struct gpart* const gparts, size_t Ngas,
-                               size_t Ndm) {
+void io_prepare_dm_gparts(struct threadpool* tp, struct gpart* const gparts,
+                          size_t Ndm) {
+
+  threadpool_map(tp, io_prepare_dm_gparts_mapper, gparts, Ndm,
+                 sizeof(struct gpart), 0, NULL);
+}
+
+struct duplication_data {
 
-  for (size_t i = 0; i < Ngas; ++i) {
+  struct part* parts;
+  struct gpart* gparts;
+  struct spart* sparts;
+  int Ndm;
+  int Ngas;
+  int Nstars;
+};
+
+void io_duplicate_hydro_gparts_mapper(void* restrict data, int Ngas,
+                                      void* restrict extra_data) {
+
+  struct duplication_data* temp = (struct duplication_data*)extra_data;
+  const int Ndm = temp->Ndm;
+  struct part* parts = (struct part*)data;
+  const ptrdiff_t offset = parts - temp->parts;
+  struct gpart* gparts = temp->gparts + offset;
+
+  for (int i = 0; i < Ngas; ++i) {
 
     /* Duplicate the crucial information */
     gparts[i + Ndm].x[0] = parts[i].x[0];
@@ -687,22 +687,40 @@ void io_duplicate_hydro_gparts(struct part* const parts,
 }
 
 /**
- * @brief Copy every #spart into the corresponding #gpart and link them.
+ * @brief Copy every #part into the corresponding #gpart and link them.
  *
- * This function assumes that the DM particles and gas particles are all at
- * the start of the gparts array and adds the star particles afterwards
+ * This function assumes that the DM particles are all at the start of the
+ * gparts array and adds the hydro particles afterwards
  *
- * @param sparts The array of #spart freshly read in.
- * @param gparts The array of #gpart freshly read in with all the DM and gas
- * particles at the start.
- * @param Nstars The number of stars particles read in.
- * @param Ndm The number of DM and gas particles read in.
+ * @param tp The current #threadpool.
+ * @param parts The array of #part freshly read in.
+ * @param gparts The array of #gpart freshly read in with all the DM particles
+ * at the start
+ * @param Ngas The number of gas particles read in.
+ * @param Ndm The number of DM particles read in.
  */
-void io_duplicate_star_gparts(struct spart* const sparts,
-                              struct gpart* const gparts, size_t Nstars,
-                              size_t Ndm) {
+void io_duplicate_hydro_gparts(struct threadpool* tp, struct part* const parts,
+                               struct gpart* const gparts, size_t Ngas,
+                               size_t Ndm) {
+  struct duplication_data data;
+  data.parts = parts;
+  data.gparts = gparts;
+  data.Ndm = Ndm;
+
+  threadpool_map(tp, io_duplicate_hydro_gparts_mapper, parts, Ngas,
+                 sizeof(struct part), 0, &data);
+}
 
-  for (size_t i = 0; i < Nstars; ++i) {
+void io_duplicate_hydro_sparts_mapper(void* restrict data, int Nstars,
+                                      void* restrict extra_data) {
+
+  struct duplication_data* temp = (struct duplication_data*)extra_data;
+  const int Ndm = temp->Ndm;
+  struct spart* sparts = (struct spart*)data;
+  const ptrdiff_t offset = sparts - temp->sparts;
+  struct gpart* gparts = temp->gparts + offset;
+
+  for (int i = 0; i < Nstars; ++i) {
 
     /* Duplicate the crucial information */
     gparts[i + Ndm].x[0] = sparts[i].x[0];
@@ -724,6 +742,32 @@ void io_duplicate_star_gparts(struct spart* const sparts,
   }
 }
 
+/**
+ * @brief Copy every #spart into the corresponding #gpart and link them.
+ *
+ * This function assumes that the DM particles and gas particles are all at
+ * the start of the gparts array and adds the star particles afterwards
+ *
+ * @param tp The current #threadpool.
+ * @param sparts The array of #spart freshly read in.
+ * @param gparts The array of #gpart freshly read in with all the DM and gas
+ * particles at the start.
+ * @param Nstars The number of stars particles read in.
+ * @param Ndm The number of DM and gas particles read in.
+ */
+void io_duplicate_star_gparts(struct threadpool* tp, struct spart* const sparts,
+                              struct gpart* const gparts, size_t Nstars,
+                              size_t Ndm) {
+
+  struct duplication_data data;
+  data.gparts = gparts;
+  data.sparts = sparts;
+  data.Ndm = Ndm;
+
+  threadpool_map(tp, io_duplicate_hydro_sparts_mapper, sparts, Nstars,
+                 sizeof(struct spart), 0, &data);
+}
+
 /**
  * @brief Copy every DM #gpart into the dmparts array.
  *

src/common_io.h

@@ -35,6 +35,7 @@
 /* Avoid cyclic inclusion problems */
 struct io_props;
 struct engine;
+struct threadpool;
 
 #if defined(HAVE_HDF5)
 
@@ -86,11 +87,12 @@ void io_copy_temp_buffer(void* temp, const struct engine* e,
 
 void io_collect_dm_gparts(const struct gpart* const gparts, size_t Ntot,
                           struct gpart* const dmparts, size_t Ndm);
-void io_prepare_dm_gparts(struct gpart* const gparts, size_t Ndm);
-void io_duplicate_hydro_gparts(struct part* const parts,
+void io_prepare_dm_gparts(struct threadpool* tp, struct gpart* const gparts,
+                          size_t Ndm);
+void io_duplicate_hydro_gparts(struct threadpool* tp, struct part* const parts,
                                struct gpart* const gparts, size_t Ngas,
                                size_t Ndm);
-void io_duplicate_star_gparts(struct spart* const sparts,
+void io_duplicate_star_gparts(struct threadpool* tp, struct spart* const sparts,
                               struct gpart* const gparts, size_t Nstars,
                               size_t Ndm);
 

src/gravity/Default/gravity_io.h

@@ -55,7 +55,7 @@ void darkmatter_read_particles(struct gpart* gparts, struct io_props* list,
                                 UNIT_CONV_SPEED, gparts, v_full);
   list[2] = io_make_input_field("Masses", FLOAT, 1, COMPULSORY, UNIT_CONV_MASS,
                                 gparts, mass);
-  list[3] = io_make_input_field("ParticleIDs", ULONGLONG, 1, COMPULSORY,
+  list[3] = io_make_input_field("ParticleIDs", LONGLONG, 1, COMPULSORY,
                                 UNIT_CONV_NO_UNITS, gparts, id_or_neg_offset);
 }
 

src/parallel_io.c

@@ -530,6 +530,7 @@ void writeArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile,
  * @param mpi_size The number of MPI ranks
  * @param comm The MPI communicator
  * @param info The MPI information object
+ * @param n_threads The number of threads to use for local operations.
  * @param dry_run If 1, don't read the particle. Only allocates the arrays.
  *
  */
@@ -539,7 +540,7 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units,
                       size_t* Nstars, int* periodic, int* flag_entropy,
                       int with_hydro, int with_gravity, int with_stars,
                       int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info,
-                      int dry_run) {
+                      int n_threads, int dry_run) {
 
   hid_t h_file = 0, h_grp = 0;
   /* GADGET has only cubic boxes (in cosmological mode) */
@@ -759,16 +760,24 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units,
     H5Gclose(h_grp);
   }
 
-  /* Prepare the DM particles */
-  if (!dry_run && with_gravity) io_prepare_dm_gparts(*gparts, Ndm);
+  if (!dry_run && with_gravity) {
 
-  /* Duplicate the hydro particles into gparts */
-  if (!dry_run && with_gravity && with_hydro)
-    io_duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm);
+    /* Let's initialise a bit of thread parallelism here */
+    struct threadpool tp;
+    threadpool_init(&tp, n_threads);
 
-  /* Duplicate the star particles into gparts */
-  if (!dry_run && with_gravity && with_stars)
-    io_duplicate_star_gparts(*sparts, *gparts, *Nstars, Ndm + *Ngas);
+    /* Prepare the DM particles */
+    io_prepare_dm_gparts(&tp, *gparts, Ndm);
+
+    /* Duplicate the hydro particles into gparts */
+    if (with_hydro) io_duplicate_hydro_gparts(&tp, *parts, *gparts, *Ngas, Ndm);
+
+    /* Duplicate the star particles into gparts */
+    if (with_stars)
+      io_duplicate_star_gparts(&tp, *sparts, *gparts, *Nstars, Ndm + *Ngas);
+
+    threadpool_clean(&tp);
+  }
 
   /* message("Done Reading particles..."); */
 

src/parallel_io.h

@@ -40,7 +40,7 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units,
                       size_t* Nsparts, int* periodic, int* flag_entropy,
                       int with_hydro, int with_gravity, int with_stars,
                       int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info,
-                      int dry_run);
+                      int nr_threads, int dry_run);
 
 void write_output_parallel(struct engine* e, const char* baseName,
                            const struct unit_system* internal_units,

src/serial_io.c

@@ -380,6 +380,7 @@ void writeArray(const struct engine* e, hid_t grp, char* fileName,
  * @param mpi_size The number of MPI ranks
  * @param comm The MPI communicator
  * @param info The MPI information object
+ * @param n_threads The number of threads to use for local operations.
  * @param dry_run If 1, don't read the particle. Only allocates the arrays.
  *
  * Opens the HDF5 file fileName and reads the particles contained
@@ -396,7 +397,7 @@ void read_ic_serial(char* fileName, const struct unit_system* internal_units,
                     size_t* Nstars, int* periodic, int* flag_entropy,
                     int with_hydro, int with_gravity, int with_stars,
                     int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info,
-                    int dry_run) {
+                    int n_threads, int dry_run) {
 
   hid_t h_file = 0, h_grp = 0;
   /* GADGET has only cubic boxes (in cosmological mode) */
@@ -647,16 +648,25 @@ void read_ic_serial(char* fileName, const struct unit_system* internal_units,
     MPI_Barrier(comm);
   }
 
-  /* Prepare the DM particles */
-  if (!dry_run && with_gravity) io_prepare_dm_gparts(*gparts, Ndm);
+  /* Duplicate the parts for gravity */
+  if (!dry_run && with_gravity) {
 
-  /* Duplicate the hydro particles into gparts */
-  if (!dry_run && with_gravity && with_hydro)
-    io_duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm);
+    /* Let's initialise a bit of thread parallelism here */
+    struct threadpool tp;
+    threadpool_init(&tp, n_threads);
 
-  /* Duplicate the star particles into gparts */
-  if (!dry_run && with_gravity && with_stars)
-    io_duplicate_star_gparts(*sparts, *gparts, *Nstars, Ndm + *Ngas);
+    /* Prepare the DM particles */
+    io_prepare_dm_gparts(&tp, *gparts, Ndm);
+
+    /* Duplicate the hydro particles into gparts */
+    if (with_hydro) io_duplicate_hydro_gparts(&tp, *parts, *gparts, *Ngas, Ndm);
+
+    /* Duplicate the star particles into gparts */
+    if (with_stars)
+      io_duplicate_star_gparts(&tp, *sparts, *gparts, *Nstars, Ndm + *Ngas);
+
+    threadpool_clean(&tp);
+  }
 
   /* message("Done Reading particles..."); */
 

src/serial_io.h

@@ -40,7 +40,7 @@ void read_ic_serial(char* fileName, const struct unit_system* internal_units,
                     size_t* Nstars, int* periodic, int* flag_entropy,
                     int with_hydro, int with_gravity, int with_stars,
                     int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info,
-                    int dry_run);
+                    int nr_threads, int dry_run);
 
 void write_output_serial(struct engine* e, const char* baseName,
                          const struct unit_system* internal_units,

src/single_io.c

@@ -310,7 +310,7 @@ void read_ic_single(char* fileName, const struct unit_system* internal_units,
                     struct spart** sparts, size_t* Ngas, size_t* Ngparts,
                     size_t* Nstars, int* periodic, int* flag_entropy,
                     int with_hydro, int with_gravity, int with_stars,
-                    int dry_run) {
+                    int n_threads, int dry_run) {
 
   hid_t h_file = 0, h_grp = 0;
   /* GADGET has only cubic boxes (in cosmological mode) */
@@ -515,16 +515,25 @@ void read_ic_single(char* fileName, const struct unit_system* internal_units,
     H5Gclose(h_grp);
   }
 
-  /* Prepare the DM particles */
-  if (!dry_run && with_gravity) io_prepare_dm_gparts(*gparts, Ndm);
+  /* Duplicate the parts for gravity */
+  if (!dry_run && with_gravity) {
 
-  /* Duplicate the hydro particles into gparts */
-  if (!dry_run && with_gravity && with_hydro)
-    io_duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm);
+    /* Let's initialise a bit of thread parallelism here */
+    struct threadpool tp;
+    threadpool_init(&tp, n_threads);
 
-  /* Duplicate the star particles into gparts */
-  if (!dry_run && with_gravity && with_stars)
-    io_duplicate_star_gparts(*sparts, *gparts, *Nstars, Ndm + *Ngas);
+    /* Prepare the DM particles */
+    io_prepare_dm_gparts(&tp, *gparts, Ndm);
+
+    /* Duplicate the hydro particles into gparts */
+    if (with_hydro) io_duplicate_hydro_gparts(&tp, *parts, *gparts, *Ngas, Ndm);
+
+    /* Duplicate the star particles into gparts */
+    if (with_stars)
+      io_duplicate_star_gparts(&tp, *sparts, *gparts, *Nstars, Ndm + *Ngas);
+
+    threadpool_clean(&tp);
+  }
 
   /* message("Done Reading particles..."); */
 

src/single_io.h

@@ -34,7 +34,7 @@ void read_ic_single(char* fileName, const struct unit_system* internal_units,
                     struct spart** sparts, size_t* Ngas, size_t* Ndm,
                     size_t* Nstars, int* periodic, int* flag_entropy,
                     int with_hydro, int with_gravity, int with_stars,
-                    int dry_run);
+                    int nr_threads, int dry_run);
 
 void write_output_single(struct engine* e, const char* baseName,
                          const struct unit_system* internal_units,

tests/testReading.c

@@ -45,7 +45,7 @@ int main() {
 
   /* Read data */
   read_ic_single("input.hdf5", &us, dim, &parts, &gparts, &sparts, &Ngas,
-                 &Ngpart, &Nspart, &periodic, &flag_entropy_ICs, 1, 1, 0, 0);
+                 &Ngpart, &Nspart, &periodic, &flag_entropy_ICs, 1, 1, 0, 1, 0);
 
   /* Check global properties read are correct */
   assert(dim[0] == boxSize);