Commit a3f65931 authored by Matthieu Schaller's avatar Matthieu Schaller
Browse files

Make use of a temporary threadpool in the reading routines to speed-up the...

Make use of a temporary threadpool in the reading routines to speed-up the duplication of parts into gparts.
parent 5e35e0b7
...@@ -505,18 +505,20 @@ int main(int argc, char *argv[]) { ...@@ -505,18 +505,20 @@ int main(int argc, char *argv[]) {
read_ic_parallel(ICfileName, &us, dim, &parts, &gparts, &sparts, &Ngas, read_ic_parallel(ICfileName, &us, dim, &parts, &gparts, &sparts, &Ngas,
&Ngpart, &Nspart, &periodic, &flag_entropy_ICs, with_hydro, &Ngpart, &Nspart, &periodic, &flag_entropy_ICs, with_hydro,
(with_external_gravity || with_self_gravity), with_stars, (with_external_gravity || with_self_gravity), with_stars,
myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL, dry_run); myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL, nr_threads,
dry_run);
#else #else
read_ic_serial(ICfileName, &us, dim, &parts, &gparts, &sparts, &Ngas, &Ngpart, read_ic_serial(ICfileName, &us, dim, &parts, &gparts, &sparts, &Ngas, &Ngpart,
&Nspart, &periodic, &flag_entropy_ICs, with_hydro, &Nspart, &periodic, &flag_entropy_ICs, with_hydro,
(with_external_gravity || with_self_gravity), with_stars, (with_external_gravity || with_self_gravity), with_stars,
myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL, dry_run); myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL, nr_threads,
dry_run);
#endif #endif
#else #else
read_ic_single(ICfileName, &us, dim, &parts, &gparts, &sparts, &Ngas, &Ngpart, read_ic_single(ICfileName, &us, dim, &parts, &gparts, &sparts, &Ngas, &Ngpart,
&Nspart, &periodic, &flag_entropy_ICs, with_hydro, &Nspart, &periodic, &flag_entropy_ICs, with_hydro,
(with_external_gravity || with_self_gravity), with_stars, (with_external_gravity || with_self_gravity), with_stars,
dry_run); nr_threads, dry_run);
#endif #endif
if (myrank == 0) { if (myrank == 0) {
clocks_gettime(&toc); clocks_gettime(&toc);
......
...@@ -610,6 +610,23 @@ void io_copy_temp_buffer(void* temp, const struct engine* e, ...@@ -610,6 +610,23 @@ void io_copy_temp_buffer(void* temp, const struct engine* e,
} }
} }
void io_prepare_dm_gparts_mapper(void* restrict data, int Ndm, void* dummy) {
struct gpart* restrict gparts = (struct gpart*)data;
/* Let's give all these gparts a negative id */
for (int i = 0; i < Ndm; ++i) {
/* 0 or negative ids are not allowed */
if (gparts[i].id_or_neg_offset <= 0)
error("0 or negative ID for DM particle %i: ID=%lld", i,
gparts[i].id_or_neg_offset);
/* Set gpart type */
gparts[i].type = swift_type_dark_matter;
}
}
/** /**
* @brief Prepare the DM particles (in gparts) read in for the addition of the * @brief Prepare the DM particles (in gparts) read in for the addition of the
* other particle types * other particle types
...@@ -617,21 +634,15 @@ void io_copy_temp_buffer(void* temp, const struct engine* e, ...@@ -617,21 +634,15 @@ void io_copy_temp_buffer(void* temp, const struct engine* e,
* This function assumes that the DM particles are all at the start of the * This function assumes that the DM particles are all at the start of the
* gparts array * gparts array
* *
* @param tp The current #threadpool.
* @param gparts The array of #gpart freshly read in. * @param gparts The array of #gpart freshly read in.
* @param Ndm The number of DM particles read in. * @param Ndm The number of DM particles read in.
*/ */
void io_prepare_dm_gparts(struct gpart* const gparts, size_t Ndm) { void io_prepare_dm_gparts(struct threadpool* tp, struct gpart* const gparts,
size_t Ndm) {
/* Let's give all these gparts a negative id */
for (size_t i = 0; i < Ndm; ++i) {
/* 0 or negative ids are not allowed */
if (gparts[i].id_or_neg_offset <= 0)
error("0 or negative ID for DM particle %zu: ID=%lld", i,
gparts[i].id_or_neg_offset);
/* Set gpart type */ threadpool_map(tp, io_prepare_dm_gparts_mapper, gparts, Ndm,
gparts[i].type = swift_type_dark_matter; sizeof(struct gpart), 0, NULL);
}
} }
/** /**
......
...@@ -35,6 +35,7 @@ ...@@ -35,6 +35,7 @@
/* Avoid cyclic inclusion problems */ /* Avoid cyclic inclusion problems */
struct io_props; struct io_props;
struct engine; struct engine;
struct threadpool;
#if defined(HAVE_HDF5) #if defined(HAVE_HDF5)
...@@ -86,7 +87,8 @@ void io_copy_temp_buffer(void* temp, const struct engine* e, ...@@ -86,7 +87,8 @@ void io_copy_temp_buffer(void* temp, const struct engine* e,
void io_collect_dm_gparts(const struct gpart* const gparts, size_t Ntot, void io_collect_dm_gparts(const struct gpart* const gparts, size_t Ntot,
struct gpart* const dmparts, size_t Ndm); struct gpart* const dmparts, size_t Ndm);
void io_prepare_dm_gparts(struct gpart* const gparts, size_t Ndm); void io_prepare_dm_gparts(struct threadpool* tp, struct gpart* const gparts,
size_t Ndm);
void io_duplicate_hydro_gparts(struct part* const parts, void io_duplicate_hydro_gparts(struct part* const parts,
struct gpart* const gparts, size_t Ngas, struct gpart* const gparts, size_t Ngas,
size_t Ndm); size_t Ndm);
......
...@@ -539,7 +539,7 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units, ...@@ -539,7 +539,7 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units,
size_t* Nstars, int* periodic, int* flag_entropy, size_t* Nstars, int* periodic, int* flag_entropy,
int with_hydro, int with_gravity, int with_stars, int with_hydro, int with_gravity, int with_stars,
int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info, int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info,
int dry_run) { int n_threads, int dry_run) {
hid_t h_file = 0, h_grp = 0; hid_t h_file = 0, h_grp = 0;
/* GADGET has only cubic boxes (in cosmological mode) */ /* GADGET has only cubic boxes (in cosmological mode) */
...@@ -759,16 +759,24 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units, ...@@ -759,16 +759,24 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units,
H5Gclose(h_grp); H5Gclose(h_grp);
} }
/* Prepare the DM particles */ if (!dry_run && with_gravity) {
if (!dry_run && with_gravity) io_prepare_dm_gparts(*gparts, Ndm);
/* Duplicate the hydro particles into gparts */ /* Let's initialise a bit of thread parallelism here */
if (!dry_run && with_gravity && with_hydro) struct threadpool tp;
io_duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm); threadpool_init(&tp, 4);
/* Duplicate the star particles into gparts */ /* Prepare the DM particles */
if (!dry_run && with_gravity && with_stars) io_prepare_dm_gparts(*gparts, Ndm);
io_duplicate_star_gparts(*sparts, *gparts, *Nstars, Ndm + *Ngas);
/* Duplicate the hydro particles into gparts */
if (with_hydro) io_duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm);
/* Duplicate the star particles into gparts */
if (with_stars)
io_duplicate_star_gparts(*sparts, *gparts, *Nstars, Ndm + *Ngas);
threadpool_clean(&tp);
}
/* message("Done Reading particles..."); */ /* message("Done Reading particles..."); */
......
...@@ -40,7 +40,7 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units, ...@@ -40,7 +40,7 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units,
size_t* Nsparts, int* periodic, int* flag_entropy, size_t* Nsparts, int* periodic, int* flag_entropy,
int with_hydro, int with_gravity, int with_stars, int with_hydro, int with_gravity, int with_stars,
int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info, int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info,
int dry_run); int nr_threads, int dry_run);
void write_output_parallel(struct engine* e, const char* baseName, void write_output_parallel(struct engine* e, const char* baseName,
const struct unit_system* internal_units, const struct unit_system* internal_units,
......
...@@ -396,7 +396,7 @@ void read_ic_serial(char* fileName, const struct unit_system* internal_units, ...@@ -396,7 +396,7 @@ void read_ic_serial(char* fileName, const struct unit_system* internal_units,
size_t* Nstars, int* periodic, int* flag_entropy, size_t* Nstars, int* periodic, int* flag_entropy,
int with_hydro, int with_gravity, int with_stars, int with_hydro, int with_gravity, int with_stars,
int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info, int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info,
int dry_run) { int n_threads, int dry_run) {
hid_t h_file = 0, h_grp = 0; hid_t h_file = 0, h_grp = 0;
/* GADGET has only cubic boxes (in cosmological mode) */ /* GADGET has only cubic boxes (in cosmological mode) */
...@@ -647,16 +647,25 @@ void read_ic_serial(char* fileName, const struct unit_system* internal_units, ...@@ -647,16 +647,25 @@ void read_ic_serial(char* fileName, const struct unit_system* internal_units,
MPI_Barrier(comm); MPI_Barrier(comm);
} }
/* Prepare the DM particles */ /* Duplicate the parts for gravity */
if (!dry_run && with_gravity) io_prepare_dm_gparts(*gparts, Ndm); if (!dry_run && with_gravity) {
/* Duplicate the hydro particles into gparts */ /* Let's initialise a bit of thread parallelism here */
if (!dry_run && with_gravity && with_hydro) struct threadpool tp;
io_duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm); threadpool_init(&tp, n_threads);
/* Duplicate the star particles into gparts */ /* Prepare the DM particles */
if (!dry_run && with_gravity && with_stars) io_prepare_dm_gparts(&tp, *gparts, Ndm);
io_duplicate_star_gparts(*sparts, *gparts, *Nstars, Ndm + *Ngas);
/* Duplicate the hydro particles into gparts */
if (with_hydro) io_duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm);
/* Duplicate the star particles into gparts */
if (with_stars)
io_duplicate_star_gparts(*sparts, *gparts, *Nstars, Ndm + *Ngas);
threadpool_clean(&tp);
}
/* message("Done Reading particles..."); */ /* message("Done Reading particles..."); */
......
...@@ -40,7 +40,7 @@ void read_ic_serial(char* fileName, const struct unit_system* internal_units, ...@@ -40,7 +40,7 @@ void read_ic_serial(char* fileName, const struct unit_system* internal_units,
size_t* Nstars, int* periodic, int* flag_entropy, size_t* Nstars, int* periodic, int* flag_entropy,
int with_hydro, int with_gravity, int with_stars, int with_hydro, int with_gravity, int with_stars,
int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info, int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info,
int dry_run); int nr_threads, int dry_run);
void write_output_serial(struct engine* e, const char* baseName, void write_output_serial(struct engine* e, const char* baseName,
const struct unit_system* internal_units, const struct unit_system* internal_units,
......
...@@ -310,7 +310,7 @@ void read_ic_single(char* fileName, const struct unit_system* internal_units, ...@@ -310,7 +310,7 @@ void read_ic_single(char* fileName, const struct unit_system* internal_units,
struct spart** sparts, size_t* Ngas, size_t* Ngparts, struct spart** sparts, size_t* Ngas, size_t* Ngparts,
size_t* Nstars, int* periodic, int* flag_entropy, size_t* Nstars, int* periodic, int* flag_entropy,
int with_hydro, int with_gravity, int with_stars, int with_hydro, int with_gravity, int with_stars,
int dry_run) { int n_threads, int dry_run) {
hid_t h_file = 0, h_grp = 0; hid_t h_file = 0, h_grp = 0;
/* GADGET has only cubic boxes (in cosmological mode) */ /* GADGET has only cubic boxes (in cosmological mode) */
...@@ -515,16 +515,25 @@ void read_ic_single(char* fileName, const struct unit_system* internal_units, ...@@ -515,16 +515,25 @@ void read_ic_single(char* fileName, const struct unit_system* internal_units,
H5Gclose(h_grp); H5Gclose(h_grp);
} }
/* Prepare the DM particles */ /* Duplicate the parts for gravity */
if (!dry_run && with_gravity) io_prepare_dm_gparts(*gparts, Ndm); if (!dry_run && with_gravity) {
/* Duplicate the hydro particles into gparts */ /* Let's initialise a bit of thread parallelism here */
if (!dry_run && with_gravity && with_hydro) struct threadpool tp;
io_duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm); threadpool_init(&tp, n_threads);
/* Duplicate the star particles into gparts */ /* Prepare the DM particles */
if (!dry_run && with_gravity && with_stars) io_prepare_dm_gparts(&tp, *gparts, Ndm);
io_duplicate_star_gparts(*sparts, *gparts, *Nstars, Ndm + *Ngas);
/* Duplicate the hydro particles into gparts */
if (with_hydro) io_duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm);
/* Duplicate the star particles into gparts */
if (with_stars)
io_duplicate_star_gparts(*sparts, *gparts, *Nstars, Ndm + *Ngas);
threadpool_clean(&tp);
}
/* message("Done Reading particles..."); */ /* message("Done Reading particles..."); */
......
...@@ -34,7 +34,7 @@ void read_ic_single(char* fileName, const struct unit_system* internal_units, ...@@ -34,7 +34,7 @@ void read_ic_single(char* fileName, const struct unit_system* internal_units,
struct spart** sparts, size_t* Ngas, size_t* Ndm, struct spart** sparts, size_t* Ngas, size_t* Ndm,
size_t* Nstars, int* periodic, int* flag_entropy, size_t* Nstars, int* periodic, int* flag_entropy,
int with_hydro, int with_gravity, int with_stars, int with_hydro, int with_gravity, int with_stars,
int dry_run); int nr_threads, int dry_run);
void write_output_single(struct engine* e, const char* baseName, void write_output_single(struct engine* e, const char* baseName,
const struct unit_system* internal_units, const struct unit_system* internal_units,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment