diff --git a/doc/RTD/source/ImplementationDetails/index.rst b/doc/RTD/source/ImplementationDetails/index.rst
new file mode 100644
index 0000000000000000000000000000000000000000..dda984293e7cce7aeffd9886be8c4a633ee03c14
--- /dev/null
+++ b/doc/RTD/source/ImplementationDetails/index.rst
@@ -0,0 +1,41 @@
+.. Implementation details
+ Loic Hausammann, 2020
+ Matthieu Schaller, 2020
+
+.. _implementation_details:
+
+Implementation details
+======================
+
+Generating new unique IDs
+-------------------------
+
+When spawning new particles (not converting them) for star formation or other
+similar processes, the code needs to create new unique particle IDs. This is
+implemented in the file ``space_unique_id.c`` and can be switched on/off in the
+star formation file ``star_formation_struct.h`` by setting the variable
+``star_formation_need_unique_id`` to 1 or 0.
+
+The generation of new IDs is done by computing the maximal ID present in the
+initial condition (across all particle types) and then attributing two batches
+of new, unused IDs to each MPI rank. The size of each batch is computed in the
+same way as the count of extra particles in order to ensure that we will have
+enough available IDs between two tree rebuilds (where the extra particles are
+regenerated).
+
+When a new ID is requested, the next available ID in the first batch is
+returned. If the last available ID in the first batch is requested, we switch to
+the next batch of IDs and flag it for regeneration at the next rebuild time. If
+the second batch is also fully used, the code will exit with an error message
+[#f1]_. At each tree-rebuild steps, the ranks will request a new batch if
+required and make sure the batches are unique across all MPI ranks.
+
+As we are using the maximal ID from the ICs, nothing can be done against the user
+providing the maximal integer possible as an ID (that can for instance be the
+case in some of the EAGLE ICs as the ID encode their Lagrangian position on a
+Peano-Hilbert curve).
+
+
+.. [#f1] Thanks to the size of the fresh ID batches, the code should run out of
+ extra particles before reaching this point and triggered a new rebuild
+ if this is allowed by the star formation scheme.
diff --git a/doc/RTD/source/index.rst b/doc/RTD/source/index.rst
index d0493e69d0f160889836bb20ae13ad3a1ff592cc..2f5a567456ea2230ce4f343c6e4358fc04118cbb 100644
--- a/doc/RTD/source/index.rst
+++ b/doc/RTD/source/index.rst
@@ -32,3 +32,4 @@ difference is the parameter file that will need to be adapted for SWIFT.
VELOCIraptorInterface/index
AnalysisTools/index
Logger/index
+ ImplementationDetails/index
diff --git a/examples/main.c b/examples/main.c
index 2528ce589ef7e0eca09181d5976329c9f193bf83..e197830459c6b3ba7c888c66527871b91c1d988a 100644
--- a/examples/main.c
+++ b/examples/main.c
@@ -1058,7 +1058,7 @@ int main(int argc, char *argv[]) {
space_init(&s, params, &cosmo, dim, parts, gparts, sparts, bparts, Ngas,
Ngpart, Nspart, Nbpart, periodic, replicate, generate_gas_in_ics,
with_hydro, with_self_gravity, with_star_formation,
- with_DM_background_particles, talking, dry_run);
+ with_DM_background_particles, talking, dry_run, nr_nodes);
if (myrank == 0) {
clocks_gettime(&toc);
diff --git a/src/Makefile.am b/src/Makefile.am
index f1f9157b1881a673456f7dbf394aa32e09b35379..2f73608f2ee927d890c20be38737ec448a14cedc 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -57,7 +57,8 @@ include_HEADERS = space.h runner.h queue.h task.h lock.h cell.h part.h const.h \
pressure_floor.h pressure_floor_struct.h pressure_floor_iact.h \
velociraptor_struct.h velociraptor_io.h random.h memuse.h mpiuse.h memuse_rnodes.h \
black_holes.h black_holes_io.h black_holes_properties.h black_holes_struct.h \
- feedback.h feedback_struct.h feedback_properties.h task_order.h
+ feedback.h feedback_struct.h feedback_properties.h task_order.h \
+ space_unique_id.h
# source files for EAGLE cooling
QLA_COOLING_SOURCES =
@@ -107,7 +108,7 @@ AM_SOURCES = space.c runner_main.c runner_doiact_hydro.c runner_doiact_limiter.c
collectgroup.c hydro_space.c equation_of_state.c \
chemistry.c cosmology.c restart.c mesh_gravity.c velociraptor_interface.c \
outputlist.c velociraptor_dummy.c logger_io.c memuse.c mpiuse.c memuse_rnodes.c fof.c \
- hashmap.c pressure_floor.c \
+ hashmap.c pressure_floor.c space_unique_id.c \
$(QLA_COOLING_SOURCES) \
$(EAGLE_COOLING_SOURCES) $(EAGLE_FEEDBACK_SOURCES) \
$(GRACKLE_COOLING_SOURCES) $(GEAR_FEEDBACK_SOURCES)
diff --git a/src/cell.c b/src/cell.c
index c4a4ecc1872934da79f98ac2fd8e30d85cb988ea..0e0a65f23446816efde86c7fae416fbd84c5091b 100644
--- a/src/cell.c
+++ b/src/cell.c
@@ -6149,8 +6149,8 @@ struct spart *cell_spawn_new_spart_from_part(struct engine *e, struct cell *c,
/* Copy the gpart */
*gp = *p->gpart;
- /* Assign the ID back */
- sp->id = p->id;
+ /* Assign the ID. */
+ sp->id = space_get_new_unique_id(e->s);
gp->type = swift_type_stars;
/* Re-link things */
diff --git a/src/space.c b/src/space.c
index 9b20f33e9a41c6d6a56fce7ee1d6ce0fee6b62ba..f336e88e1bb8f062888cdc9b867e4d6513f85ebb 100644
--- a/src/space.c
+++ b/src/space.c
@@ -60,6 +60,7 @@
#include "proxy.h"
#include "restart.h"
#include "sort_part.h"
+#include "space_unique_id.h"
#include "star_formation.h"
#include "star_formation_logger.h"
#include "stars.h"
@@ -1817,6 +1818,9 @@ void space_rebuild(struct space *s, int repartitioned, int verbose) {
swift_free("b_index", b_index);
swift_free("cell_bpart_counts", cell_bpart_counts);
+ /* Update the slice of unique IDs. */
+ space_update_unique_id(s);
+
#ifdef WITH_MPI
/* Re-allocate the index array for the gparts if needed.. */
@@ -4789,6 +4793,7 @@ void space_convert_quantities(struct space *s, int verbose) {
* @param DM_background Are we running with some DM background particles?
* @param verbose Print messages to stdout or not.
* @param dry_run If 1, just initialise stuff, don't do anything with the parts.
+ * @param nr_nodes The number of MPI rank.
*
* Makes a grid of edge length > r_max and fills the particles
* into the respective cells. Cells containing more than #space_splitsize
@@ -4801,8 +4806,8 @@ void space_init(struct space *s, struct swift_params *params,
struct bpart *bparts, size_t Npart, size_t Ngpart,
size_t Nspart, size_t Nbpart, int periodic, int replicate,
int generate_gas_in_ics, int hydro, int self_gravity,
- int star_formation, int DM_background, int verbose,
- int dry_run) {
+ int star_formation, int DM_background, int verbose, int dry_run,
+ int nr_nodes) {
/* Clean-up everything */
bzero(s, sizeof(struct space));
@@ -5082,11 +5087,17 @@ void space_init(struct space *s, struct swift_params *params,
#endif
/* Do we want any spare particles for on the fly creation? */
- if (!star_formation || !swift_star_formation_model_creates_stars)
+ if (!star_formation || !swift_star_formation_model_creates_stars) {
space_extra_sparts = 0;
+ }
/* Build the cells recursively. */
if (!dry_run) space_regrid(s, verbose);
+
+ /* Compute the max id for the generation of unique id. */
+ if (star_formation && swift_star_formation_model_creates_stars) {
+ space_init_unique_id(s, nr_nodes);
+ }
}
/**
@@ -5735,6 +5746,9 @@ void space_clean(struct space *s) {
swift_free("gparts_foreign", s->gparts_foreign);
swift_free("bparts_foreign", s->bparts_foreign);
#endif
+
+ if (lock_destroy(&s->unique_id.lock) != 0)
+ error("Failed to destroy spinlocks.");
}
/**
diff --git a/src/space.h b/src/space.h
index acb55bd26128364c69f56dc557a5877115e30804..b7ba589482d8bd0a464cc6d3cc01076e1f4d2442 100644
--- a/src/space.h
+++ b/src/space.h
@@ -34,6 +34,7 @@
#include "lock.h"
#include "parser.h"
#include "part.h"
+#include "space_unique_id.h"
#include "velociraptor_struct.h"
/* Avoid cyclic inclusions */
@@ -277,6 +278,9 @@ struct space {
/*! The group information returned by VELOCIraptor for each #gpart. */
struct velociraptor_gpart_data *gpart_group_data;
+ /*! Structure dealing with the computation of a unique ID */
+ struct unique_id unique_id;
+
#ifdef WITH_MPI
/*! Buffers for parts that we will receive from foreign cells. */
@@ -316,8 +320,8 @@ void space_init(struct space *s, struct swift_params *params,
struct bpart *bparts, size_t Npart, size_t Ngpart,
size_t Nspart, size_t Nbpart, int periodic, int replicate,
int generate_gas_in_ics, int hydro, int gravity,
- int star_formation, int DM_background, int verbose,
- int dry_run);
+ int star_formation, int DM_background, int verbose, int dry_run,
+ int nr_nodes);
void space_sanitize(struct space *s);
void space_map_cells_pre(struct space *s, int full,
void (*fun)(struct cell *c, void *data), void *data);
diff --git a/src/space_unique_id.c b/src/space_unique_id.c
new file mode 100644
index 0000000000000000000000000000000000000000..3a6f0c48fb6d092cdd8e7613b2ff27027278913f
--- /dev/null
+++ b/src/space_unique_id.c
@@ -0,0 +1,247 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2020 Loic Hausammann (loic.hausammann@epfl.ch)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see .
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* Some standard headers. */
+#include
+
+/* MPI headers. */
+#ifdef WITH_MPI
+#include
+#endif
+
+/* This object's header. */
+#include "space_unique_id.h"
+
+/* Local headers. */
+#include "engine.h"
+#include "lock.h"
+#include "space.h"
+
+/**
+ * @brief Update the unique id structure by requesting a
+ * new batch if required.
+ *
+ * @param s The #space.
+ */
+void space_update_unique_id(struct space *s) {
+ /* Do we need unique IDs? */
+ if (!star_formation_need_unique_id) {
+ return;
+ }
+
+ const int require_new_batch = s->unique_id.next_batch.current == 0;
+
+#ifdef WITH_MPI
+ const struct engine *e = s->e;
+
+ /* Check if the other ranks need a batch. */
+ int *all_requires = (int *)malloc(sizeof(int) * e->nr_nodes);
+
+ /* Do the communication */
+ MPI_Allgather(&require_new_batch, 1, MPI_INT, all_requires, 1, MPI_INT,
+ MPI_COMM_WORLD);
+
+ /* Compute the position of this rank batch and the position of
+ the next free batch. */
+ int local_index = 0;
+ int total_shift = 0;
+ for (int i = 0; i < e->nr_nodes; i++) {
+ total_shift += all_requires[i];
+ if (i < e->nodeID) {
+ local_index += all_requires[i];
+ }
+ }
+
+ /* Free the allocated resources. */
+ free(all_requires);
+
+#else
+
+ int local_index = 0;
+ int total_shift = require_new_batch;
+
+#endif // WITH_MPI
+
+ /* Compute the size of each batch. */
+ const long long batch_size = (space_extra_parts + space_extra_sparts +
+ space_extra_gparts + space_extra_bparts) *
+ s->nr_cells;
+
+ /* Get a new batch. */
+ if (require_new_batch) {
+ /* First check against an overflow. */
+ const long long local_shift = local_index * batch_size;
+ if (s->unique_id.global_next_id > LLONG_MAX - (local_shift + batch_size)) {
+ error("Overflow for the unique IDs.");
+ }
+ /* Now assign it. */
+ s->unique_id.next_batch.current = s->unique_id.global_next_id + local_shift;
+ s->unique_id.next_batch.max =
+ s->unique_id.global_next_id + local_shift + batch_size;
+ }
+
+ /* Shift the position of the next available batch. */
+ const long long shift = total_shift * batch_size;
+ if (s->unique_id.global_next_id > LLONG_MAX - shift) {
+ error("Overflow for the unique IDs.");
+ }
+ s->unique_id.global_next_id += shift;
+}
+
+/**
+ * @brief Get a new unique ID.
+ *
+ * @param s the #space.
+ *
+ * @return The new unique ID
+ */
+long long space_get_new_unique_id(struct space *s) {
+ /* Do we need unique IDs? */
+ if (!star_formation_need_unique_id) {
+ error("The scheme selected does not seem to use unique ID.");
+ }
+
+ /* Get the lock. */
+ lock_lock(&s->unique_id.lock);
+
+ /* Get the current available id. */
+ const long long id = s->unique_id.current_batch.current;
+
+ /* Update the counter. */
+ s->unique_id.current_batch.current++;
+
+ /* Check if everything is fine */
+ if (s->unique_id.current_batch.current > s->unique_id.current_batch.max) {
+ error("Failed to get a new ID");
+ }
+
+ /* Check if need to move to the next batch. */
+ else if (s->unique_id.current_batch.current ==
+ s->unique_id.current_batch.max) {
+
+ /* Check if the next batch is already used */
+ if (s->unique_id.next_batch.current == 0) {
+ error("Failed to obtain a new unique ID.");
+ }
+
+ s->unique_id.current_batch = s->unique_id.next_batch;
+
+ /* Reset the next batch. */
+ s->unique_id.next_batch.current = 0;
+ s->unique_id.next_batch.max = 0;
+ }
+
+ /* Release the lock. */
+ if (lock_unlock(&s->unique_id.lock) != 0) {
+ error("Impossible to unlock the unique id.");
+ }
+
+ return id;
+}
+
+/**
+ * @brief Initialize the computation of unique IDs.
+ *
+ * @param s The #space.
+ * @param nr_nodes The number of MPI ranks.
+ */
+void space_init_unique_id(struct space *s, int nr_nodes) {
+ /* Do we need unique IDs? */
+ if (!star_formation_need_unique_id) {
+ return;
+ }
+
+ /* Set the counter to 0. */
+ s->unique_id.global_next_id = 0;
+
+ /* Check the parts for the max id. */
+ for (size_t i = 0; i < s->nr_parts; i++) {
+ s->unique_id.global_next_id =
+ max(s->unique_id.global_next_id, s->parts[i].id);
+ }
+
+ /* Check the gparts for the max id. */
+ for (size_t i = 0; i < s->nr_gparts; i++) {
+ s->unique_id.global_next_id =
+ max(s->unique_id.global_next_id, s->gparts[i].id_or_neg_offset);
+ }
+
+ /* Check the sparts for the max id. */
+ for (size_t i = 0; i < s->nr_sparts; i++) {
+ s->unique_id.global_next_id =
+ max(s->unique_id.global_next_id, s->sparts[i].id);
+ }
+
+ /* Check the bparts for the max id. */
+ for (size_t i = 0; i < s->nr_bparts; i++) {
+ s->unique_id.global_next_id =
+ max(s->unique_id.global_next_id, s->bparts[i].id);
+ }
+
+#ifdef WITH_MPI
+ /* Find the global max. */
+ MPI_Allreduce(MPI_IN_PLACE, &s->unique_id.global_next_id, 1, MPI_LONG_LONG,
+ MPI_MAX, MPI_COMM_WORLD);
+#endif // WITH_MPI
+
+ /* Get the first unique id. */
+ if (s->unique_id.global_next_id == LLONG_MAX) {
+ error("Overflow for the unique id.");
+ }
+ s->unique_id.global_next_id++;
+
+ /* Compute the size of each batch. */
+ const long long batch_size = (space_extra_parts + space_extra_sparts +
+ space_extra_gparts + space_extra_bparts) *
+ s->nr_cells;
+
+ /* Compute the initial batchs (each rank has 2 batchs). */
+ if (s->unique_id.global_next_id > LLONG_MAX - 2 * engine_rank * batch_size) {
+ error("Overflow for the unique id.");
+ }
+ const long long init =
+ s->unique_id.global_next_id + 2 * engine_rank * batch_size;
+
+ /* Set the batchs and check for overflows. */
+ s->unique_id.current_batch.current = init;
+
+ if (init > LLONG_MAX - batch_size) {
+ error("Overflow for the unique id.");
+ }
+ s->unique_id.current_batch.max = init + batch_size;
+ s->unique_id.next_batch.current = s->unique_id.current_batch.max;
+
+ if (s->unique_id.next_batch.current > LLONG_MAX - batch_size) {
+ error("Overflow for the unique id.");
+ }
+ s->unique_id.next_batch.max = s->unique_id.next_batch.current + batch_size;
+
+ /* Update the next available id */
+ if (s->unique_id.global_next_id > LLONG_MAX - 2 * batch_size * nr_nodes) {
+ error("Overflow for the unique id.");
+ }
+ s->unique_id.global_next_id += 2 * batch_size * nr_nodes;
+
+ /* Initialize the lock. */
+ if (lock_init(&s->unique_id.lock) != 0)
+ error("Failed to init spinlock for the unique ids.");
+}
diff --git a/src/space_unique_id.h b/src/space_unique_id.h
new file mode 100644
index 0000000000000000000000000000000000000000..49ec49d2bdc09b46451a4decdadd6dce046d5a20
--- /dev/null
+++ b/src/space_unique_id.h
@@ -0,0 +1,62 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2020 Loic Hausammann (loic.hausammann@epfl.ch)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see .
+ *
+ ******************************************************************************/
+
+#ifndef SWIFT_SPACE_UNIQUE_ID_H
+#define SWIFT_SPACE_UNIQUE_ID_H
+
+/* Config parameters. */
+#include "../config.h"
+
+/* Local includes */
+#include "lock.h"
+
+/* Predefine the space structure */
+struct space;
+
+/**
+ * @brief Batch of unique IDs for particle creation.
+ */
+struct batch {
+ /*! Current free unique id */
+ long long current;
+
+ /*! Maximal unique id in this batch (not included) */
+ long long max;
+};
+
+/*! Structure dealing with the computation of a unique ID */
+struct unique_id {
+ /*! Current batch of unique ids */
+ struct batch current_batch;
+
+ /*! Next batch of unique ids */
+ struct batch next_batch;
+
+ /* Global next slot available */
+ long long global_next_id;
+
+ /* Lock for the unique ids */
+ swift_lock_type lock;
+};
+
+void space_update_unique_id(struct space *s);
+long long space_get_new_unique_id(struct space *s);
+void space_init_unique_id(struct space *s, int nr_nodes);
+
+#endif // SWIFT_SPACE_UNIQUE_ID_H
diff --git a/src/star_formation/EAGLE/star_formation_struct.h b/src/star_formation/EAGLE/star_formation_struct.h
index f5f0f76e8d59f73148f51f462cc619ec70e82192..556d7bea6fbaacb00bc3df0cc7f673ad89c02896 100644
--- a/src/star_formation/EAGLE/star_formation_struct.h
+++ b/src/star_formation/EAGLE/star_formation_struct.h
@@ -19,6 +19,10 @@
#ifndef SWIFT_EAGLE_STAR_FORMATION_STRUCT_H
#define SWIFT_EAGLE_STAR_FORMATION_STRUCT_H
+/* Do we need unique IDs (only useful when spawning
+ new particles, conversion gas->stars does not need unique IDs) */
+#define star_formation_need_unique_id 0
+
/**
* @brief Star-formation-related properties stored in the extended particle
* data.
diff --git a/src/star_formation/GEAR/star_formation_struct.h b/src/star_formation/GEAR/star_formation_struct.h
index c03a8ae0a4178a4a4fd372e6e62775893ae9862d..372a9ea8263ab2ad2b8804700ebce02a0f8b5f65 100644
--- a/src/star_formation/GEAR/star_formation_struct.h
+++ b/src/star_formation/GEAR/star_formation_struct.h
@@ -19,6 +19,10 @@
#ifndef SWIFT_GEAR_STAR_FORMATION_STRUCT_H
#define SWIFT_GEAR_STAR_FORMATION_STRUCT_H
+/* Do we need unique IDs (only useful when spawning
+ new particles, conversion gas->stars does not need unique IDs) */
+#define star_formation_need_unique_id 1
+
/**
* @brief Star-formation-related properties stored in the extended particle
* data.
diff --git a/src/star_formation/QLA/star_formation_struct.h b/src/star_formation/QLA/star_formation_struct.h
index 872577e0f5f161c58acc460e53884d2ab642206a..034885726a71c644ffec51a1dfd6820ee9e9a657 100644
--- a/src/star_formation/QLA/star_formation_struct.h
+++ b/src/star_formation/QLA/star_formation_struct.h
@@ -19,6 +19,10 @@
#ifndef SWIFT_QLA_STAR_FORMATION_STRUCT_H
#define SWIFT_QLA_STAR_FORMATION_STRUCT_H
+/* Do we need unique IDs (only useful when spawning
+ new particles, conversion gas->stars does not need unique IDs) */
+#define star_formation_need_unique_id 1
+
/**
* @brief Star-formation-related properties stored in the extended particle
* data.
diff --git a/src/star_formation/none/star_formation_struct.h b/src/star_formation/none/star_formation_struct.h
index 27a2adaf83d0a02a0d08e7eef8b45bea630689e4..c1596c1d4aed318beee3da1626502c2970c5ff90 100644
--- a/src/star_formation/none/star_formation_struct.h
+++ b/src/star_formation/none/star_formation_struct.h
@@ -19,6 +19,10 @@
#ifndef SWIFT_NONE_STAR_FORMATION_STRUCT_H
#define SWIFT_NONE_STAR_FORMATION_STRUCT_H
+/* Do we need unique IDs (only useful when spawning
+ new particles, conversion gas->stars does not need unique IDs) */
+#define star_formation_need_unique_id 0
+
/**
* @brief Star-formation-related properties stored in the extended particle
* data.