Commit 9f4888c7 authored by Loic Hausammann's avatar Loic Hausammann Committed by Matthieu Schaller
Browse files

New unique ID

parent 3e6cbe0c
.. Implementation details
Loic Hausammann, 2020
Matthieu Schaller, 2020
.. _implementation_details:
Implementation details
======================
Generating new unique IDs
-------------------------
When spawning new particles (not converting them) for star formation or other
similar processes, the code needs to create new unique particle IDs. This is
implemented in the file ``space_unique_id.c`` and can be switched on/off in the
star formation file ``star_formation_struct.h`` by setting the variable
``star_formation_need_unique_id`` to 1 or 0.
The generation of new IDs is done by computing the maximal ID present in the
initial condition (across all particle types) and then attributing two batches
of new, unused IDs to each MPI rank. The size of each batch is computed in the
same way as the count of extra particles in order to ensure that we will have
enough available IDs between two tree rebuilds (where the extra particles are
regenerated).
When a new ID is requested, the next available ID in the first batch is
returned. If the last available ID in the first batch is requested, we switch to
the next batch of IDs and flag it for regeneration at the next rebuild time. If
the second batch is also fully used, the code will exit with an error message
[#f1]_. At each tree-rebuild steps, the ranks will request a new batch if
required and make sure the batches are unique across all MPI ranks.
As we are using the maximal ID from the ICs, nothing can be done against the user
providing the maximal integer possible as an ID (that can for instance be the
case in some of the EAGLE ICs as the ID encode their Lagrangian position on a
Peano-Hilbert curve).
.. [#f1] Thanks to the size of the fresh ID batches, the code should run out of
extra particles before reaching this point and triggered a new rebuild
if this is allowed by the star formation scheme.
......@@ -33,3 +33,4 @@ difference is the parameter file that will need to be adapted for SWIFT.
VELOCIraptorInterface/index
AnalysisTools/index
Logger/index
ImplementationDetails/index
......@@ -1058,7 +1058,7 @@ int main(int argc, char *argv[]) {
space_init(&s, params, &cosmo, dim, parts, gparts, sparts, bparts, Ngas,
Ngpart, Nspart, Nbpart, periodic, replicate, generate_gas_in_ics,
with_hydro, with_self_gravity, with_star_formation,
with_DM_background_particles, talking, dry_run);
with_DM_background_particles, talking, dry_run, nr_nodes);
if (myrank == 0) {
clocks_gettime(&toc);
......
......@@ -57,7 +57,8 @@ include_HEADERS = space.h runner.h queue.h task.h lock.h cell.h part.h const.h \
pressure_floor.h pressure_floor_struct.h pressure_floor_iact.h \
velociraptor_struct.h velociraptor_io.h random.h memuse.h mpiuse.h memuse_rnodes.h \
black_holes.h black_holes_io.h black_holes_properties.h black_holes_struct.h \
feedback.h feedback_struct.h feedback_properties.h task_order.h
feedback.h feedback_struct.h feedback_properties.h task_order.h \
space_unique_id.h
# source files for EAGLE cooling
QLA_COOLING_SOURCES =
......@@ -107,7 +108,7 @@ AM_SOURCES = space.c runner_main.c runner_doiact_hydro.c runner_doiact_limiter.c
collectgroup.c hydro_space.c equation_of_state.c \
chemistry.c cosmology.c restart.c mesh_gravity.c velociraptor_interface.c \
outputlist.c velociraptor_dummy.c logger_io.c memuse.c mpiuse.c memuse_rnodes.c fof.c \
hashmap.c pressure_floor.c \
hashmap.c pressure_floor.c space_unique_id.c \
$(QLA_COOLING_SOURCES) \
$(EAGLE_COOLING_SOURCES) $(EAGLE_FEEDBACK_SOURCES) \
$(GRACKLE_COOLING_SOURCES) $(GEAR_FEEDBACK_SOURCES)
......
......@@ -6173,8 +6173,8 @@ struct spart *cell_spawn_new_spart_from_part(struct engine *e, struct cell *c,
/* Copy the gpart */
*gp = *p->gpart;
/* Assign the ID back */
sp->id = p->id;
/* Assign the ID. */
sp->id = space_get_new_unique_id(e->s);
gp->type = swift_type_stars;
/* Re-link things */
......
......@@ -60,6 +60,7 @@
#include "proxy.h"
#include "restart.h"
#include "sort_part.h"
#include "space_unique_id.h"
#include "star_formation.h"
#include "star_formation_logger.h"
#include "stars.h"
......@@ -1818,6 +1819,9 @@ void space_rebuild(struct space *s, int repartitioned, int verbose) {
swift_free("b_index", b_index);
swift_free("cell_bpart_counts", cell_bpart_counts);
/* Update the slice of unique IDs. */
space_update_unique_id(s);
#ifdef WITH_MPI
/* Re-allocate the index array for the gparts if needed.. */
......@@ -4818,6 +4822,7 @@ void space_convert_quantities(struct space *s, int verbose) {
* @param DM_background Are we running with some DM background particles?
* @param verbose Print messages to stdout or not.
* @param dry_run If 1, just initialise stuff, don't do anything with the parts.
* @param nr_nodes The number of MPI rank.
*
* Makes a grid of edge length > r_max and fills the particles
* into the respective cells. Cells containing more than #space_splitsize
......@@ -4830,8 +4835,8 @@ void space_init(struct space *s, struct swift_params *params,
struct bpart *bparts, size_t Npart, size_t Ngpart,
size_t Nspart, size_t Nbpart, int periodic, int replicate,
int generate_gas_in_ics, int hydro, int self_gravity,
int star_formation, int DM_background, int verbose,
int dry_run) {
int star_formation, int DM_background, int verbose, int dry_run,
int nr_nodes) {
/* Clean-up everything */
bzero(s, sizeof(struct space));
......@@ -5111,11 +5116,17 @@ void space_init(struct space *s, struct swift_params *params,
#endif
/* Do we want any spare particles for on the fly creation? */
if (!star_formation || !swift_star_formation_model_creates_stars)
if (!star_formation || !swift_star_formation_model_creates_stars) {
space_extra_sparts = 0;
}
/* Build the cells recursively. */
if (!dry_run) space_regrid(s, verbose);
/* Compute the max id for the generation of unique id. */
if (star_formation && swift_star_formation_model_creates_stars) {
space_init_unique_id(s, nr_nodes);
}
}
/**
......@@ -5764,6 +5775,9 @@ void space_clean(struct space *s) {
swift_free("gparts_foreign", s->gparts_foreign);
swift_free("bparts_foreign", s->bparts_foreign);
#endif
if (lock_destroy(&s->unique_id.lock) != 0)
error("Failed to destroy spinlocks.");
}
/**
......
......@@ -34,6 +34,7 @@
#include "lock.h"
#include "parser.h"
#include "part.h"
#include "space_unique_id.h"
#include "velociraptor_struct.h"
/* Avoid cyclic inclusions */
......@@ -277,6 +278,9 @@ struct space {
/*! The group information returned by VELOCIraptor for each #gpart. */
struct velociraptor_gpart_data *gpart_group_data;
/*! Structure dealing with the computation of a unique ID */
struct unique_id unique_id;
#ifdef WITH_MPI
/*! Buffers for parts that we will receive from foreign cells. */
......@@ -316,8 +320,8 @@ void space_init(struct space *s, struct swift_params *params,
struct bpart *bparts, size_t Npart, size_t Ngpart,
size_t Nspart, size_t Nbpart, int periodic, int replicate,
int generate_gas_in_ics, int hydro, int gravity,
int star_formation, int DM_background, int verbose,
int dry_run);
int star_formation, int DM_background, int verbose, int dry_run,
int nr_nodes);
void space_sanitize(struct space *s);
void space_map_cells_pre(struct space *s, int full,
void (*fun)(struct cell *c, void *data), void *data);
......
/*******************************************************************************
* This file is part of SWIFT.
* Copyright (c) 2020 Loic Hausammann (loic.hausammann@epfl.ch)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
******************************************************************************/
/* Config parameters. */
#include "../config.h"
/* Some standard headers. */
#include <limits.h>
/* MPI headers. */
#ifdef WITH_MPI
#include <mpi.h>
#endif
/* This object's header. */
#include "space_unique_id.h"
/* Local headers. */
#include "engine.h"
#include "lock.h"
#include "space.h"
/**
* @brief Update the unique id structure by requesting a
* new batch if required.
*
* @param s The #space.
*/
void space_update_unique_id(struct space *s) {
/* Do we need unique IDs? */
if (!star_formation_need_unique_id) {
return;
}
const int require_new_batch = s->unique_id.next_batch.current == 0;
#ifdef WITH_MPI
const struct engine *e = s->e;
/* Check if the other ranks need a batch. */
int *all_requires = (int *)malloc(sizeof(int) * e->nr_nodes);
/* Do the communication */
MPI_Allgather(&require_new_batch, 1, MPI_INT, all_requires, 1, MPI_INT,
MPI_COMM_WORLD);
/* Compute the position of this rank batch and the position of
the next free batch. */
int local_index = 0;
int total_shift = 0;
for (int i = 0; i < e->nr_nodes; i++) {
total_shift += all_requires[i];
if (i < e->nodeID) {
local_index += all_requires[i];
}
}
/* Free the allocated resources. */
free(all_requires);
#else
int local_index = 0;
int total_shift = require_new_batch;
#endif // WITH_MPI
/* Compute the size of each batch. */
const long long batch_size = (space_extra_parts + space_extra_sparts +
space_extra_gparts + space_extra_bparts) *
s->nr_cells;
/* Get a new batch. */
if (require_new_batch) {
/* First check against an overflow. */
const long long local_shift = local_index * batch_size;
if (s->unique_id.global_next_id > LLONG_MAX - (local_shift + batch_size)) {
error("Overflow for the unique IDs.");
}
/* Now assign it. */
s->unique_id.next_batch.current = s->unique_id.global_next_id + local_shift;
s->unique_id.next_batch.max =
s->unique_id.global_next_id + local_shift + batch_size;
}
/* Shift the position of the next available batch. */
const long long shift = total_shift * batch_size;
if (s->unique_id.global_next_id > LLONG_MAX - shift) {
error("Overflow for the unique IDs.");
}
s->unique_id.global_next_id += shift;
}
/**
* @brief Get a new unique ID.
*
* @param s the #space.
*
* @return The new unique ID
*/
long long space_get_new_unique_id(struct space *s) {
/* Do we need unique IDs? */
if (!star_formation_need_unique_id) {
error("The scheme selected does not seem to use unique ID.");
}
/* Get the lock. */
lock_lock(&s->unique_id.lock);
/* Get the current available id. */
const long long id = s->unique_id.current_batch.current;
/* Update the counter. */
s->unique_id.current_batch.current++;
/* Check if everything is fine */
if (s->unique_id.current_batch.current > s->unique_id.current_batch.max) {
error("Failed to get a new ID");
}
/* Check if need to move to the next batch. */
else if (s->unique_id.current_batch.current ==
s->unique_id.current_batch.max) {
/* Check if the next batch is already used */
if (s->unique_id.next_batch.current == 0) {
error("Failed to obtain a new unique ID.");
}
s->unique_id.current_batch = s->unique_id.next_batch;
/* Reset the next batch. */
s->unique_id.next_batch.current = 0;
s->unique_id.next_batch.max = 0;
}
/* Release the lock. */
if (lock_unlock(&s->unique_id.lock) != 0) {
error("Impossible to unlock the unique id.");
}
return id;
}
/**
* @brief Initialize the computation of unique IDs.
*
* @param s The #space.
* @param nr_nodes The number of MPI ranks.
*/
void space_init_unique_id(struct space *s, int nr_nodes) {
/* Do we need unique IDs? */
if (!star_formation_need_unique_id) {
return;
}
/* Set the counter to 0. */
s->unique_id.global_next_id = 0;
/* Check the parts for the max id. */
for (size_t i = 0; i < s->nr_parts; i++) {
s->unique_id.global_next_id =
max(s->unique_id.global_next_id, s->parts[i].id);
}
/* Check the gparts for the max id. */
for (size_t i = 0; i < s->nr_gparts; i++) {
s->unique_id.global_next_id =
max(s->unique_id.global_next_id, s->gparts[i].id_or_neg_offset);
}
/* Check the sparts for the max id. */
for (size_t i = 0; i < s->nr_sparts; i++) {
s->unique_id.global_next_id =
max(s->unique_id.global_next_id, s->sparts[i].id);
}
/* Check the bparts for the max id. */
for (size_t i = 0; i < s->nr_bparts; i++) {
s->unique_id.global_next_id =
max(s->unique_id.global_next_id, s->bparts[i].id);
}
#ifdef WITH_MPI
/* Find the global max. */
MPI_Allreduce(MPI_IN_PLACE, &s->unique_id.global_next_id, 1, MPI_LONG_LONG,
MPI_MAX, MPI_COMM_WORLD);
#endif // WITH_MPI
/* Get the first unique id. */
if (s->unique_id.global_next_id == LLONG_MAX) {
error("Overflow for the unique id.");
}
s->unique_id.global_next_id++;
/* Compute the size of each batch. */
const long long batch_size = (space_extra_parts + space_extra_sparts +
space_extra_gparts + space_extra_bparts) *
s->nr_cells;
/* Compute the initial batchs (each rank has 2 batchs). */
if (s->unique_id.global_next_id > LLONG_MAX - 2 * engine_rank * batch_size) {
error("Overflow for the unique id.");
}
const long long init =
s->unique_id.global_next_id + 2 * engine_rank * batch_size;
/* Set the batchs and check for overflows. */
s->unique_id.current_batch.current = init;
if (init > LLONG_MAX - batch_size) {
error("Overflow for the unique id.");
}
s->unique_id.current_batch.max = init + batch_size;
s->unique_id.next_batch.current = s->unique_id.current_batch.max;
if (s->unique_id.next_batch.current > LLONG_MAX - batch_size) {
error("Overflow for the unique id.");
}
s->unique_id.next_batch.max = s->unique_id.next_batch.current + batch_size;
/* Update the next available id */
if (s->unique_id.global_next_id > LLONG_MAX - 2 * batch_size * nr_nodes) {
error("Overflow for the unique id.");
}
s->unique_id.global_next_id += 2 * batch_size * nr_nodes;
/* Initialize the lock. */
if (lock_init(&s->unique_id.lock) != 0)
error("Failed to init spinlock for the unique ids.");
}
/*******************************************************************************
* This file is part of SWIFT.
* Copyright (c) 2020 Loic Hausammann (loic.hausammann@epfl.ch)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
******************************************************************************/
#ifndef SWIFT_SPACE_UNIQUE_ID_H
#define SWIFT_SPACE_UNIQUE_ID_H
/* Config parameters. */
#include "../config.h"
/* Local includes */
#include "lock.h"
/* Predefine the space structure */
struct space;
/**
* @brief Batch of unique IDs for particle creation.
*/
struct batch {
/*! Current free unique id */
long long current;
/*! Maximal unique id in this batch (not included) */
long long max;
};
/*! Structure dealing with the computation of a unique ID */
struct unique_id {
/*! Current batch of unique ids */
struct batch current_batch;
/*! Next batch of unique ids */
struct batch next_batch;
/* Global next slot available */
long long global_next_id;
/* Lock for the unique ids */
swift_lock_type lock;
};
void space_update_unique_id(struct space *s);
long long space_get_new_unique_id(struct space *s);
void space_init_unique_id(struct space *s, int nr_nodes);
#endif // SWIFT_SPACE_UNIQUE_ID_H
......@@ -19,6 +19,10 @@
#ifndef SWIFT_EAGLE_STAR_FORMATION_STRUCT_H
#define SWIFT_EAGLE_STAR_FORMATION_STRUCT_H
/* Do we need unique IDs (only useful when spawning
new particles, conversion gas->stars does not need unique IDs) */
#define star_formation_need_unique_id 0
/**
* @brief Star-formation-related properties stored in the extended particle
* data.
......
......@@ -19,6 +19,10 @@
#ifndef SWIFT_GEAR_STAR_FORMATION_STRUCT_H
#define SWIFT_GEAR_STAR_FORMATION_STRUCT_H
/* Do we need unique IDs (only useful when spawning
new particles, conversion gas->stars does not need unique IDs) */
#define star_formation_need_unique_id 1
/**
* @brief Star-formation-related properties stored in the extended particle
* data.
......
......@@ -19,6 +19,10 @@
#ifndef SWIFT_QLA_STAR_FORMATION_STRUCT_H
#define SWIFT_QLA_STAR_FORMATION_STRUCT_H
/* Do we need unique IDs (only useful when spawning
new particles, conversion gas->stars does not need unique IDs) */
#define star_formation_need_unique_id 1
/**
* @brief Star-formation-related properties stored in the extended particle
* data.
......
......@@ -19,6 +19,10 @@
#ifndef SWIFT_NONE_STAR_FORMATION_STRUCT_H
#define SWIFT_NONE_STAR_FORMATION_STRUCT_H
/* Do we need unique IDs (only useful when spawning
new particles, conversion gas->stars does not need unique IDs) */
#define star_formation_need_unique_id 0
/**
* @brief Star-formation-related properties stored in the extended particle
* data.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment