Commit ed274c46 authored by Peter W. Draper's avatar Peter W. Draper
Browse files

Merge branch 'new_h_max_definition' into 'master'

New h max definition

Since `h_max` was always an annoyance, here is a better definition. 

There is a new parameter now called `max_top_level_cells` that allows you to set the (cube root of the) maximal number of top-level cells you want. So instead of referring to it as a maximal smoothing length you directly give the number of cells.
Of course if the physics requires larger cells, the parameter is ignored. This default to 12 if unspecified, giving 1728 top-level cells. 

The only reason you would want more top-level cells is for simulations on large systems to obtain a finer MPI decomposition.

I also force a rebuild after time-step 0 now to make sure we have good grid and set of tasks even if the original mesh was based on garbage smoothing lengths values entered by the user.

What do you think ?

See merge request !275
parents b27c40b1 1ad77e4b
......@@ -27,7 +27,6 @@ Statistics:
SPH:
resolution_eta: 1.2348 # Target smoothing length in units of the mean inter-particle separation (1.2348 == 48Ngbs with the cubic spline kernel).
delta_neighbours: 0.1 # The tolerance for the targetted number of neighbours.
max_smoothing_length: 0.1 # Maximal smoothing length allowed (in internal units).
CFL_condition: 0.1 # Courant-Friedrich-Levy condition for time integration.
# Parameters related to the initial conditions
......
......@@ -27,7 +27,6 @@ Statistics:
SPH:
resolution_eta: 1.2348 # Target smoothing length in units of the mean inter-particle separation (1.2348 == 48Ngbs with the cubic spline kernel).
delta_neighbours: 0.1 # The tolerance for the targetted number of neighbours.
max_smoothing_length: 0.4 # Maximal smoothing length allowed (in internal units).
CFL_condition: 0.1 # Courant-Friedrich-Levy condition for time integration.
# Parameters related to the initial conditions
......
......@@ -27,7 +27,6 @@ Statistics:
SPH:
resolution_eta: 1.2348 # Target smoothing length in units of the mean inter-particle separation (1.2348 == 48Ngbs with the cubic spline kernel).
delta_neighbours: 0.1 # The tolerance for the targetted number of neighbours.
max_smoothing_length: 0.02 # Maximal smoothing length allowed (in internal units).
CFL_condition: 0.1 # Courant-Friedrich-Levy condition for time integration.
# Parameters related to the initial conditions
......
......@@ -27,7 +27,6 @@ Statistics:
SPH:
resolution_eta: 1.2348 # Target smoothing length in units of the mean inter-particle separation (1.2348 == 48Ngbs with the cubic spline kernel).
delta_neighbours: 0.1 # The tolerance for the targetted number of neighbours.
max_smoothing_length: 0.05 # Maximal smoothing length allowed (in internal units).
CFL_condition: 0.1 # Courant-Friedrich-Levy condition for time integration.
# Parameters related to the initial conditions
......
......@@ -27,7 +27,6 @@ Statistics:
SPH:
resolution_eta: 1.2348 # Target smoothing length in units of the mean inter-particle separation (1.2348 == 48Ngbs with the cubic spline kernel).
delta_neighbours: 0.1 # The tolerance for the targetted number of neighbours.
max_smoothing_length: 0.02 # Maximal smoothing length allowed (in internal units).
CFL_condition: 0.1 # Courant-Friedrich-Levy condition for time integration.
# Parameters related to the initial conditions
......
......@@ -27,7 +27,6 @@ Statistics:
SPH:
resolution_eta: 1.2348 # Target smoothing length in units of the mean inter-particle separation (1.2348 == 48Ngbs with the cubic spline kernel).
delta_neighbours: 0.1 # The tolerance for the targetted number of neighbours.
max_smoothing_length: 0.1 # Maximal smoothing length allowed (in internal units).
CFL_condition: 0.1 # Courant-Friedrich-Levy condition for time integration.
# Parameters related to the initial conditions
......
......@@ -27,7 +27,6 @@ Statistics:
SPH:
resolution_eta: 1.2348 # Target smoothing length in units of the mean inter-particle separation (1.2348 == 48Ngbs with the cubic spline kernel).
delta_neighbours: 0.1 # The tolerance for the targetted number of neighbours.
max_smoothing_length: 0.1 # Maximal smoothing length allowed (in internal units).
CFL_condition: 0.1 # Courant-Friedrich-Levy condition for time integration.
# Parameters related to the initial conditions
......
......@@ -8,11 +8,12 @@ InternalUnitSystem:
# Parameters for the task scheduling
Scheduler:
nr_queues: 0 # (Optional) The number of task queues to use. Use 0 to let the system decide.
cell_max_size: 8000000 # (Optional) Maximal number of interactions per task if we force the split (this is the default value).
cell_sub_size: 64000000 # (Optional) Maximal number of interactions per sub-task (this is the default value).
cell_split_size: 400 # (Optional) Maximal number of particles per cell (this is the default value).
cell_max_count: 10000 # (Optional) Maximal number of particles per cell allowed before triggering a sanitizing (this is the default value).
nr_queues: 0 # (Optional) The number of task queues to use. Use 0 to let the system decide.
cell_max_size: 8000000 # (Optional) Maximal number of interactions per task if we force the split (this is the default value).
cell_sub_size: 64000000 # (Optional) Maximal number of interactions per sub-task (this is the default value).
cell_split_size: 400 # (Optional) Maximal number of particles per cell (this is the default value).
cell_max_count: 10000 # (Optional) Maximal number of particles per cell allowed before triggering a sanitizing (this is the default value).
max_top_level_cells: 12 # (Optional) Maximal number of top-level cells in any dimension. The number of top-level cells will be the cube of this (this is the default value).
# Parameters governing the time integration
TimeIntegration:
......@@ -43,9 +44,8 @@ Statistics:
SPH:
resolution_eta: 1.2348 # Target smoothing length in units of the mean inter-particle separation (1.2348 == 48Ngbs with the cubic spline kernel).
delta_neighbours: 0.1 # The tolerance for the targetted number of neighbours.
max_ghost_iterations: 30 # (Optional) Maximal number of iterations allowed to converge towards the smoothing length.
max_smoothing_length: 0.1 # Maximal smoothing length allowed (in internal units).
CFL_condition: 0.1 # Courant-Friedrich-Levy condition for time integration.
max_ghost_iterations: 30 # (Optional) Maximal number of iterations allowed to converge towards the smoothing length.
max_volume_change: 2. # (Optional) Maximal allowed change of kernel volume over one time-step
# Parameters related to the initial conditions
......
......@@ -2242,9 +2242,10 @@ void engine_rebuild(struct engine *e) {
e->forcerebuild = 0;
/* Re-build the space. */
space_rebuild(e->s, 0.0, e->verbose);
space_rebuild(e->s, e->verbose);
if (e->ti_current == 0) space_sanitize(e->s);
/* Initial cleaning up session ? */
if (e->s->sanitized == 0) space_sanitize(e->s);
/* If in parallel, exchange the cell structure. */
#ifdef WITH_MPI
......@@ -2638,6 +2639,7 @@ void engine_init_particles(struct engine *e, int flag_entropy_ICs) {
/* Ready to go */
e->step = -1;
e->forcerebuild = 1;
e->wallclock_time = (float)clocks_diff(&time1, &time2);
if (e->verbose) message("took %.3f %s.", e->wallclock_time, clocks_getunit());
......
......@@ -31,6 +31,7 @@
* @param parameter_file The parsed parameter file
* @param phys_const Physical constants in internal units
* @param us The current internal system of units
* @param s The #space we run in.
* @param potential The external potential properties to initialize
*/
void potential_init(const struct swift_params* parameter_file,
......
......@@ -143,6 +143,7 @@ external_gravity_get_potential_energy(
* @param parameter_file The parsed parameter file
* @param phys_const Physical constants in internal units
* @param us The current internal system of units
* @param s The #space we run in.
* @param potential The external potential properties to initialize
*/
static INLINE void potential_init_backend(
......
......@@ -188,10 +188,9 @@ void space_rebuild_recycle(struct space *s, struct cell *c) {
* @brief Re-build the top-level cell grid.
*
* @param s The #space.
* @param cell_max Maximum cell edge length.
* @param verbose Print messages to stdout or not.
*/
void space_regrid(struct space *s, double cell_max, int verbose) {
void space_regrid(struct space *s, int verbose) {
const size_t nr_parts = s->nr_parts;
const ticks tic = getticks();
......@@ -226,13 +225,16 @@ void space_regrid(struct space *s, double cell_max, int verbose) {
h_max = buff;
}
#endif
if (verbose) message("h_max is %.3e (cell_max=%.3e).", h_max, cell_max);
if (verbose) message("h_max is %.3e (cell_min=%.3e).", h_max, s->cell_min);
/* Get the new putative cell dimensions. */
const int cdim[3] = {
floor(s->dim[0] / fmax(h_max * kernel_gamma * space_stretch, cell_max)),
floor(s->dim[1] / fmax(h_max * kernel_gamma * space_stretch, cell_max)),
floor(s->dim[2] / fmax(h_max * kernel_gamma * space_stretch, cell_max))};
floor(s->dim[0] /
fmax(h_max * kernel_gamma * space_stretch, s->cell_min)),
floor(s->dim[1] /
fmax(h_max * kernel_gamma * space_stretch, s->cell_min)),
floor(s->dim[2] /
fmax(h_max * kernel_gamma * space_stretch, s->cell_min))};
/* Check if we have enough cells for periodicity. */
if (s->periodic && (cdim[0] < 3 || cdim[1] < 3 || cdim[2] < 3))
......@@ -431,11 +433,10 @@ void space_regrid(struct space *s, double cell_max, int verbose) {
* @brief Re-build the cells as well as the tasks.
*
* @param s The #space in which to update the cells.
* @param cell_max Maximal cell size.
* @param verbose Print messages to stdout or not
*
*/
void space_rebuild(struct space *s, double cell_max, int verbose) {
void space_rebuild(struct space *s, int verbose) {
const ticks tic = getticks();
......@@ -443,7 +444,7 @@ void space_rebuild(struct space *s, double cell_max, int verbose) {
// message("re)building space..."); fflush(stdout);
/* Re-grid if necessary, or just re-set the cell data. */
space_regrid(s, cell_max, verbose);
space_regrid(s, verbose);
size_t nr_parts = s->nr_parts;
size_t nr_gparts = s->nr_gparts;
......@@ -730,6 +731,8 @@ void space_split(struct space *s, struct cell *cells, int nr_cells,
*/
void space_sanitize(struct space *s) {
s->sanitized = 1;
for (int k = 0; k < s->nr_cells; k++) {
struct cell *c = &s->cells_top[k];
......@@ -1737,6 +1740,7 @@ void space_init(struct space *s, const struct swift_params *params,
s->dim[0] = dim[0];
s->dim[1] = dim[1];
s->dim[2] = dim[2];
s->sanitized = 0;
s->periodic = periodic;
s->gravity = gravity;
s->nr_parts = Npart;
......@@ -1745,9 +1749,22 @@ void space_init(struct space *s, const struct swift_params *params,
s->nr_gparts = Ngpart;
s->size_gparts = Ngpart;
s->gparts = gparts;
s->cell_min = parser_get_param_double(params, "SPH:max_smoothing_length");
s->nr_queues = 1; /* Temporary value until engine construction */
/* Decide on the minimal top-level cell size */
const double dmax = max(max(dim[0], dim[1]), dim[2]);
int maxtcells = parser_get_opt_param_int(params,
"Scheduler:max_top_level_cells",
space_max_top_level_cells_default);
s->cell_min = 0.99 * dmax / maxtcells;
/* Check that it is big enough. */
const double dmin = min(min(dim[0], dim[1]), dim[2]);
int needtcells = 3 * dmax / dmin;
if (maxtcells < needtcells)
error("Scheduler:max_top_level_cells is too small %d, needs to be at "
"least %d", maxtcells, needtcells);
/* Get the constants for the scheduler */
space_maxsize = parser_get_opt_param_int(params, "Scheduler:cell_max_size",
space_maxsize_default);
......@@ -1761,14 +1778,6 @@ void space_init(struct space *s, const struct swift_params *params,
message("max_size set to %d, sub_size set to %d, split_size set to %d",
space_maxsize, space_subsize, space_splitsize);
/* Check that we have enough cells */
if (s->cell_min * 3 > dim[0] || s->cell_min * 3 > dim[1] ||
s->cell_min * 3 > dim[2])
error(
"Maximal smoothing length (%e) too large. Needs to be "
"smaller than 1/3 the simulation box size [%e %e %e]",
s->cell_min, dim[0], dim[1], dim[2]);
/* Apply h scaling */
const double scaling =
parser_get_opt_param_double(params, "InitialConditions:h_scaling", 1.0);
......@@ -1847,7 +1856,7 @@ void space_init(struct space *s, const struct swift_params *params,
if (lock_init(&s->lock) != 0) error("Failed to create space spin-lock.");
/* Build the cells and the tasks. */
if (!dry_run) space_regrid(s, s->cell_min, verbose);
if (!dry_run) space_regrid(s, verbose);
}
/**
......
......@@ -42,6 +42,7 @@
#define space_maxsize_default 8000000
#define space_subsize_default 64000000
#define space_maxcount_default 10000
#define space_max_top_level_cells_default 12
#define space_stretch 1.10f
#define space_maxreldx 0.25f
......@@ -122,6 +123,9 @@ struct space {
/*! Number of queues in the system. */
int nr_queues;
/*! Has this space already been sanitized ? */
int sanitized;
/*! The associated engine. */
struct engine *e;
......@@ -165,7 +169,7 @@ void space_parts_sort_mapper(void *map_data, int num_elements,
void *extra_data);
void space_gparts_sort_mapper(void *map_data, int num_elements,
void *extra_data);
void space_rebuild(struct space *s, double h_max, int verbose);
void space_rebuild(struct space *s, int verbose);
void space_recycle(struct space *s, struct cell *c);
void space_split(struct space *s, struct cell *cells, int nr_cells,
int verbose);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment