Commit f7ae2c9b authored by Matthieu Schaller's avatar Matthieu Schaller
Browse files

Domain decomposition scheme now also read from parameter file.

parent 04bec391
......@@ -52,18 +52,18 @@ void print_help_message() {
printf(" %2s %8s %s\n", "-c", "", "Run with cosmological time integration");
printf(" %2s %8s %s\n", "-e", "",
"Enable floating-point exceptions (debugging mode)");
printf(" %2s %8s %s\n", "-f", "[value] ",
"Overwrites the CPU frequency (Hz) to be used for time measurements");
printf(" %2s %8s %s\n", "-f", "{int}",
"Overwrite the CPU frequency (Hz) to be used for time measurements");
printf(" %2s %8s %s\n", "-g", "",
"Run with an external gravitational potential");
printf(" %2s %8s %s\n", "-G", "", "Run with self-gravity");
printf(" %2s %8s %s\n", "-s", "", "Run with SPH");
printf(" %2s %8s %s\n", "-v", "[12] ",
printf(" %2s %8s %s\n", "-v", "[12]",
"Increase the level of verbosity 1: MPI-rank 0 writes "
"2: All MPI ranks write");
printf(" %2s %8s %s\n", "-y", "",
"2: All MPI-ranks write");
printf(" %2s %8s %s\n", "-y", "{int}",
"Time-step frequency at which task graphs are dumped");
printf(" %2s %8s %s\n", "-h", "", "Prints this help message and exits");
printf(" %2s %8s %s\n", "-h", "", "Print this help message and exit");
printf("\nSee the file example.yml for an example of parameter file.\n");
}
......@@ -75,22 +75,8 @@ int main(int argc, char *argv[]) {
struct clocks_time tic, toc;
#ifdef WITH_MPI
struct partition initial_partition;
enum repartition_type reparttype = REPART_NONE;
initial_partition.type = INITPART_GRID;
initial_partition.grid[0] = 1;
initial_partition.grid[1] = 1;
initial_partition.grid[2] = 1;
#ifdef HAVE_METIS
/* Defaults make use of METIS. */
reparttype = REPART_METIS_BOTH;
initial_partition.type = INITPART_METIS_NOWEIGHT;
#endif
#endif
int nr_nodes = 1, myrank = 0;
#ifdef WITH_MPI
/* Start by initializing MPI. */
int res = 0, prov = 0;
......@@ -111,19 +97,13 @@ int main(int argc, char *argv[]) {
if (myrank == 0)
printf("[0000][00000.0] MPI is up and running with %i node(s).\n",
nr_nodes);
if (nr_nodes == 1) {
message("WARNING: you are running with one MPI rank.");
message("WARNING: you should use the non-MPI version of this program.");
}
fflush(stdout);
/* Set a default grid so that grid[0]*grid[1]*grid[2] == nr_nodes. */
factor(nr_nodes, &initial_partition.grid[0], &initial_partition.grid[1]);
factor(nr_nodes / initial_partition.grid[1], &initial_partition.grid[0],
&initial_partition.grid[2]);
factor(initial_partition.grid[0] * initial_partition.grid[1],
&initial_partition.grid[1], &initial_partition.grid[0]);
#endif
/* Greeting message */
if (myrank == 0) greetings();
#if defined(HAVE_SETAFFINITY) && defined(HAVE_LIBNUMA) && defined(_GNU_SOURCE)
if ((ENGINE_POLICY) & engine_policy_setaffinity) {
/* Ensure the NUMA node on which we initialise (first touch) everything
......@@ -139,6 +119,9 @@ int main(int argc, char *argv[]) {
}
#endif
/* Welcome to SWIFT, you made the right choice */
if (myrank == 0) greetings();
int dump_tasks = 0;
int with_cosmology = 0;
int with_external_gravity = 0;
......@@ -209,7 +192,7 @@ int main(int argc, char *argv[]) {
exit(1);
}
/* And then, there was time ! */
/* Genesis 1.1: And then, there was time ! */
clocks_set_cpufreq(cpufreq);
/* Report CPU frequency. */
......@@ -244,7 +227,7 @@ int main(int argc, char *argv[]) {
}
#ifdef WITH_MPI
/* Broadcast the parameter file */
MPI_Bcast(&params, sizeof(struct swift_params), MPI_BYTE, MPI_COMM_WORLD);
MPI_Bcast(&params, sizeof(struct swift_params), MPI_BYTE, 0, MPI_COMM_WORLD);
#endif
/* Initialize unit system */
......@@ -258,25 +241,21 @@ int main(int argc, char *argv[]) {
message("Unit system: U_T = %e K.", us.UnitTemperature_in_cgs);
}
/* Some initial information about domain decomposition */
/* Prepare the domain decomposition scheme */
#ifdef WITH_MPI
struct partition initial_partition;
enum repartition_type reparttype;
partition_init(&initial_partition, &reparttype, &params, nr_nodes);
/* Let's report what we did */
if (myrank == 0) {
// message("Running with %i thread(s) per node.", nr_threads);
message("Using initial partition %s",
initial_partition_name[initial_partition.type]);
if (initial_partition.type == INITPART_GRID)
message("grid set to [ %i %i %i ].", initial_partition.grid[0],
initial_partition.grid[1], initial_partition.grid[2]);
message("Using %s repartitioning", repartition_name[reparttype]);
if (nr_nodes == 1) {
message("WARNING: you are running with one MPI rank.");
message("WARNING: you should use the non-MPI version of this program.");
}
fflush(stdout);
}
#else
// if (myrank == 0) message("Running with %i thread(s).", nr_threads);
#endif
/* Read particles and space information from (GADGET) ICs */
......@@ -432,11 +411,10 @@ int main(int argc, char *argv[]) {
/* Legend */
if (myrank == 0)
printf("# %6s %14s %14s %10s %10s %16s [%s]\n",
"Step", "Time", "Time-step", "Updates", "g-Updates",
"Wall-clock time", clocks_getunit());
printf("# %6s %14s %14s %10s %10s %16s [%s]\n", "Step", "Time", "Time-step",
"Updates", "g-Updates", "Wall-clock time", clocks_getunit());
/* Let loose a runner on the space. */
/* Main simulation loop */
for (int j = 0; !engine_is_done(&e); j++) {
/* Repartition the space amongst the nodes? */
......
......@@ -2325,8 +2325,9 @@ static bool hyperthreads_present(void) {
* @param verbose Is this #engine talkative ?
*/
void engine_init(struct engine *e, struct space *s, struct swift_params *params,
int nr_nodes, int nodeID, int policy, int verbose) {
void engine_init(struct engine *e, struct space *s,
const struct swift_params *params, int nr_nodes, int nodeID,
int policy, int verbose) {
/* Clean-up everything */
bzero(e, sizeof(struct engine));
......
......@@ -168,8 +168,9 @@ struct engine {
/* Function prototypes. */
void engine_barrier(struct engine *e, int tid);
void engine_init(struct engine *e, struct space *s, struct swift_params *params,
int nr_nodes, int nodeID, int policy, int verbose);
void engine_init(struct engine *e, struct space *s,
const struct swift_params *params, int nr_nodes, int nodeID,
int policy, int verbose);
void engine_launch(struct engine *e, int nr_runners, unsigned int mask,
unsigned int submask);
void engine_prepare(struct engine *e);
......
......@@ -313,6 +313,38 @@ int parser_get_param_int(const struct swift_params *params, const char *name) {
return 0;
}
/**
* @brief Retrieve char parameter from structure.
*
* @param params Structure that holds the parameters
* @param name Name of the parameter to be found
* @return Value of the parameter found
*/
char parser_get_param_char(const struct swift_params *params,
const char *name) {
char str[PARSER_MAX_LINE_SIZE];
char retParam = 0;
for (int i = 0; i < params->count; i++) {
/*strcmp returns 0 if both strings are the same.*/
if (!strcmp(name, params->data[i].name)) {
/* Check that exactly one number is parsed. */
if (sscanf(params->data[i].value, "%c%s", &retParam, str) != 1) {
error(
"Tried parsing char '%s' but found '%s' with illegal char "
"characters '%s'.",
params->data[i].name, params->data[i].value, str);
}
return retParam;
}
}
error("Cannot find '%s' in the structure.", name);
return 0;
}
/**
* @brief Retrieve float parameter from structure.
*
......
......@@ -43,6 +43,7 @@ void parser_read_file(const char *file_name, struct swift_params *params);
void parser_print_params(const struct swift_params *params);
void parser_write_params_to_file(const struct swift_params *params,
const char *file_name);
char parser_get_param_char(const struct swift_params *params, const char *name);
int parser_get_param_int(const struct swift_params *params, const char *name);
float parser_get_param_float(const struct swift_params *params,
const char *name);
......
......@@ -52,6 +52,7 @@
#include "error.h"
#include "partition.h"
#include "space.h"
#include "tools.h"
/* Maximum weight used for METIS. */
#define metis_maxweight 10000.0f
......@@ -907,6 +908,111 @@ void partition_initial_partition(struct partition *initial_partition,
}
}
/**
* @brief Initialises the partition and re-partition scheme from the parameter
*file
*
* @param partition The #partition scheme to initialise.
* @param reparttype The repartition scheme to initialise.
* @param params The parsed parameter file.
* @param nr_nodes The number of MPI nodes we are running on.
*/
void partition_init(struct partition *partition,
enum repartition_type *reparttype,
const struct swift_params *params, int nr_nodes) {
#ifdef WITH_MPI
/* Defaults make use of METIS if available */
#ifdef HAVE_METIS
*reparttype = REPART_METIS_BOTH;
partition->type = INITPART_METIS_NOWEIGHT;
#else
*reparttype = REPART_NONE;
partition->type = INITPART_GRID;
#endif
/* Set a default grid so that grid[0]*grid[1]*grid[2] == nr_nodes. */
factor(nr_nodes, &partition->grid[0], &partition->grid[1]);
factor(nr_nodes / partition->grid[1], &partition->grid[0],
&partition->grid[2]);
factor(partition->grid[0] * partition->grid[1], &partition->grid[1],
&partition->grid[0]);
/* Now let's check what the user wants as an initial domain*/
const char part_type =
parser_get_param_char(params, "DomainDecomposition:initial_type");
switch (part_type) {
case 'g':
partition->type = INITPART_GRID;
break;
case 'v':
partition->type = INITPART_VECTORIZE;
break;
#ifdef HAVE_METIS
case 'm':
partition->type = INITPART_METIS_NOWEIGHT;
break;
case 'w':
partition->type = INITPART_METIS_WEIGHT;
break;
default:
message("Invalid choice of initial partition type '%c'.", part_type);
error("Permitted values are: 'g','m','v' or 'w'.");
#else
default:
message("Invalid choice of initial partition type '%c'.", part_type);
error("Permitted values are: 'g' or 'v' when compiled without metis.");
#endif
}
/* In case of grid, read more parameters */
if (part_type == 'g') {
partition->grid[0] =
parser_get_param_int(params, "DomainDecomposition:initial_grid_x");
partition->grid[1] =
parser_get_param_int(params, "DomainDecomposition:initial_grid_y");
partition->grid[2] =
parser_get_param_int(params, "DomainDecomposition:initial_grid_z");
}
/* Now let's check what the user wants as a repartition strategy */
const char repart_type =
parser_get_param_char(params, "DomainDecomposition:repartition_type");
switch (repart_type) {
case 'n':
*reparttype = REPART_NONE;
break;
#ifdef HAVE_METIS
case 'b':
*reparttype = REPART_METIS_BOTH;
break;
case 'e':
*reparttype = REPART_METIS_EDGE;
break;
case 'v':
*reparttype = REPART_METIS_VERTEX;
break;
case 'x':
*reparttype = REPART_METIS_VERTEX_EDGE;
break;
default:
message("Invalid choice of re-partition type '%c'.", repart_type);
error("Permitted values are: 'b','e','n', 'v' or 'x'.");
#else
default:
message("Invalid choice of re-partition type '%c'.", repart_type);
error("Permitted values are: 'n' when compiled without metis.");
#endif
}
#else
error("SWIFT was not compiled with MPI support");
#endif
}
/* General support */
/* =============== */
......
......@@ -19,6 +19,7 @@
#ifndef SWIFT_PARTITION_H
#define SWIFT_PARTITION_H
#include "parser.h"
#include "space.h"
#include "task.h"
......@@ -56,4 +57,8 @@ void partition_repartition(enum repartition_type reparttype, int nodeID,
void partition_initial_partition(struct partition *initial_partition,
int nodeID, int nr_nodes, struct space *s);
void partition_init(struct partition *partition,
enum repartition_type *reparttypestruct,
const struct swift_params *params, int nr_nodes);
#endif /* SWIFT_PARTITION_H */
......@@ -1277,9 +1277,9 @@ struct cell *space_getcell(struct space *s) {
* recursively.
*/
void space_init(struct space *s, struct swift_params *params, double dim[3],
struct part *parts, struct gpart *gparts, size_t Npart,
size_t Ngpart, int periodic, int verbose) {
void space_init(struct space *s, const struct swift_params *params,
double dim[3], struct part *parts, struct gpart *gparts,
size_t Npart, size_t Ngpart, int periodic, int verbose) {
/* Clean-up everything */
bzero(s, sizeof(struct space));
......
......@@ -132,9 +132,9 @@ void space_gparts_sort(struct gpart *gparts, int *ind, size_t N, int min,
struct cell *space_getcell(struct space *s);
int space_getsid(struct space *s, struct cell **ci, struct cell **cj,
double *shift);
void space_init(struct space *s, struct swift_params *params, double dim[3],
struct part *parts, struct gpart *gparts, size_t Npart,
size_t Ngpart, int periodic, int verbose);
void space_init(struct space *s, const struct swift_params *params,
double dim[3], struct part *parts, struct gpart *gparts,
size_t Npart, size_t Ngpart, int periodic, int verbose);
void space_map_cells_pre(struct space *s, int full,
void (*fun)(struct cell *c, void *data), void *data);
void space_map_parts(struct space *s,
......
......@@ -241,7 +241,7 @@ const char *metis_version(void) {
*/
void greetings(void) {
printf(" Welcome to the cosmological code\n");
printf(" Welcome to the cosmological hydrodynamical code\n");
printf(" ______ _________________\n");
printf(" / ___/ | / / _/ ___/_ __/\n");
printf(" \\__ \\| | /| / // // /_ / / \n");
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment