From f7ae2c9ba75d73d912a1e7703188fb4817c5556c Mon Sep 17 00:00:00 2001
From: Matthieu Schaller <matthieu.schaller@durham.ac.uk>
Date: Sun, 3 Apr 2016 19:39:29 +0100
Subject: [PATCH] Domain decomposition scheme now also read from parameter
file.
---
examples/main.c | 72 ++++++++++++--------------------
src/engine.c | 5 ++-
src/engine.h | 5 ++-
src/parser.c | 32 +++++++++++++++
src/parser.h | 1 +
src/partition.c | 106 ++++++++++++++++++++++++++++++++++++++++++++++++
src/partition.h | 5 +++
src/space.c | 6 +--
src/space.h | 6 +--
src/version.c | 2 +-
10 files changed, 182 insertions(+), 58 deletions(-)
diff --git a/examples/main.c b/examples/main.c
index 023e8be500..08e036621e 100644
--- a/examples/main.c
+++ b/examples/main.c
@@ -52,18 +52,18 @@ void print_help_message() {
printf(" %2s %8s %s\n", "-c", "", "Run with cosmological time integration");
printf(" %2s %8s %s\n", "-e", "",
"Enable floating-point exceptions (debugging mode)");
- printf(" %2s %8s %s\n", "-f", "[value] ",
- "Overwrites the CPU frequency (Hz) to be used for time measurements");
+ printf(" %2s %8s %s\n", "-f", "{int}",
+ "Overwrite the CPU frequency (Hz) to be used for time measurements");
printf(" %2s %8s %s\n", "-g", "",
"Run with an external gravitational potential");
printf(" %2s %8s %s\n", "-G", "", "Run with self-gravity");
printf(" %2s %8s %s\n", "-s", "", "Run with SPH");
- printf(" %2s %8s %s\n", "-v", "[12] ",
+ printf(" %2s %8s %s\n", "-v", "[12]",
"Increase the level of verbosity 1: MPI-rank 0 writes "
- "2: All MPI ranks write");
- printf(" %2s %8s %s\n", "-y", "",
+ "2: All MPI-ranks write");
+ printf(" %2s %8s %s\n", "-y", "{int}",
"Time-step frequency at which task graphs are dumped");
- printf(" %2s %8s %s\n", "-h", "", "Prints this help message and exits");
+ printf(" %2s %8s %s\n", "-h", "", "Print this help message and exit");
printf("\nSee the file example.yml for an example of parameter file.\n");
}
@@ -75,22 +75,8 @@ int main(int argc, char *argv[]) {
struct clocks_time tic, toc;
-#ifdef WITH_MPI
- struct partition initial_partition;
- enum repartition_type reparttype = REPART_NONE;
-
- initial_partition.type = INITPART_GRID;
- initial_partition.grid[0] = 1;
- initial_partition.grid[1] = 1;
- initial_partition.grid[2] = 1;
-#ifdef HAVE_METIS
- /* Defaults make use of METIS. */
- reparttype = REPART_METIS_BOTH;
- initial_partition.type = INITPART_METIS_NOWEIGHT;
-#endif
-#endif
-
int nr_nodes = 1, myrank = 0;
+
#ifdef WITH_MPI
/* Start by initializing MPI. */
int res = 0, prov = 0;
@@ -111,19 +97,13 @@ int main(int argc, char *argv[]) {
if (myrank == 0)
printf("[0000][00000.0] MPI is up and running with %i node(s).\n",
nr_nodes);
+ if (nr_nodes == 1) {
+ message("WARNING: you are running with one MPI rank.");
+ message("WARNING: you should use the non-MPI version of this program.");
+ }
fflush(stdout);
-
- /* Set a default grid so that grid[0]*grid[1]*grid[2] == nr_nodes. */
- factor(nr_nodes, &initial_partition.grid[0], &initial_partition.grid[1]);
- factor(nr_nodes / initial_partition.grid[1], &initial_partition.grid[0],
- &initial_partition.grid[2]);
- factor(initial_partition.grid[0] * initial_partition.grid[1],
- &initial_partition.grid[1], &initial_partition.grid[0]);
#endif
- /* Greeting message */
- if (myrank == 0) greetings();
-
#if defined(HAVE_SETAFFINITY) && defined(HAVE_LIBNUMA) && defined(_GNU_SOURCE)
if ((ENGINE_POLICY) & engine_policy_setaffinity) {
/* Ensure the NUMA node on which we initialise (first touch) everything
@@ -139,6 +119,9 @@ int main(int argc, char *argv[]) {
}
#endif
+ /* Welcome to SWIFT, you made the right choice */
+ if (myrank == 0) greetings();
+
int dump_tasks = 0;
int with_cosmology = 0;
int with_external_gravity = 0;
@@ -209,7 +192,7 @@ int main(int argc, char *argv[]) {
exit(1);
}
- /* And then, there was time ! */
+ /* Genesis 1.1: And then, there was time ! */
clocks_set_cpufreq(cpufreq);
/* Report CPU frequency. */
@@ -244,7 +227,7 @@ int main(int argc, char *argv[]) {
}
#ifdef WITH_MPI
/* Broadcast the parameter file */
- MPI_Bcast(¶ms, sizeof(struct swift_params), MPI_BYTE, MPI_COMM_WORLD);
+ MPI_Bcast(¶ms, sizeof(struct swift_params), MPI_BYTE, 0, MPI_COMM_WORLD);
#endif
/* Initialize unit system */
@@ -258,25 +241,21 @@ int main(int argc, char *argv[]) {
message("Unit system: U_T = %e K.", us.UnitTemperature_in_cgs);
}
-/* Some initial information about domain decomposition */
+/* Prepare the domain decomposition scheme */
#ifdef WITH_MPI
+ struct partition initial_partition;
+ enum repartition_type reparttype;
+ partition_init(&initial_partition, &reparttype, ¶ms, nr_nodes);
+
+ /* Let's report what we did */
if (myrank == 0) {
- // message("Running with %i thread(s) per node.", nr_threads);
message("Using initial partition %s",
initial_partition_name[initial_partition.type]);
if (initial_partition.type == INITPART_GRID)
message("grid set to [ %i %i %i ].", initial_partition.grid[0],
initial_partition.grid[1], initial_partition.grid[2]);
message("Using %s repartitioning", repartition_name[reparttype]);
-
- if (nr_nodes == 1) {
- message("WARNING: you are running with one MPI rank.");
- message("WARNING: you should use the non-MPI version of this program.");
- }
- fflush(stdout);
}
-#else
-// if (myrank == 0) message("Running with %i thread(s).", nr_threads);
#endif
/* Read particles and space information from (GADGET) ICs */
@@ -432,11 +411,10 @@ int main(int argc, char *argv[]) {
/* Legend */
if (myrank == 0)
- printf("# %6s %14s %14s %10s %10s %16s [%s]\n",
- "Step", "Time", "Time-step", "Updates", "g-Updates",
- "Wall-clock time", clocks_getunit());
+ printf("# %6s %14s %14s %10s %10s %16s [%s]\n", "Step", "Time", "Time-step",
+ "Updates", "g-Updates", "Wall-clock time", clocks_getunit());
- /* Let loose a runner on the space. */
+ /* Main simulation loop */
for (int j = 0; !engine_is_done(&e); j++) {
/* Repartition the space amongst the nodes? */
diff --git a/src/engine.c b/src/engine.c
index 82ff038bb2..b2997ea2c7 100644
--- a/src/engine.c
+++ b/src/engine.c
@@ -2325,8 +2325,9 @@ static bool hyperthreads_present(void) {
* @param verbose Is this #engine talkative ?
*/
-void engine_init(struct engine *e, struct space *s, struct swift_params *params,
- int nr_nodes, int nodeID, int policy, int verbose) {
+void engine_init(struct engine *e, struct space *s,
+ const struct swift_params *params, int nr_nodes, int nodeID,
+ int policy, int verbose) {
/* Clean-up everything */
bzero(e, sizeof(struct engine));
diff --git a/src/engine.h b/src/engine.h
index 4145089ec6..e1c3f61d12 100644
--- a/src/engine.h
+++ b/src/engine.h
@@ -168,8 +168,9 @@ struct engine {
/* Function prototypes. */
void engine_barrier(struct engine *e, int tid);
-void engine_init(struct engine *e, struct space *s, struct swift_params *params,
- int nr_nodes, int nodeID, int policy, int verbose);
+void engine_init(struct engine *e, struct space *s,
+ const struct swift_params *params, int nr_nodes, int nodeID,
+ int policy, int verbose);
void engine_launch(struct engine *e, int nr_runners, unsigned int mask,
unsigned int submask);
void engine_prepare(struct engine *e);
diff --git a/src/parser.c b/src/parser.c
index 4b78270ebe..5856bd5015 100644
--- a/src/parser.c
+++ b/src/parser.c
@@ -313,6 +313,38 @@ int parser_get_param_int(const struct swift_params *params, const char *name) {
return 0;
}
+/**
+ * @brief Retrieve char parameter from structure.
+ *
+ * @param params Structure that holds the parameters
+ * @param name Name of the parameter to be found
+ * @return Value of the parameter found
+ */
+char parser_get_param_char(const struct swift_params *params,
+ const char *name) {
+
+ char str[PARSER_MAX_LINE_SIZE];
+ char retParam = 0;
+
+ for (int i = 0; i < params->count; i++) {
+ /*strcmp returns 0 if both strings are the same.*/
+ if (!strcmp(name, params->data[i].name)) {
+ /* Check that exactly one number is parsed. */
+ if (sscanf(params->data[i].value, "%c%s", &retParam, str) != 1) {
+ error(
+ "Tried parsing char '%s' but found '%s' with illegal char "
+ "characters '%s'.",
+ params->data[i].name, params->data[i].value, str);
+ }
+
+ return retParam;
+ }
+ }
+
+ error("Cannot find '%s' in the structure.", name);
+ return 0;
+}
+
/**
* @brief Retrieve float parameter from structure.
*
diff --git a/src/parser.h b/src/parser.h
index cf178e7ce6..15e58f9778 100644
--- a/src/parser.h
+++ b/src/parser.h
@@ -43,6 +43,7 @@ void parser_read_file(const char *file_name, struct swift_params *params);
void parser_print_params(const struct swift_params *params);
void parser_write_params_to_file(const struct swift_params *params,
const char *file_name);
+char parser_get_param_char(const struct swift_params *params, const char *name);
int parser_get_param_int(const struct swift_params *params, const char *name);
float parser_get_param_float(const struct swift_params *params,
const char *name);
diff --git a/src/partition.c b/src/partition.c
index ea25bc132d..c54bef7080 100644
--- a/src/partition.c
+++ b/src/partition.c
@@ -52,6 +52,7 @@
#include "error.h"
#include "partition.h"
#include "space.h"
+#include "tools.h"
/* Maximum weight used for METIS. */
#define metis_maxweight 10000.0f
@@ -907,6 +908,111 @@ void partition_initial_partition(struct partition *initial_partition,
}
}
+/**
+ * @brief Initialises the partition and re-partition scheme from the parameter
+ *file
+ *
+ * @param partition The #partition scheme to initialise.
+ * @param reparttype The repartition scheme to initialise.
+ * @param params The parsed parameter file.
+ * @param nr_nodes The number of MPI nodes we are running on.
+ */
+void partition_init(struct partition *partition,
+ enum repartition_type *reparttype,
+ const struct swift_params *params, int nr_nodes) {
+
+#ifdef WITH_MPI
+
+/* Defaults make use of METIS if available */
+#ifdef HAVE_METIS
+ *reparttype = REPART_METIS_BOTH;
+ partition->type = INITPART_METIS_NOWEIGHT;
+#else
+ *reparttype = REPART_NONE;
+ partition->type = INITPART_GRID;
+#endif
+
+ /* Set a default grid so that grid[0]*grid[1]*grid[2] == nr_nodes. */
+ factor(nr_nodes, &partition->grid[0], &partition->grid[1]);
+ factor(nr_nodes / partition->grid[1], &partition->grid[0],
+ &partition->grid[2]);
+ factor(partition->grid[0] * partition->grid[1], &partition->grid[1],
+ &partition->grid[0]);
+
+ /* Now let's check what the user wants as an initial domain*/
+ const char part_type =
+ parser_get_param_char(params, "DomainDecomposition:initial_type");
+
+ switch (part_type) {
+ case 'g':
+ partition->type = INITPART_GRID;
+ break;
+ case 'v':
+ partition->type = INITPART_VECTORIZE;
+ break;
+#ifdef HAVE_METIS
+ case 'm':
+ partition->type = INITPART_METIS_NOWEIGHT;
+ break;
+ case 'w':
+ partition->type = INITPART_METIS_WEIGHT;
+ break;
+ default:
+ message("Invalid choice of initial partition type '%c'.", part_type);
+ error("Permitted values are: 'g','m','v' or 'w'.");
+#else
+ default:
+ message("Invalid choice of initial partition type '%c'.", part_type);
+ error("Permitted values are: 'g' or 'v' when compiled without metis.");
+#endif
+ }
+
+ /* In case of grid, read more parameters */
+ if (part_type == 'g') {
+ partition->grid[0] =
+ parser_get_param_int(params, "DomainDecomposition:initial_grid_x");
+ partition->grid[1] =
+ parser_get_param_int(params, "DomainDecomposition:initial_grid_y");
+ partition->grid[2] =
+ parser_get_param_int(params, "DomainDecomposition:initial_grid_z");
+ }
+
+ /* Now let's check what the user wants as a repartition strategy */
+ const char repart_type =
+ parser_get_param_char(params, "DomainDecomposition:repartition_type");
+
+ switch (repart_type) {
+ case 'n':
+ *reparttype = REPART_NONE;
+ break;
+#ifdef HAVE_METIS
+ case 'b':
+ *reparttype = REPART_METIS_BOTH;
+ break;
+ case 'e':
+ *reparttype = REPART_METIS_EDGE;
+ break;
+ case 'v':
+ *reparttype = REPART_METIS_VERTEX;
+ break;
+ case 'x':
+ *reparttype = REPART_METIS_VERTEX_EDGE;
+ break;
+ default:
+ message("Invalid choice of re-partition type '%c'.", repart_type);
+ error("Permitted values are: 'b','e','n', 'v' or 'x'.");
+#else
+ default:
+ message("Invalid choice of re-partition type '%c'.", repart_type);
+ error("Permitted values are: 'n' when compiled without metis.");
+#endif
+ }
+
+#else
+ error("SWIFT was not compiled with MPI support");
+#endif
+}
+
/* General support */
/* =============== */
diff --git a/src/partition.h b/src/partition.h
index 3ab5f5a817..af3d0be924 100644
--- a/src/partition.h
+++ b/src/partition.h
@@ -19,6 +19,7 @@
#ifndef SWIFT_PARTITION_H
#define SWIFT_PARTITION_H
+#include "parser.h"
#include "space.h"
#include "task.h"
@@ -56,4 +57,8 @@ void partition_repartition(enum repartition_type reparttype, int nodeID,
void partition_initial_partition(struct partition *initial_partition,
int nodeID, int nr_nodes, struct space *s);
+void partition_init(struct partition *partition,
+ enum repartition_type *reparttypestruct,
+ const struct swift_params *params, int nr_nodes);
+
#endif /* SWIFT_PARTITION_H */
diff --git a/src/space.c b/src/space.c
index 1d3878b109..53b4054be5 100644
--- a/src/space.c
+++ b/src/space.c
@@ -1277,9 +1277,9 @@ struct cell *space_getcell(struct space *s) {
* recursively.
*/
-void space_init(struct space *s, struct swift_params *params, double dim[3],
- struct part *parts, struct gpart *gparts, size_t Npart,
- size_t Ngpart, int periodic, int verbose) {
+void space_init(struct space *s, const struct swift_params *params,
+ double dim[3], struct part *parts, struct gpart *gparts,
+ size_t Npart, size_t Ngpart, int periodic, int verbose) {
/* Clean-up everything */
bzero(s, sizeof(struct space));
diff --git a/src/space.h b/src/space.h
index 56a0e2bb9d..3e0b9ff84e 100644
--- a/src/space.h
+++ b/src/space.h
@@ -132,9 +132,9 @@ void space_gparts_sort(struct gpart *gparts, int *ind, size_t N, int min,
struct cell *space_getcell(struct space *s);
int space_getsid(struct space *s, struct cell **ci, struct cell **cj,
double *shift);
-void space_init(struct space *s, struct swift_params *params, double dim[3],
- struct part *parts, struct gpart *gparts, size_t Npart,
- size_t Ngpart, int periodic, int verbose);
+void space_init(struct space *s, const struct swift_params *params,
+ double dim[3], struct part *parts, struct gpart *gparts,
+ size_t Npart, size_t Ngpart, int periodic, int verbose);
void space_map_cells_pre(struct space *s, int full,
void (*fun)(struct cell *c, void *data), void *data);
void space_map_parts(struct space *s,
diff --git a/src/version.c b/src/version.c
index 6aeee2d8bc..27841a1601 100644
--- a/src/version.c
+++ b/src/version.c
@@ -241,7 +241,7 @@ const char *metis_version(void) {
*/
void greetings(void) {
- printf(" Welcome to the cosmological code\n");
+ printf(" Welcome to the cosmological hydrodynamical code\n");
printf(" ______ _________________\n");
printf(" / ___/ | / / _/ ___/_ __/\n");
printf(" \\__ \\| | /| / // // /_ / / \n");
--
GitLab