Skip to content
Snippets Groups Projects
Commit 070794d2 authored by Peter W. Draper's avatar Peter W. Draper
Browse files

Make timebins a repartition option and restore the old behaviour

parent cf8980e5
Branches
Tags
1 merge request!458Add time based edge repartitioning
...@@ -79,8 +79,8 @@ DomainDecomposition: ...@@ -79,8 +79,8 @@ DomainDecomposition:
initial_grid_x: 10 # (Optional) Grid size if the "grid" strategy is chosen. initial_grid_x: 10 # (Optional) Grid size if the "grid" strategy is chosen.
initial_grid_y: 10 # "" initial_grid_y: 10 # ""
initial_grid_z: 10 # "" initial_grid_z: 10 # ""
repartition_type: task_weights # (Optional) The re-decomposition strategy: "none", "task_weights", "particle_weights", repartition_type: task_costs # (Optional) The re-decomposition strategy: "none", "task_costs", "particle_weights",
# "edge_task_weights" or "hybrid_weights". # "edge_task_costs", "hybrid_weights" or "bin_weights"
trigger: 0.05 # (Optional) Fractional (<1) CPU time difference between MPI ranks required to trigger a trigger: 0.05 # (Optional) Fractional (<1) CPU time difference between MPI ranks required to trigger a
# new decomposition, or number of steps (>1) between decompositions # new decomposition, or number of steps (>1) between decompositions
minfrac: 0.9 # (Optional) Fractional of all particles that should be updated in previous step when minfrac: 0.9 # (Optional) Fractional of all particles that should be updated in previous step when
......
...@@ -316,7 +316,7 @@ static void dumpCells_map(struct cell *c, void *data) { ...@@ -316,7 +316,7 @@ static void dumpCells_map(struct cell *c, void *data) {
* for all cells to a simple text file. * for all cells to a simple text file.
* *
* @param prefix base output filename, result is written to * @param prefix base output filename, result is written to
* <prefix>_<rank>_<step>.dat * %prefix%_%rank%_%step%.dat
* @param s the space holding the cells to dump. * @param s the space holding the cells to dump.
* @param rank node ID of MPI rank, or 0 if not relevant. * @param rank node ID of MPI rank, or 0 if not relevant.
* @param step the current engine step, or some unique integer. * @param step the current engine step, or some unique integer.
......
...@@ -63,10 +63,13 @@ const char *initial_partition_name[] = { ...@@ -63,10 +63,13 @@ const char *initial_partition_name[] = {
/* Simple descriptions of repartition types for reports. */ /* Simple descriptions of repartition types for reports. */
const char *repartition_name[] = { const char *repartition_name[] = {
"no", "METIS edge and vertex time weighted cells", "none",
"METIS particle count vertex weighted cells", "METIS edge and vertex cost weights",
"METIS time edge weighted cells", "METIS particle count vertex weights",
"METIS particle count vertex and time edge cells"}; "METIS cost edge weights",
"METIS particle count vertex and cost edge weights",
"METIS vertex costs and edge delta timebin weights",
};
/* Local functions, if needed. */ /* Local functions, if needed. */
static int check_complete(struct space *s, int verbose, int nregions); static int check_complete(struct space *s, int verbose, int nregions);
...@@ -476,15 +479,16 @@ static void pick_metis(struct space *s, int nregions, int *vertexw, int *edgew, ...@@ -476,15 +479,16 @@ static void pick_metis(struct space *s, int nregions, int *vertexw, int *edgew,
* @param partweights whether particle counts will be used as vertex weights. * @param partweights whether particle counts will be used as vertex weights.
* @param bothweights whether vertex and edge weights will be used, otherwise * @param bothweights whether vertex and edge weights will be used, otherwise
* only edge weights will be used. * only edge weights will be used.
* @param timebins use timebins as edge weights.
* @param nodeID our nodeID. * @param nodeID our nodeID.
* @param nr_nodes the number of nodes. * @param nr_nodes the number of nodes.
* @param s the space of cells holding our local particles. * @param s the space of cells holding our local particles.
* @param tasks the completed tasks from the last engine step for our node. * @param tasks the completed tasks from the last engine step for our node.
* @param nr_tasks the number of tasks. * @param nr_tasks the number of tasks.
*/ */
static void repart_edge_metis(int partweights, int bothweights, int nodeID, static void repart_edge_metis(int partweights, int bothweights, int timebins,
int nr_nodes, struct space *s, struct task *tasks, int nodeID, int nr_nodes, struct space *s,
int nr_tasks) { struct task *tasks, int nr_tasks) {
/* Create weight arrays using task ticks for vertices and edges (edges /* Create weight arrays using task ticks for vertices and edges (edges
* assume the same graph structure as used in the part_ calls). Note that * assume the same graph structure as used in the part_ calls). Note that
...@@ -599,42 +603,39 @@ static void repart_edge_metis(int partweights, int bothweights, int nodeID, ...@@ -599,42 +603,39 @@ static void repart_edge_metis(int partweights, int bothweights, int nodeID,
if (cj->nodeID == nodeID) weights_v[cjd] += 0.5 * w; if (cj->nodeID == nodeID) weights_v[cjd] += 0.5 * w;
} }
/* Add weights to edge for all cells based on the expected interaction if (timebins) {
* time. We want to avoid having active cells on the edges, so we cut /* Add weights to edge for all cells based on the expected
* for that. */ * interaction time (calculated as the time to the last expected
int dti = num_time_bins - get_time_bin(ci->ti_end_min); * time) as we want to avoid having active cells on the edges, so we
int dtj = num_time_bins - get_time_bin(cj->ti_end_min); * cut for that. Note that weight is added to the local and remote
* cells, as we want to keep both away from any cuts. */
int kk; int dti = num_time_bins - get_time_bin(ci->ti_end_min);
for (kk = 26 * cid; inds[kk] != cjd; kk++) int dtj = num_time_bins - get_time_bin(cj->ti_end_min);
; int dt = (1<<(dti + dtj));
/* If a foreign cell we repeat the interaction remotely, so these get /* ci */
* 1/2 of the local time bin weight. */ int kk;
int dt; for (kk = 26 * cid; inds[kk] != cjd; kk++);
if (ci->nodeID == nodeID) wtot[1] += dt;
dt = dti + dtj/2; weights_e[kk] += dt;
else
dt = dtj/2; /* cj */
dt = dti + dtj; for (kk = 26 * cjd; inds[kk] != cid; kk++);
dt = (1<<dt); wtot[1] += dt;
weights_e[kk] += dt;
wtot[1] += dt; } else {
weights_e[kk] += dt;
/* Add weights from task costs to the edge. */
/* cj */
for (kk = 26 * cjd; inds[kk] != cid; kk++) /* ci */
; int kk;
if (cj->nodeID == nodeID) for (kk = 26 * cid; inds[kk] != cjd; kk++);
dt = dtj + dti/2; weights_e[kk] += w;
else
dt = dti/2; /* cj */
for (kk = 26 * cjd; inds[kk] != cid; kk++);
dt = dti + dtj; weights_e[kk] += w;
dt = (1<<dt); }
wtot[1] += dt;
weights_e[kk] += dt;
} }
} }
} }
...@@ -705,7 +706,8 @@ static void repart_edge_metis(int partweights, int bothweights, int nodeID, ...@@ -705,7 +706,8 @@ static void repart_edge_metis(int partweights, int bothweights, int nodeID,
} }
} }
/* Now edge weights. Unlikely these will ever need scaling. */ /* Now edge weights. XXX need to balance weights if very different in
* range. */
wmin[1] = wmax[1]; wmin[1] = wmax[1];
wmax[1] = 0; wmax[1] = 0;
for (int k = 0; k < nr_cells; k++) { for (int k = 0; k < nr_cells; k++) {
...@@ -838,35 +840,26 @@ void partition_repartition(struct repartition *reparttype, int nodeID, ...@@ -838,35 +840,26 @@ void partition_repartition(struct repartition *reparttype, int nodeID,
#if defined(WITH_MPI) && defined(HAVE_METIS) #if defined(WITH_MPI) && defined(HAVE_METIS)
if (reparttype->type == REPART_METIS_BOTH || if (reparttype->type == REPART_METIS_VERTEX_COSTS_EDGE_COSTS) {
reparttype->type == REPART_METIS_EDGE || repart_edge_metis(1, 1, 0, nodeID, nr_nodes, s, tasks, nr_tasks);
reparttype->type == REPART_METIS_VERTEX_EDGE) {
int partweights; } else if (reparttype->type == REPART_METIS_EDGE_COSTS) {
int bothweights; repart_edge_metis(0, 0, 0, nodeID, nr_nodes, s, tasks, nr_tasks);
if (reparttype->type == REPART_METIS_VERTEX_EDGE)
partweights = 1;
else
partweights = 0;
if (reparttype->type == REPART_METIS_BOTH) } else if (reparttype->type == REPART_METIS_VERTEX_COUNTS_EDGE_COSTS) {
bothweights = 1; repart_edge_metis(1, 0, 0, nodeID, nr_nodes, s, tasks, nr_tasks);
else
bothweights = 0;
repart_edge_metis(partweights, bothweights, nodeID, nr_nodes, s, tasks, } else if (reparttype->type == REPART_METIS_VERTEX_COSTS_EDGE_TIMEBINS) {
nr_tasks); repart_edge_metis(1, 1, 1, nodeID, nr_nodes, s, tasks, nr_tasks);
} else if (reparttype->type == REPART_METIS_VERTEX) {
} else if (reparttype->type == REPART_METIS_VERTEX_COUNTS) {
repart_vertex_metis(s, nodeID, nr_nodes); repart_vertex_metis(s, nodeID, nr_nodes);
} else if (reparttype->type == REPART_NONE) { } else if (reparttype->type == REPART_NONE) {
/* Doing nothing. */ /* Doing nothing. */
} else { } else {
error("Unknown repartition type"); error("Impossible repartition type");
} }
#else #else
error("SWIFT was not compiled with METIS support."); error("SWIFT was not compiled with METIS support.");
...@@ -1020,7 +1013,7 @@ void partition_init(struct partition *partition, ...@@ -1020,7 +1013,7 @@ void partition_init(struct partition *partition,
/* Defaults make use of METIS if available */ /* Defaults make use of METIS if available */
#ifdef HAVE_METIS #ifdef HAVE_METIS
const char *default_repart = "task_weights"; const char *default_repart = "task_costs";
const char *default_part = "simple_metis"; const char *default_part = "simple_metis";
#else #else
const char *default_repart = "none"; const char *default_repart = "none";
...@@ -1086,22 +1079,25 @@ void partition_init(struct partition *partition, ...@@ -1086,22 +1079,25 @@ void partition_init(struct partition *partition,
break; break;
#ifdef HAVE_METIS #ifdef HAVE_METIS
case 't': case 't':
repartition->type = REPART_METIS_BOTH; repartition->type = REPART_METIS_VERTEX_COSTS_EDGE_COSTS;
break; break;
case 'e': case 'e':
repartition->type = REPART_METIS_EDGE; repartition->type = REPART_METIS_EDGE_COSTS;
break; break;
case 'p': case 'p':
repartition->type = REPART_METIS_VERTEX; repartition->type = REPART_METIS_VERTEX_COUNTS;
break; break;
case 'h': case 'h':
repartition->type = REPART_METIS_VERTEX_EDGE; repartition->type = REPART_METIS_VERTEX_COUNTS_EDGE_COSTS;
break;
case 'b':
repartition->type = REPART_METIS_VERTEX_COSTS_EDGE_TIMEBINS;
break; break;
default: default:
message("Invalid choice of re-partition type '%s'.", part_type); message("Invalid choice of re-partition type '%s'.", part_type);
error( error(
"Permitted values are: 'none', 'task_weights', 'particle_weights'," "Permitted values are: 'none', 'task_costs', 'particle_weights',"
"'edge_task_weights' or 'hybrid_weights'"); "'edge_task_costs', 'hybrid_weights' or 'bin_weights'");
#else #else
default: default:
message("Invalid choice of re-partition type '%s'.", part_type); message("Invalid choice of re-partition type '%s'.", part_type);
......
...@@ -43,10 +43,11 @@ struct partition { ...@@ -43,10 +43,11 @@ struct partition {
/* Repartition type to use. */ /* Repartition type to use. */
enum repartition_type { enum repartition_type {
REPART_NONE = 0, REPART_NONE = 0,
REPART_METIS_BOTH, REPART_METIS_VERTEX_COSTS_EDGE_COSTS,
REPART_METIS_VERTEX, REPART_METIS_VERTEX_COUNTS,
REPART_METIS_EDGE, REPART_METIS_EDGE_COSTS,
REPART_METIS_VERTEX_EDGE REPART_METIS_VERTEX_COUNTS_EDGE_COSTS,
REPART_METIS_VERTEX_COSTS_EDGE_TIMEBINS,
}; };
/* Repartition preferences. */ /* Repartition preferences. */
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment