Commit 2b8930c4 authored by Peter W. Draper's avatar Peter W. Draper
Browse files

Merge branch 'more_policies' into 'master'

Engine policies for gravity and improved task mask construction

This is a straightforward change to engine_step() and engine_init_particles() that should allow other users to add or remove tasks without having to comment-out portions of the code. 

The right list of policies simply has to be passed to the engine when constructed. I have added three policies (hydro, external gravity and self gravity) to be future proof.

I have also resurrected the two functions that print the content of a mask. These had disappeared following a merge. Finally, some more robust bit-mask checking tests are done when testing policies. 

That should allow @jregan and @tt to add their external gravity tasks to the mask on line 1901 (in engine_step()) of engine.c without having to comment-out other parts of the code.

See merge request !91
parents 3fd353ba fe44d13b
......@@ -56,9 +56,6 @@
#define ENGINE_POLICY engine_policy_none
#endif
/**
* @brief Main routine that loads a few particles and generates some output.
*
......@@ -121,7 +118,8 @@ int main(int argc, char *argv[]) {
#if defined(HAVE_SETAFFINITY) && defined(HAVE_LIBNUMA) && defined(_GNU_SOURCE)
if ((ENGINE_POLICY) & engine_policy_setaffinity) {
/* Ensure the NUMA node on which we initialise (first touch) everything
* doesn't change before engine_init allocates NUMA-local workers. Otherwise,
* doesn't change before engine_init allocates NUMA-local workers.
* Otherwise,
* we may be scheduled elsewhere between the two times.
*/
cpu_set_t affinity;
......@@ -342,7 +340,8 @@ int main(int argc, char *argv[]) {
tic = getticks();
if (myrank == 0) message("nr_nodes is %i.", nr_nodes);
engine_init(&e, &s, dt_max, nr_threads, nr_queues, nr_nodes, myrank,
ENGINE_POLICY | engine_policy_steal, 0, time_end, dt_min, dt_max);
ENGINE_POLICY | engine_policy_steal | engine_policy_hydro, 0,
time_end, dt_min, dt_max);
if (myrank == 0)
message("engine_init took %.3f ms.",
((double)(getticks() - tic)) / CPU_TPS * 1000);
......
......@@ -55,9 +55,10 @@
#include "part.h"
#include "timers.h"
const char *engine_policy_names[10] = {"none", "rand", "steal",
"keep", "block", "fix_dt",
"cpu_tight", "mpi", "numa_affinity"};
const char *engine_policy_names[12] = {
"none", "rand", "steal", "keep",
"block", "fix_dt", "cpu_tight", "mpi",
"numa_affinity", "hydro", "self_gravity", "external_gravity"};
/** The rank of the engine as a global variable (for messages). */
int engine_rank;
......@@ -1326,7 +1327,7 @@ int engine_marktasks(struct engine *e) {
// ticks tic = getticks();
/* Much less to do here if we're on a fixed time-step. */
if (e->policy & engine_policy_fixdt) {
if ((e->policy & engine_policy_fixdt) == engine_policy_fixdt) {
/* Run through the tasks and mark as skip or not. */
for (k = 0; k < nr_tasks; k++) {
......@@ -1721,15 +1722,48 @@ void engine_init_particles(struct engine *e) {
// message("\n0th DENSITY CALC\n");
/* Now do a density calculation */
TIMER_TIC;
engine_launch(e, e->nr_threads,
(1 << task_type_sort) | (1 << task_type_self) |
(1 << task_type_pair) | (1 << task_type_sub) |
(1 << task_type_init) | (1 << task_type_ghost) |
(1 << task_type_send) | (1 << task_type_recv),
1 << task_subtype_density);
/* Build the masks corresponding to the policy */
unsigned int mask = 0;
unsigned int submask = 0;
/* We always have sort tasks */
mask |= 1 << task_type_sort;
/* Add the tasks corresponding to hydro operations to the masks */
if ((e->policy & engine_policy_hydro) == engine_policy_hydro) {
mask |= 1 << task_type_init;
mask |= 1 << task_type_self;
mask |= 1 << task_type_pair;
mask |= 1 << task_type_sub;
mask |= 1 << task_type_ghost;
submask |= 1 << task_subtype_density;
}
/* Add the tasks corresponding to self-gravity to the masks */
if ((e->policy & engine_policy_self_gravity) == engine_policy_self_gravity) {
/* Nothing here for now */
}
/* Add the tasks corresponding to self-gravity to the masks */
if ((e->policy & engine_policy_external_gravity) ==
engine_policy_external_gravity) {
/* Nothing here for now */
}
/* Add MPI tasks if need be */
if ((e->policy & engine_policy_mpi) == engine_policy_mpi) {
mask |= 1 << task_type_send;
mask |= 1 << task_type_recv;
}
/* Now, launch the calculation */
TIMER_TIC;
engine_launch(e, e->nr_threads, mask, submask);
TIMER_TOC(timer_runners);
// message("\n0th ENTROPY CONVERSION\n");
......@@ -1833,16 +1867,50 @@ if ( e->nodeID == 0 )
/* Prepare the space. */
engine_prepare(e);
/* Build the masks corresponding to the policy */
unsigned int mask = 0;
unsigned int submask = 0;
/* We always have sort tasks and kick tasks */
mask |= 1 << task_type_sort;
mask |= 1 << task_type_kick;
/* Add the tasks corresponding to hydro operations to the masks */
if ((e->policy & engine_policy_hydro) == engine_policy_hydro) {
mask |= 1 << task_type_init;
mask |= 1 << task_type_self;
mask |= 1 << task_type_pair;
mask |= 1 << task_type_sub;
mask |= 1 << task_type_ghost;
submask |= 1 << task_subtype_density;
submask |= 1 << task_subtype_force;
}
/* Add the tasks corresponding to self-gravity to the masks */
if ((e->policy & engine_policy_self_gravity) == engine_policy_self_gravity) {
/* Nothing here for now */
}
/* Add the tasks corresponding to self-gravity to the masks */
if ((e->policy & engine_policy_external_gravity) ==
engine_policy_external_gravity) {
/* Nothing here for now */
}
/* Add MPI tasks if need be */
if ((e->policy & engine_policy_mpi) == engine_policy_mpi) {
mask |= 1 << task_type_send;
mask |= 1 << task_type_recv;
}
/* Send off the runners. */
TIMER_TIC;
engine_launch(e, e->nr_threads,
(1 << task_type_sort) | (1 << task_type_self) |
(1 << task_type_pair) | (1 << task_type_sub) |
(1 << task_type_init) | (1 << task_type_ghost) |
(1 << task_type_kick) | (1 << task_type_send) |
(1 << task_type_recv),
(1 << task_subtype_density) | (1 << task_subtype_force));
engine_launch(e, e->nr_threads, mask, submask);
TIMER_TOC(timer_runners);
TIMER_TOC2(timer_step);
......@@ -2060,7 +2128,7 @@ void engine_init(struct engine *e, struct space *s, float dt, int nr_threads,
int nr_cores = sysconf(_SC_NPROCESSORS_ONLN);
int j, cpuid[nr_cores];
cpu_set_t cpuset;
if (policy & engine_policy_cputight) {
if ((policy & engine_policy_cputight) == engine_policy_cputight) {
for (k = 0; k < nr_cores; k++) cpuid[k] = k;
} else {
/* Get next highest power of 2. */
......@@ -2166,7 +2234,7 @@ void engine_init(struct engine *e, struct space *s, float dt, int nr_threads,
engine_print_policy(e);
/* Deal with timestep */
if (e->policy & engine_policy_fixdt) {
if ((e->policy & engine_policy_fixdt) == engine_policy_fixdt) {
e->dt_min = e->dt_max;
}
......@@ -2211,7 +2279,7 @@ void engine_init(struct engine *e, struct space *s, float dt, int nr_threads,
if (pthread_create(&e->runners[k].thread, NULL, &runner_main,
&e->runners[k]) != 0)
error("Failed to create runner thread.");
if (e->policy & engine_policy_setaffinity) {
if ((e->policy & engine_policy_setaffinity) == engine_policy_setaffinity) {
#if defined(HAVE_SETAFFINITY)
/* Set a reasonable queue ID. */
......@@ -2258,14 +2326,14 @@ void engine_print_policy(struct engine *e) {
if (e->nodeID == 0) {
printf("[000] engine_policy: engine policies are [ ");
for (int k = 1; k < 32; k++)
if (e->policy & 1 << k) printf(" %s,", engine_policy_names[k + 1]);
if (e->policy & (1 << k)) printf(" %s,", engine_policy_names[k + 1]);
printf(" ]\n");
fflush(stdout);
}
#else
printf("engine_policy: engine policies are [ ");
for (int k = 1; k < 32; k++)
if (e->policy & 1 << k) printf(" %s,", engine_policy_names[k + 1]);
if (e->policy & (1 << k)) printf(" %s,", engine_policy_names[k + 1]);
printf(" ]\n");
fflush(stdout);
#endif
......
......@@ -41,14 +41,17 @@
/* Some constants. */
enum engine_policy {
engine_policy_none = 0,
engine_policy_rand = 1,
engine_policy_steal = 2,
engine_policy_keep = 4,
engine_policy_block = 8,
engine_policy_fixdt = 16,
engine_policy_cputight = 32,
engine_policy_mpi = 64,
engine_policy_setaffinity = 128
engine_policy_rand = (1 << 0),
engine_policy_steal = (1 << 1),
engine_policy_keep = (1 << 2),
engine_policy_block = (1 << 3),
engine_policy_fixdt = (1 << 4),
engine_policy_cputight = (1 << 5),
engine_policy_mpi = (1 << 6),
engine_policy_setaffinity = (1 << 7),
engine_policy_hydro = (1 << 8),
engine_policy_self_gravity = (1 << 9),
engine_policy_external_gravity = (1 << 10)
};
extern const char *engine_policy_names[];
......
......@@ -304,3 +304,29 @@ void task_addunlock_old(struct task *ta, struct task *tb) {
lock_unlock_blind(&ta->lock);
}
/**
* @brief Prints the list of tasks contained in a given mask
*
* @param mask The mask to analyse
*/
void task_print_mask(unsigned int mask) {
printf("task_print_mask: The tasks to run are [");
for (int k = 1; k < task_type_count; k++)
printf(" %s=%s", taskID_names[k], (mask & (1 << k)) ? "yes" : "no");
printf(" ]\n");
}
/**
* @brief Prints the list of subtasks contained in a given submask
*
* @param submask The submask to analyse
*/
void task_print_submask(unsigned int submask) {
printf("task_print_submask: The subtasks to run are [");
for (int k = 1; k < task_subtype_count; k++)
printf(" %s=%s", subtaskID_names[k], (submask & (1 << k)) ? "yes" : "no");
printf(" ]\n");
}
......@@ -96,6 +96,7 @@ void task_unlock(struct task *t);
float task_overlap(const struct task *ta, const struct task *tb);
int task_lock(struct task *t);
void task_print_mask(unsigned int mask);
void task_print_submask(unsigned int submask);
void task_do_rewait(struct task *t);
#endif /* SWIFT_TASK_H */
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment