Merged latest changes from master in the branch.

examples/main.c

@@ -56,9 +56,6 @@
 #define ENGINE_POLICY engine_policy_none
 #endif
 
-
-
-
 /**
  * @brief Main routine that loads a few particles and generates some output.
  *
@@ -121,7 +118,8 @@ int main(int argc, char *argv[]) {
 #if defined(HAVE_SETAFFINITY) && defined(HAVE_LIBNUMA) && defined(_GNU_SOURCE)
   if ((ENGINE_POLICY) & engine_policy_setaffinity) {
     /* Ensure the NUMA node on which we initialise (first touch) everything
-     * doesn't change before engine_init allocates NUMA-local workers. Otherwise,
+     * doesn't change before engine_init allocates NUMA-local workers.
+     * Otherwise,
      * we may be scheduled elsewhere between the two times.
      */
     cpu_set_t affinity;
@@ -342,7 +340,8 @@ int main(int argc, char *argv[]) {
   tic = getticks();
   if (myrank == 0) message("nr_nodes is %i.", nr_nodes);
   engine_init(&e, &s, dt_max, nr_threads, nr_queues, nr_nodes, myrank,
-              ENGINE_POLICY | engine_policy_steal, 0, time_end, dt_min, dt_max);
+              ENGINE_POLICY | engine_policy_steal | engine_policy_hydro, 0,
+              time_end, dt_min, dt_max);
   if (myrank == 0)
     message("engine_init took %.3f ms.",
             ((double)(getticks() - tic)) / CPU_TPS * 1000);

src/engine.c

@@ -55,9 +55,10 @@
 #include "part.h"
 #include "timers.h"
 
-const char *engine_policy_names[10] = {"none",      "rand",  "steal",
-                                       "keep",      "block", "fix_dt",
-                                       "cpu_tight", "mpi",   "numa_affinity"};
+const char *engine_policy_names[12] = {
+    "none",          "rand",   "steal",        "keep",
+    "block",         "fix_dt", "cpu_tight",    "mpi",
+    "numa_affinity", "hydro",  "self_gravity", "external_gravity"};
 
 /** The rank of the engine as a global variable (for messages). */
 int engine_rank;
@@ -1326,7 +1327,7 @@ int engine_marktasks(struct engine *e) {
   // ticks tic = getticks();
 
   /* Much less to do here if we're on a fixed time-step. */
-  if (e->policy & engine_policy_fixdt) {
+  if ((e->policy & engine_policy_fixdt) == engine_policy_fixdt) {
 
     /* Run through the tasks and mark as skip or not. */
     for (k = 0; k < nr_tasks; k++) {
@@ -1615,13 +1616,13 @@ void engine_barrier(struct engine *e, int tid) {
 
 void engine_collect_kick(struct cell *c) {
 
-  int updated = 0;
+  int k, updated = 0;
   float t_end_min = FLT_MAX, t_end_max = 0.0f;
   double e_kin = 0.0, e_int = 0.0, e_pot = 0.0;
   float mom[3] = {0.0f, 0.0f, 0.0f}, ang[3] = {0.0f, 0.0f, 0.0f};
   struct cell *cp;
 
-  /* Skip super-cells (There values are already set) */
+  /* Skip super-cells (Their values are already set) */
   if (c->kick != NULL) return;
 
   /* Only do something is the cell is non-empty */
@@ -1726,20 +1727,63 @@ void engine_init_particles(struct engine *e) {
 
   engine_marktasks(e);
 
-  /* Now do a density calculation */
-  TIMER_TIC;
-  engine_launch(e, e->nr_threads,
-                (1 << task_type_sort) | (1 << task_type_self) |
-                    (1 << task_type_pair) | (1 << task_type_sub) |
-                    (1 << task_type_init) | (1 << task_type_ghost) |
-                    (1 << task_type_send) | (1 << task_type_recv),
-                1 << task_subtype_density);
+  // printParticle(e->s->parts, 1000, e->s->nr_parts);
+  // printParticle(e->s->parts, 515050, e->s->nr_parts);
+
+  // message("\n0th DENSITY CALC\n");
+
+  /* Build the masks corresponding to the policy */
+  unsigned int mask = 0;
+  unsigned int submask = 0;
+
+  /* We always have sort tasks */
+  mask |= 1 << task_type_sort;
+
+  /* Add the tasks corresponding to hydro operations to the masks */
+  if ((e->policy & engine_policy_hydro) == engine_policy_hydro) {
+
+    mask |= 1 << task_type_init;
+    mask |= 1 << task_type_self;
+    mask |= 1 << task_type_pair;
+    mask |= 1 << task_type_sub;
+    mask |= 1 << task_type_ghost;
+
+    submask |= 1 << task_subtype_density;
+  }
+
+  /* Add the tasks corresponding to self-gravity to the masks */
+  if ((e->policy & engine_policy_self_gravity) == engine_policy_self_gravity) {
+
+    /* Nothing here for now */
+  }
 
+  /* Add the tasks corresponding to self-gravity to the masks */
+  if ((e->policy & engine_policy_external_gravity) ==
+      engine_policy_external_gravity) {
+
+    /* Nothing here for now */
+  }
+
+  /* Add MPI tasks if need be */
+  if ((e->policy & engine_policy_mpi) == engine_policy_mpi) {
+
+    mask |= 1 << task_type_send;
+    mask |= 1 << task_type_recv;
+  }
+
+  /* Now, launch the calculation */
+  TIMER_TIC;
+  engine_launch(e, e->nr_threads, mask, submask);
   TIMER_TOC(timer_runners);
 
+// message("\n0th ENTROPY CONVERSION\n")
+
   /* Apply some conversions (e.g. internal energy -> entropy) */
   space_map_cells_pre(s, 1, cell_convert_hydro, NULL);
 
+  // printParticle(e->s->parts, e->s->xparts,1000, e->s->nr_parts);
+  // printParticle(e->s->parts, e->s->xparts,515050, e->s->nr_parts);
+
   /* Ready to go */
   e->step = -1;
 }
@@ -1811,6 +1855,8 @@ void engine_step(struct engine *e) {
   e_pot = in[3];
 #endif
 
+  // message("\nDRIFT\n");
+
   /* Move forward in time */
   e->timeOld = e->time;
   e->time = t_end_min;
@@ -1820,22 +1866,63 @@ void engine_step(struct engine *e) {
   /* Drift everybody */
   engine_launch(e, e->nr_threads, 1 << task_type_drift, 0);
 
+  // printParticle(e->s->parts, e->s->xparts, 1000, e->s->nr_parts);
+  // printParticle(e->s->parts, e->s->xparts, 515050, e->s->nr_parts);
+
+  // if(e->step == 2)   exit(0);
+
+  // message("\nACCELERATION AND KICK\n");
+
   /* Re-distribute the particles amongst the nodes? */
   if (e->forcerepart) engine_repartition(e);
 
   /* Prepare the space. */
   engine_prepare(e);
 
+  /* Build the masks corresponding to the policy */
+  unsigned int mask = 0;
+  unsigned int submask = 0;
+
+  /* We always have sort tasks and kick tasks */
+  mask |= 1 << task_type_sort;
+  mask |= 1 << task_type_kick;
+
+  /* Add the tasks corresponding to hydro operations to the masks */
+  if ((e->policy & engine_policy_hydro) == engine_policy_hydro) {
+
+    mask |= 1 << task_type_init;
+    mask |= 1 << task_type_self;
+    mask |= 1 << task_type_pair;
+    mask |= 1 << task_type_sub;
+    mask |= 1 << task_type_ghost;
+
+    submask |= 1 << task_subtype_density;
+    submask |= 1 << task_subtype_force;
+  }
+
+  /* Add the tasks corresponding to self-gravity to the masks */
+  if ((e->policy & engine_policy_self_gravity) == engine_policy_self_gravity) {
+
+    /* Nothing here for now */
+  }
+
+  /* Add the tasks corresponding to self-gravity to the masks */
+  if ((e->policy & engine_policy_external_gravity) ==
+      engine_policy_external_gravity) {
+
+    /* Nothing here for now */
+  }
+
+  /* Add MPI tasks if need be */
+  if ((e->policy & engine_policy_mpi) == engine_policy_mpi) {
+
+    mask |= 1 << task_type_send;
+    mask |= 1 << task_type_recv;
+  }
+
   /* Send off the runners. */
   TIMER_TIC;
-  engine_launch(e, e->nr_threads,
-                (1 << task_type_sort) | (1 << task_type_self) |
-                    (1 << task_type_pair) | (1 << task_type_sub) |
-                    (1 << task_type_init) | (1 << task_type_ghost) |
-                    (1 << task_type_kick) | (1 << task_type_send) |
-                    (1 << task_type_recv),
-                (1 << task_subtype_density) | (1 << task_subtype_force));
-
+  engine_launch(e, e->nr_threads, mask, submask);
   TIMER_TOC(timer_runners);
 
   TIMER_TOC2(timer_step);
@@ -1853,6 +1940,9 @@ void engine_step(struct engine *e) {
             mom[2], ang[0], ang[1], ang[2]);
     fflush(e->file_stats);
   }
+
+  // printParticle(e->s->parts, e->s->xparts,1000, e->s->nr_parts);
+  // printParticle(e->s->parts, e->s->xparts,515050, e->s->nr_parts);
 }
 
 /**
@@ -2058,7 +2148,7 @@ void engine_init(struct engine *e, struct space *s, float dt, int nr_threads,
   int nr_cores = sysconf(_SC_NPROCESSORS_ONLN);
   int j, cpuid[nr_cores];
   cpu_set_t cpuset;
-  if (policy & engine_policy_cputight) {
+  if ((policy & engine_policy_cputight) == engine_policy_cputight) {
     for (k = 0; k < nr_cores; k++) cpuid[k] = k;
   } else {
     /*  Get next highest power of 2. */
@@ -2173,7 +2263,7 @@ void engine_init(struct engine *e, struct space *s, float dt, int nr_threads,
   engine_print_policy(e);
 
   /* Deal with timestep */
-  if (e->policy & engine_policy_fixdt) {
+  if ((e->policy & engine_policy_fixdt) == engine_policy_fixdt) {
     e->dt_min = e->dt_max;
   }
 
@@ -2218,7 +2308,7 @@ void engine_init(struct engine *e, struct space *s, float dt, int nr_threads,
     if (pthread_create(&e->runners[k].thread, NULL, &runner_main,
                        &e->runners[k]) != 0)
       error("Failed to create runner thread.");
-    if (e->policy & engine_policy_setaffinity) {
+    if ((e->policy & engine_policy_setaffinity) == engine_policy_setaffinity) {
 #if defined(HAVE_SETAFFINITY)
 
       /* Set a reasonable queue ID. */
@@ -2265,14 +2355,14 @@ void engine_print_policy(struct engine *e) {
   if (e->nodeID == 0) {
     printf("[000] engine_policy: engine policies are [ ");
     for (int k = 1; k < 32; k++)
-      if (e->policy & 1 << k) printf(" %s,", engine_policy_names[k + 1]);
+      if (e->policy & (1 << k)) printf(" %s,", engine_policy_names[k + 1]);
     printf(" ]\n");
     fflush(stdout);
   }
 #else
   printf("engine_policy: engine policies are [ ");
   for (int k = 1; k < 32; k++)
-    if (e->policy & 1 << k) printf(" %s,", engine_policy_names[k + 1]);
+    if (e->policy & (1 << k)) printf(" %s,", engine_policy_names[k + 1]);
   printf(" ]\n");
   fflush(stdout);
 #endif

src/engine.h

@@ -42,14 +42,17 @@
 /* Some constants. */
 enum engine_policy {
   engine_policy_none = 0,
-  engine_policy_rand = 1,
-  engine_policy_steal = 2,
-  engine_policy_keep = 4,
-  engine_policy_block = 8,
-  engine_policy_fixdt = 16,
-  engine_policy_cputight = 32,
-  engine_policy_mpi = 64,
-  engine_policy_setaffinity = 128
+  engine_policy_rand = (1 << 0),
+  engine_policy_steal = (1 << 1),
+  engine_policy_keep = (1 << 2),
+  engine_policy_block = (1 << 3),
+  engine_policy_fixdt = (1 << 4),
+  engine_policy_cputight = (1 << 5),
+  engine_policy_mpi = (1 << 6),
+  engine_policy_setaffinity = (1 << 7),
+  engine_policy_hydro = (1 << 8),
+  engine_policy_self_gravity = (1 << 9),
+  engine_policy_external_gravity = (1 << 10)
 };
 
 extern const char *engine_policy_names[];

src/task.c

@@ -304,3 +304,29 @@ void task_addunlock_old(struct task *ta, struct task *tb) {
 
   lock_unlock_blind(&ta->lock);
 }
+
+/**
+ * @brief Prints the list of tasks contained in a given mask
+ *
+ * @param mask The mask to analyse
+ */
+void task_print_mask(unsigned int mask) {
+
+  printf("task_print_mask: The tasks to run are [");
+  for (int k = 1; k < task_type_count; k++)
+    printf(" %s=%s", taskID_names[k], (mask & (1 << k)) ? "yes" : "no");
+  printf(" ]\n");
+}
+
+/**
+ * @brief Prints the list of subtasks contained in a given submask
+ *
+ * @param submask The submask to analyse
+ */
+void task_print_submask(unsigned int submask) {
+
+  printf("task_print_submask: The subtasks to run are [");
+  for (int k = 1; k < task_subtype_count; k++)
+    printf(" %s=%s", subtaskID_names[k], (submask & (1 << k)) ? "yes" : "no");
+  printf(" ]\n");
+}

src/task.h

@@ -96,6 +96,7 @@ void task_unlock(struct task *t);
 float task_overlap(const struct task *ta, const struct task *tb);
 int task_lock(struct task *t);
 void task_print_mask(unsigned int mask);
+void task_print_submask(unsigned int submask);
 void task_do_rewait(struct task *t);
 
 #endif /* SWIFT_TASK_H */