Commit b07c911a authored by Loic Hausammann's avatar Loic Hausammann
Browse files

Logger: first review of MPI

parent 511bef09
...@@ -169,6 +169,7 @@ int main(int argc, char *argv[]) { ...@@ -169,6 +169,7 @@ int main(int argc, char *argv[]) {
int with_drift_all = 0; int with_drift_all = 0;
int with_mpole_reconstruction = 0; int with_mpole_reconstruction = 0;
int with_structure_finding = 0; int with_structure_finding = 0;
int with_logger = 0;
int with_eagle = 0; int with_eagle = 0;
int verbose = 0; int verbose = 0;
int nr_threads = 1; int nr_threads = 1;
...@@ -252,6 +253,8 @@ int main(int argc, char *argv[]) { ...@@ -252,6 +253,8 @@ int main(int argc, char *argv[]) {
"Overwrite the CPU " "Overwrite the CPU "
"frequency (Hz) to be used for time measurements.", "frequency (Hz) to be used for time measurements.",
NULL, 0, 0), NULL, 0, 0),
OPT_BOOLEAN(0, "logger", &with_logger,
"Run with the logger.", NULL, 0, 0),
OPT_INTEGER('n', "steps", &nsteps, OPT_INTEGER('n', "steps", &nsteps,
"Execute a fixed number of time steps. When unset use the " "Execute a fixed number of time steps. When unset use the "
"time_end parameter to stop.", "time_end parameter to stop.",
...@@ -330,6 +333,13 @@ int main(int argc, char *argv[]) { ...@@ -330,6 +333,13 @@ int main(int argc, char *argv[]) {
} }
#endif #endif
#if !defined(WITH_LOGGER)
if (with_logger) {
printf("Error: the logger is not available, please compile with --enable-logger.");
return 1;
}
#endif
#ifndef HAVE_FE_ENABLE_EXCEPT #ifndef HAVE_FE_ENABLE_EXCEPT
if (with_fp_exceptions) { if (with_fp_exceptions) {
printf("Error: no support for floating point exceptions\n"); printf("Error: no support for floating point exceptions\n");
...@@ -1144,6 +1154,7 @@ int main(int argc, char *argv[]) { ...@@ -1144,6 +1154,7 @@ int main(int argc, char *argv[]) {
if (with_structure_finding) if (with_structure_finding)
engine_policies |= engine_policy_structure_finding; engine_policies |= engine_policy_structure_finding;
if (with_fof) engine_policies |= engine_policy_fof; if (with_fof) engine_policies |= engine_policy_fof;
if (with_logger) engine_policies |= engine_policy_logger;
/* Initialize the engine with the space and policies. */ /* Initialize the engine with the space and policies. */
if (myrank == 0) clocks_gettime(&tic); if (myrank == 0) clocks_gettime(&tic);
...@@ -1206,7 +1217,15 @@ int main(int argc, char *argv[]) { ...@@ -1206,7 +1217,15 @@ int main(int argc, char *argv[]) {
#ifdef WITH_MPI #ifdef WITH_MPI
/* Split the space. */ /* Split the space. */
engine_split(&e, &initial_partition); engine_split(&e, &initial_partition);
engine_redistribute(&e, /* initial_redistribute */ 1); if (with_logger) {
/* Turn off the logger to avoid writing the communications */
e.policy &= ~engine_policy_logger;
}
engine_redistribute(&e);
if (with_logger) {
/* Turn it back on */
e.policy |= engine_policy_logger;
}
#endif #endif
/* Initialise the particles */ /* Initialise the particles */
...@@ -1214,8 +1233,10 @@ int main(int argc, char *argv[]) { ...@@ -1214,8 +1233,10 @@ int main(int argc, char *argv[]) {
/* Write the state of the system before starting time integration. */ /* Write the state of the system before starting time integration. */
#ifdef WITH_LOGGER #ifdef WITH_LOGGER
logger_log_all(e.logger, &e); if (e.policy & engine_policy_logger) {
engine_dump_index(&e); logger_log_all(e.logger, &e);
engine_dump_index(&e);
}
#endif #endif
/* Dump initial state snapshot, if not working with an output list */ /* Dump initial state snapshot, if not working with an output list */
if (!e.output_list_snapshots) engine_dump_snapshot(&e); if (!e.output_list_snapshots) engine_dump_snapshot(&e);
...@@ -1417,14 +1438,16 @@ int main(int argc, char *argv[]) { ...@@ -1417,14 +1438,16 @@ int main(int argc, char *argv[]) {
engine_print_stats(&e); engine_print_stats(&e);
} }
#ifdef WITH_LOGGER #ifdef WITH_LOGGER
logger_log_all(e.logger, &e); if (e.policy & engine_policy_logger) {
logger_log_all(e.logger, &e);
/* Write a final index file */ /* Write a final index file */
engine_dump_index(&e); engine_dump_index(&e);
/* Write a sentinel timestamp */ /* Write a sentinel timestamp */
logger_log_timestamp(e.logger, e.ti_current, e.time, logger_log_timestamp(e.logger, e.ti_current, e.time,
&e.logger->timestamp_offset); &e.logger->timestamp_offset);
}
#endif #endif
/* Write final snapshot? */ /* Write final snapshot? */
......
...@@ -122,7 +122,9 @@ const char *engine_policy_names[] = {"none", ...@@ -122,7 +122,9 @@ const char *engine_policy_names[] = {"none",
"black holes", "black holes",
"fof search", "fof search",
"time-step limiter", "time-step limiter",
"time-step sync"}; "time-step sync",
"logger"
};
/** The rank of the engine as a global variable (for messages). */ /** The rank of the engine as a global variable (for messages). */
int engine_rank; int engine_rank;
...@@ -223,7 +225,7 @@ void engine_repartition(struct engine *e) { ...@@ -223,7 +225,7 @@ void engine_repartition(struct engine *e) {
Finally, the space, tasks, and proxies need to be rebuilt. */ Finally, the space, tasks, and proxies need to be rebuilt. */
/* Redistribute the particles between the nodes. */ /* Redistribute the particles between the nodes. */
engine_redistribute(e, /* initial_redistribute */ 0); engine_redistribute(e);
/* Make the proxies. */ /* Make the proxies. */
engine_makeproxies(e); engine_makeproxies(e);
...@@ -454,21 +456,17 @@ void engine_exchange_strays(struct engine *e, const size_t offset_parts, ...@@ -454,21 +456,17 @@ void engine_exchange_strays(struct engine *e, const size_t offset_parts,
&s->xparts[offset_parts + k], 1); &s->xparts[offset_parts + k], 1);
#ifdef WITH_LOGGER #ifdef WITH_LOGGER
const int logger_flag = logger_generate_flag( if (e->policy & engine_policy_logger) {
logger_flag_mpi | logger_flag_delete, node_id); const int logger_flag = logger_generate_flag(
logger_flag_mpi_exit, node_id);
/* Log the particle when leaving a rank. */
logger_log_part(e->logger, &s->parts[offset_parts + k], /* Log the particle when leaving a rank. */
logger_mask_data[logger_x].mask | logger_log_part(e->logger, &s->parts[offset_parts + k],
logger_mask_data[logger_v].mask | logger_masks_all_part |
logger_mask_data[logger_a].mask | logger_mask_data[logger_special_flags].mask,
logger_mask_data[logger_u].mask | &s->xparts[offset_parts + k].logger_data.last_offset,
logger_mask_data[logger_h].mask | logger_flag);
logger_mask_data[logger_rho].mask | }
logger_mask_data[logger_consts].mask |
logger_mask_data[logger_special_flags].mask,
&s->xparts[offset_parts + k].logger_data.last_offset,
logger_flag);
#endif #endif
} }
...@@ -507,17 +505,17 @@ void engine_exchange_strays(struct engine *e, const size_t offset_parts, ...@@ -507,17 +505,17 @@ void engine_exchange_strays(struct engine *e, const size_t offset_parts,
proxy_sparts_load(&e->proxies[pid], &s->sparts[offset_sparts + k], 1); proxy_sparts_load(&e->proxies[pid], &s->sparts[offset_sparts + k], 1);
#ifdef WITH_LOGGER #ifdef WITH_LOGGER
const int logger_flag = logger_generate_flag( if (e->policy & engine_policy_logger) {
logger_flag_mpi | logger_flag_delete, node_id); const int logger_flag = logger_generate_flag(
logger_flag_mpi_exit, node_id);
/* Log the particle when leaving a rank. */ /* Log the particle when leaving a rank. */
logger_log_spart(e->logger, &s->sparts[offset_sparts + k], logger_log_spart(e->logger, &s->sparts[offset_sparts + k],
logger_mask_data[logger_x].mask | logger_masks_all_spart |
logger_mask_data[logger_v].mask | logger_mask_data[logger_special_flags].mask,
logger_mask_data[logger_consts].mask | &s->sparts[offset_parts + k].logger_data.last_offset,
logger_mask_data[logger_special_flags].mask, logger_flag);
&s->sparts[offset_parts + k].logger_data.last_offset, }
logger_flag);
#endif #endif
} }
...@@ -556,7 +554,9 @@ void engine_exchange_strays(struct engine *e, const size_t offset_parts, ...@@ -556,7 +554,9 @@ void engine_exchange_strays(struct engine *e, const size_t offset_parts,
proxy_bparts_load(&e->proxies[pid], &s->bparts[offset_bparts + k], 1); proxy_bparts_load(&e->proxies[pid], &s->bparts[offset_bparts + k], 1);
#ifdef WITH_LOGGER #ifdef WITH_LOGGER
error("TODO"); if (e->policy & engine_policy_logger) {
error("TODO");
}
#endif #endif
} }
...@@ -590,17 +590,15 @@ void engine_exchange_strays(struct engine *e, const size_t offset_parts, ...@@ -590,17 +590,15 @@ void engine_exchange_strays(struct engine *e, const size_t offset_parts,
#ifdef WITH_LOGGER #ifdef WITH_LOGGER
/* Write only the dark matter particles */ /* Write only the dark matter particles */
if (s->gparts[offset_gparts + k].type == swift_type_dark_matter) { if ((e->policy & engine_policy_logger) &&
s->gparts[offset_gparts + k].type == swift_type_dark_matter) {
const int logger_flag = logger_generate_flag( const int logger_flag = logger_generate_flag(
logger_flag_mpi | logger_flag_delete, node_id); logger_flag_mpi_exit, node_id);
/* Log the particle when leaving a rank. */ /* Log the particle when leaving a rank. */
logger_log_gpart(e->logger, &s->gparts[offset_gparts + k], logger_log_gpart(e->logger, &s->gparts[offset_gparts + k],
logger_mask_data[logger_x].mask | logger_masks_all_gpart |
logger_mask_data[logger_v].mask |
logger_mask_data[logger_a].mask |
logger_mask_data[logger_consts].mask |
logger_mask_data[logger_special_flags].mask, logger_mask_data[logger_special_flags].mask,
&s->sparts[offset_parts + k].logger_data.last_offset, &s->sparts[offset_parts + k].logger_data.last_offset,
logger_flag); logger_flag);
...@@ -827,12 +825,57 @@ void engine_exchange_strays(struct engine *e, const size_t offset_parts, ...@@ -827,12 +825,57 @@ void engine_exchange_strays(struct engine *e, const size_t offset_parts,
sizeof(struct bpart) * prox->nr_bparts_in); sizeof(struct bpart) * prox->nr_bparts_in);
#ifdef WITH_LOGGER #ifdef WITH_LOGGER
logger_log_recv_strays(e->logger, &s->parts[offset_parts + count_parts], if (e->policy & engine_policy_logger) {
&s->xparts[offset_parts + count_parts], prox->nr_parts_in, const int flag = logger_generate_flag(logger_flag_mpi_enter,
&s->gparts[offset_gparts + count_gparts], prox->nr_gparts_in, prox->nodeID);
&s->sparts[offset_sparts + count_sparts], prox->nr_sparts_in,
&s->bparts[offset_bparts + count_bparts], prox->nr_bparts_in, struct part *parts = &s->parts[offset_parts + count_parts];
prox->nodeID); struct xpart *xparts = &s->xparts[offset_parts + count_parts];
struct spart *sparts = &s->sparts[offset_sparts + count_sparts];
struct gpart *gparts = &s->gparts[offset_gparts + count_gparts];
/* Log the gas particles */
const unsigned int mask_hydro =
logger_masks_all_part |
logger_mask_data[logger_special_flags].mask;
for(int i = 0; i < prox->nr_parts_in; i++) {
logger_log_part(e->logger, &parts[i], mask_hydro,
&xparts[i].logger_data.last_offset,
flag);
/* Reset the counter */
xparts[i].logger_data.steps_since_last_output = 0;
}
/* Log the stellar particles */
const unsigned int mask_stars = logger_masks_all_spart |
logger_mask_data[logger_special_flags].mask;
for(int i = 0; i < prox->nr_sparts_in; i++) {
logger_log_spart(e->logger, &sparts[i], mask_stars,
&sparts[i].logger_data.last_offset,
flag);
sparts[i].logger_data.steps_since_last_output = 0;
}
/* Log the gparts */
const unsigned int mask_grav =
logger_masks_all_gpart |
logger_mask_data[logger_special_flags].mask;
for(int i = 0; i < prox->nr_gparts_in; i++) {
/* Log only the dark matter */
if (gparts[i].type != swift_type_dark_matter) continue;
logger_log_gpart(e->logger, &gparts[i], mask_grav,
&gparts[i].logger_data.last_offset,
flag);
gparts[i].logger_data.steps_since_last_output = 0;
}
/* Log the bparts */
if (prox->nr_bparts_in > 0) {
error("TODO");
}
}
#endif #endif
/* for (int k = offset; k < offset + count; k++) /* for (int k = offset; k < offset + count; k++)
message( message(
...@@ -1458,7 +1501,9 @@ int engine_estimate_nr_tasks(const struct engine *e) { ...@@ -1458,7 +1501,9 @@ int engine_estimate_nr_tasks(const struct engine *e) {
} }
#if defined(WITH_LOGGER) #if defined(WITH_LOGGER)
/* each cell logs its particles */ /* each cell logs its particles */
n1 += 1; if (e->policy & engine_policy_logger) {
n1 += 1;
}
#endif #endif
#ifdef WITH_MPI #ifdef WITH_MPI
...@@ -2027,13 +2072,15 @@ void engine_init_particles(struct engine *e, int flag_entropy_ICs, ...@@ -2027,13 +2072,15 @@ void engine_init_particles(struct engine *e, int flag_entropy_ICs,
cooling_update(e->cosmology, e->cooling_func, e->s); cooling_update(e->cosmology, e->cooling_func, e->s);
#ifdef WITH_LOGGER #ifdef WITH_LOGGER
/* Mark the first time step in the particle logger file. */ if (e->policy & engine_policy_logger) {
logger_log_timestamp(e->logger, e->ti_current, e->time, /* Mark the first time step in the particle logger file. */
&e->logger->timestamp_offset); logger_log_timestamp(e->logger, e->ti_current, e->time,
/* Make sure that we have enough space in the particle logger file &e->logger->timestamp_offset);
* to store the particles in current time step. */ /* Make sure that we have enough space in the particle logger file
logger_ensure_size(e->logger, s->nr_parts, s->nr_gparts, s->nr_sparts); * to store the particles in current time step. */
logger_write_description(e->logger, e); logger_ensure_size(e->logger, s->nr_parts, s->nr_gparts, s->nr_sparts);
logger_write_description(e->logger, e);
}
#endif #endif
/* Now, launch the calculation */ /* Now, launch the calculation */
...@@ -2350,13 +2397,15 @@ void engine_step(struct engine *e) { ...@@ -2350,13 +2397,15 @@ void engine_step(struct engine *e) {
} }
#ifdef WITH_LOGGER #ifdef WITH_LOGGER
/* Mark the current time step in the particle logger file. */ if (e->policy & engine_policy_logger) {
logger_log_timestamp(e->logger, e->ti_current, e->time, /* Mark the current time step in the particle logger file. */
&e->logger->timestamp_offset); logger_log_timestamp(e->logger, e->ti_current, e->time,
/* Make sure that we have enough space in the particle logger file &e->logger->timestamp_offset);
* to store the particles in current time step. */ /* Make sure that we have enough space in the particle logger file
logger_ensure_size(e->logger, e->s->nr_parts, e->s->nr_gparts, * to store the particles in current time step. */
e->s->nr_sparts); logger_ensure_size(e->logger, e->s->nr_parts, e->s->nr_gparts,
e->s->nr_sparts);
}
#endif #endif
/* Are we drifting everything (a la Gadget/GIZMO) ? */ /* Are we drifting everything (a la Gadget/GIZMO) ? */
...@@ -2459,7 +2508,9 @@ void engine_step(struct engine *e) { ...@@ -2459,7 +2508,9 @@ void engine_step(struct engine *e) {
engine_check_for_dumps(e); engine_check_for_dumps(e);
#ifdef WITH_LOGGER #ifdef WITH_LOGGER
engine_check_for_index_dump(e); if (e->policy & engine_policy_logger) {
engine_check_for_index_dump(e);
}
#endif #endif
TIMER_TOC2(timer_step); TIMER_TOC2(timer_step);
...@@ -3645,8 +3696,10 @@ void engine_init(struct engine *e, struct space *s, struct swift_params *params, ...@@ -3645,8 +3696,10 @@ void engine_init(struct engine *e, struct space *s, struct swift_params *params,
e->total_nr_tasks = 0; e->total_nr_tasks = 0;
#if defined(WITH_LOGGER) #if defined(WITH_LOGGER)
e->logger = (struct logger_writer *)malloc(sizeof(struct logger_writer)); if (e->policy & engine_policy_logger) {
logger_init(e->logger, params); e->logger = (struct logger_writer *)malloc(sizeof(struct logger_writer));
logger_init(e->logger, params);
}
#endif #endif
/* Make the space link back to the engine. */ /* Make the space link back to the engine. */
...@@ -4168,7 +4221,7 @@ void engine_config(int restart, int fof, struct engine *e, ...@@ -4168,7 +4221,7 @@ void engine_config(int restart, int fof, struct engine *e,
#endif #endif
#if defined(WITH_LOGGER) #if defined(WITH_LOGGER)
if (e->nodeID == 0) if ((e->policy & engine_policy_logger) && e->nodeID == 0)
message( message(
"WARNING: There is currently no way of predicting the output " "WARNING: There is currently no way of predicting the output "
"size, please use it carefully"); "size, please use it carefully");
...@@ -4391,7 +4444,7 @@ void engine_config(int restart, int fof, struct engine *e, ...@@ -4391,7 +4444,7 @@ void engine_config(int restart, int fof, struct engine *e,
} }
#ifdef WITH_LOGGER #ifdef WITH_LOGGER
if (!restart) { if ((e->policy & engine_policy_logger) && !restart) {
/* Write the particle logger header */ /* Write the particle logger header */
logger_write_file_header(e->logger); logger_write_file_header(e->logger);
} }
...@@ -4917,8 +4970,10 @@ void engine_clean(struct engine *e, const int fof) { ...@@ -4917,8 +4970,10 @@ void engine_clean(struct engine *e, const int fof) {
swift_free("links", e->links); swift_free("links", e->links);
#if defined(WITH_LOGGER) #if defined(WITH_LOGGER)
logger_free(e->logger); if (e->policy & engine_policy_logger) {
free(e->logger); logger_free(e->logger);
free(e->logger);
}
#endif #endif
scheduler_clean(&e->sched); scheduler_clean(&e->sched);
space_clean(e->s); space_clean(e->s);
...@@ -4999,7 +5054,9 @@ void engine_struct_dump(struct engine *e, FILE *stream) { ...@@ -4999,7 +5054,9 @@ void engine_struct_dump(struct engine *e, FILE *stream) {
if (e->output_list_stf) output_list_struct_dump(e->output_list_stf, stream); if (e->output_list_stf) output_list_struct_dump(e->output_list_stf, stream);
#ifdef WITH_LOGGER #ifdef WITH_LOGGER
logger_struct_dump(e->logger, stream); if (e->policy & engine_policy_logger) {
logger_struct_dump(e->logger, stream);
}
#endif #endif
} }
...@@ -5146,10 +5203,12 @@ void engine_struct_restore(struct engine *e, FILE *stream) { ...@@ -5146,10 +5203,12 @@ void engine_struct_restore(struct engine *e, FILE *stream) {
} }
#ifdef WITH_LOGGER #ifdef WITH_LOGGER
struct logger_writer *log = if (e->policy & engine_policy_logger) {
struct logger_writer *log =
(struct logger_writer *)malloc(sizeof(struct logger_writer)); (struct logger_writer *)malloc(sizeof(struct logger_writer));
logger_struct_restore(log, stream); logger_struct_restore(log, stream);
e->logger = log; e->logger = log;
}
#endif #endif
#ifdef EOS_PLANETARY #ifdef EOS_PLANETARY
......
...@@ -77,9 +77,10 @@ enum engine_policy { ...@@ -77,9 +77,10 @@ enum engine_policy {
engine_policy_black_holes = (1 << 19), engine_policy_black_holes = (1 << 19),
engine_policy_fof = (1 << 20), engine_policy_fof = (1 << 20),
engine_policy_timestep_limiter = (1 << 21), engine_policy_timestep_limiter = (1 << 21),
engine_policy_timestep_sync = (1 << 22) engine_policy_timestep_sync = (1 << 22),
engine_policy_logger = (1 << 23),
}; };
#define engine_maxpolicy 23 #define engine_maxpolicy 24
extern const char *engine_policy_names[engine_maxpolicy + 1]; extern const char *engine_policy_names[engine_maxpolicy + 1];
/** /**
...@@ -544,7 +545,7 @@ void engine_rebuild(struct engine *e, int redistributed, int clean_h_values); ...@@ -544,7 +545,7 @@ void engine_rebuild(struct engine *e, int redistributed, int clean_h_values);
void engine_repartition(struct engine *e); void engine_repartition(struct engine *e);
void engine_repartition_trigger(struct engine *e); void engine_repartition_trigger(struct engine *e);
void engine_makeproxies(struct engine *e); void engine_makeproxies(struct engine *e);
void engine_redistribute(struct engine *e, int initial_redistribute); void engine_redistribute(struct engine *e);
void engine_print_policy(struct engine *e); void engine_print_policy(struct engine *e);
int engine_is_done(struct engine *e); int engine_is_done(struct engine *e);
void engine_pin(void); void engine_pin(void);
......
...@@ -831,6 +831,7 @@ void engine_make_hierarchical_tasks_common(struct engine *e, struct cell *c) { ...@@ -831,6 +831,7 @@ void engine_make_hierarchical_tasks_common(struct engine *e, struct cell *c) {
const int with_timestep_limiter = const int with_timestep_limiter =
(e->policy & engine_policy_timestep_limiter); (e->policy & engine_policy_timestep_limiter);
const int with_timestep_sync = (e->policy & engine_policy_timestep_sync); const int with_timestep_sync = (e->policy & engine_policy_timestep_sync);
const int with_logger = e->policy & engine_policy_logger;
/* Are we at the top-level? */ /* Are we at the top-level? */
if (c->top == c && c->nodeID == e->nodeID) { if (c->top == c && c->nodeID == e->nodeID) {
...@@ -855,15 +856,21 @@ void engine_make_hierarchical_tasks_common(struct engine *e, struct cell *c) { ...@@ -855,15 +856,21 @@ void engine_make_hierarchical_tasks_common(struct engine *e, struct cell *c) {
c, NULL); c, NULL);
#if defined(WITH_LOGGER) #if defined(WITH_LOGGER)
/* Add the hydro logger task. */ struct task *kick2_or_logger;
c->logger = scheduler_addtask(s, task_type_logger, task_subtype_none, 0, if (with_logger) {
0, c, NULL); /* Add the hydro logger task. */
c->logger = scheduler_addtask(s, task_type_logger, task_subtype_none, 0,
0, c, NULL);
/* Add the kick2 dependency */ /* Add the kick2 dependency */
scheduler_addunlock(s, c->kick2, c->logger); scheduler_addunlock(s, c->kick2, c->logger);
/* Create a variable in order to avoid to many ifdef */ /* Create a variable in order to avoid to many ifdef */
struct task *kick2_or_logger = c->logger; kick2_or_logger = c->logger;
}
else {
kick2_or_logger = c->kick2;
}
#else #else
struct task *kick2_or_logger = c->kick2; struct task *kick2_or_logger = c->kick2;
#endif #endif
...@@ -1070,6 +1077,7 @@ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c, ...@@ -1070,6 +1077,7 @@ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c,
const int with_cooling = (e->policy & engine_policy_cooling); const int with_cooling = (e->policy & engine_policy_cooling);
const int with_star_formation = (e->policy & engine_policy_star_formation); const int with_star_formation = (e->policy & engine_policy_star_formation);
const int with_black_holes = (e->policy & engine_policy_black_holes); const int with_black_holes = (e->policy & engine_policy_black_holes);
const int with_logger = (e->policy & engine_policy_logger);
/* Are we are the level where we create the stars' resort tasks? /* Are we are the level where we create the stars' resort tasks?
* If the tree is shallow, we need to do this at the super-level if the * If the tree is shallow, we need to do this at the super-level if the
...@@ -1176,7 +1184,12 @@ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c, ...@@ -1176,7 +1184,12 @@ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c,
task_subtype_none, 0, 0, c, NULL); task_subtype_none, 0, 0, c, NULL);
#ifdef WITH_LOGGER #ifdef WITH_LOGGER
scheduler_addunlock(s, c->super->logger, c->stars.stars_in); if (with_logger) {
scheduler_addunlock(s, c->super->logger, c->stars.stars_in);
}
else {
scheduler_addunlock(s, c->super->kick2, c->stars.stars_in);
}
#else #else
scheduler_addunlock(s, c->super->kick2, c->stars.stars_in); scheduler_addunlock(s, c->super->kick2, c->stars.stars_in);
#endif #endif
...@@ -1209,7 +1222,12 @@ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c, ...@@ -1209,7 +1222,12 @@ void engine_make_hierarchical_tasks_hydro(struct engine *e, struct cell *c,
s, task_type_bh_swallow_ghost3, task_subtype_none, 0, 0, c, NULL); s, task_type_bh_swallow_ghost3, task_subtype_none, 0, 0, c, NULL);
#ifdef WITH_LOGGER #ifdef WITH_LOGGER