Commit 46d7d0ab authored by Matthieu Schaller's avatar Matthieu Schaller
Browse files

Merge branch 'master' into Self_gravity_BH

parents f03c87cf e85ecae0
......@@ -26,6 +26,8 @@ Valid options are:
-f {int} Overwrite the CPU frequency (Hz) to be used for time measurements
-g Run with an external gravitational potential
-G Run with self-gravity
-n {int} Execute a fixed number of time steps. When unset use the time_end
parameter to stop.
-s Run with SPH
-t {int} The number of threads to use on each MPI rank. Defaults to 1 if not specified.
-v [12] Increase the level of verbosity 1: MPI-rank 0 writes
......
......@@ -72,9 +72,6 @@ if test "$enable_ipo" = "yes"; then
fi
fi
# Add libtool support.
LT_INIT
# Check for MPI. Need to do this before characterising the compiler (C99 mode),
# as this changes the compiler.
# We should consider using AX_PROG_CC_MPI to replace AC_PROG_CC when compiling
......@@ -151,6 +148,9 @@ AM_CONDITIONAL([HAVEMPI],[test $enable_mpi = "yes"])
# Indicate that MPIRUN can be modified by an environement variable
AC_ARG_VAR(MPIRUN, Path to the mpirun command if non-standard)
# Add libtool support (now that CC is defined).
LT_INIT
# Need C99 and inline support.
AC_PROG_CC_C99
AC_C_INLINE
......@@ -179,7 +179,7 @@ if test "$enable_opt" = "yes" ; then
ac_test_CFLAGS="yes"
CFLAGS="$old_CFLAGS $CFLAGS"
# Check SSE & AVX support (some overlap with AX_CC_MAXOPT).
# Check SSE & AVX support (some overlap with AX_CC_MAXOPT).
# Don't use the SIMD_FLAGS result with Intel compilers. The -x<code>
# value from AX_CC_MAXOPT should be sufficient.
AX_EXT
......@@ -287,12 +287,75 @@ AC_SUBST([METIS_LIBS])
AC_SUBST([METIS_INCS])
AM_CONDITIONAL([HAVEMETIS],[test -n "$METIS_LIBS"])
# # Check for zlib.
# AC_CHECK_LIB([z],[gzopen],[
# AC_DEFINE([HAVE_LIBZ],[1],[Set to 1 if zlib is installed.])
# LDFLAGS="$LDFLAGS -lz"
# ],[])
# Check for tcmalloc a fast malloc that is part of the gperftools.
have_tcmalloc="no"
AC_ARG_WITH([tcmalloc],
[AS_HELP_STRING([--with-tcmalloc],
[use tcmalloc library or specify the directory with lib @<:@yes/no@:>@]
)],
[with_tcmalloc="$withval"],
[with_tcmalloc="no"]
)
if test "x$with_tcmalloc" != "xno"; then
if test "x$with_tcmalloc" != "xyes" && test "x$with_tcmalloc" != "x"; then
tclibs="-L$with_tcmalloc -ltcmalloc"
else
tclibs="-ltcmalloc"
fi
AC_CHECK_LIB([tcmalloc],[tc_cfree],[have_tcmalloc="yes"],[have_tcmalloc="no"],
$tclibs)
# Could just have the minimal version.
if test "$have_tcmalloc" = "no"; then
if test "x$with_tcmalloc" != "xyes" && test "x$with_tcmalloc" != "x"; then
tclibs="-L$with_tcmalloc -ltcmalloc_minimal"
else
tclibs="-ltcmalloc_minimal"
fi
AC_CHECK_LIB([tcmalloc],[tc_cfree],[have_tcmalloc="yes"],[have_tcmalloc="no"],
$tclibs)
fi
if test "$have_tcmalloc" = "yes"; then
TCMALLOC_LIBS="$tclibs"
# These are recommended for GCC.
if test "$ax_cv_c_compiler_vendor" = "gnu"; then
CFLAGS="$CFLAGS -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free"
fi
else
TCMALLOC_LIBS=""
fi
fi
AC_SUBST([TCMALLOC_LIBS])
AM_CONDITIONAL([HAVETCMALLOC],[test -n "$TCMALLOC_LIBS"])
# Check for -lprofiler usually part of the gpreftools along with tcmalloc.
have_profiler="no"
AC_ARG_WITH([profiler],
[AS_HELP_STRING([--with-profiler],
[use cpu profiler library or specify the directory with lib @<:@yes/no@:>@]
)],
[with_profiler="$withval"],
[with_profiler="yes"]
)
if test "x$with_profiler" != "xno"; then
if test "x$with_profiler" != "xyes" && test "x$with_profiler" != "x"; then
proflibs="-L$with_profiler -lprofiler"
else
proflibs="-lprofiler"
fi
AC_CHECK_LIB([profiler],[ProfilerFlush],[have_profiler="yes"],[have_profiler="no"],
$proflibs)
if test "$have_profiler" = "yes"; then
PROFILER_LIBS="$proflibs"
else
PROFILER_LIBS=""
fi
fi
AC_SUBST([PROFILER_LIBS])
AM_CONDITIONAL([HAVEPROFILER],[test -n "$PROFILER_LIBS"])
# Check for HDF5. This is required.
AX_LIB_HDF5
......@@ -422,6 +485,8 @@ AC_MSG_RESULT([
Metis enabled : $have_metis
FFTW3 enabled : $have_fftw3
libNUMA enabled : $have_numa
Using tcmalloc : $have_tcmalloc
CPU profiler : $have_profiler
])
# Generate output.
......
......@@ -21,13 +21,17 @@ MYFLAGS = -DTIMER
# Add the source directory and debug to CFLAGS
AM_CFLAGS = -I../src $(HDF5_CPPFLAGS)
AM_LDFLAGS =
AM_LDFLAGS = $(HDF5_LDFLAGS)
MPI_THREAD_LIBS = @MPI_THREAD_LIBS@
# Extra libraries.
EXTRA_LIBS = $(HDF5_LIBS) $(FFTW_LIBS) $(PROFILER_LIBS) $(TCMALLOC_LIBS)
# MPI libraries.
MPI_LIBS = $(METIS_LIBS) $(MPI_THREAD_LIBS)
MPI_FLAGS = -DWITH_MPI $(METIS_INCS)
# Set-up the library
# Programs.
bin_PROGRAMS = swift swift_fixdt
# Build MPI versions as well?
......@@ -45,20 +49,20 @@ endif
# Sources for swift
swift_SOURCES = main.c
swift_CFLAGS = $(MYFLAGS) $(AM_CFLAGS) -DENGINE_POLICY="engine_policy_keep $(ENGINE_POLICY_SETAFFINITY)"
swift_LDADD = ../src/.libs/libswiftsim.a $(HDF5_LDFLAGS) $(HDF5_LIBS) $(FFTW_LIBS)
swift_LDADD = ../src/.libs/libswiftsim.a $(EXTRA_LIBS)
swift_fixdt_SOURCES = main.c
swift_fixdt_CFLAGS = $(MYFLAGS) $(AM_CFLAGS) -DENGINE_POLICY="engine_policy_fixdt | engine_policy_keep $(ENGINE_POLICY_SETAFFINITY)"
swift_fixdt_LDADD = ../src/.libs/libswiftsim.a $(HDF5_LDFLAGS) $(HDF5_LIBS) $(FFTW_LIBS)
swift_fixdt_LDADD = ../src/.libs/libswiftsim.a $(EXTRA_LIBS)
# Sources for swift_mpi, do we need an affinity policy for MPI?
swift_mpi_SOURCES = main.c
swift_mpi_CFLAGS = $(MYFLAGS) $(AM_CFLAGS) $(MPI_FLAGS) -DENGINE_POLICY="engine_policy_keep $(ENGINE_POLICY_SETAFFINITY)"
swift_mpi_LDADD = ../src/.libs/libswiftsim_mpi.a $(HDF5_LDFLAGS) $(HDF5_LIBS) $(FFTW_LIBS) $(MPI_LIBS)
swift_mpi_LDADD = ../src/.libs/libswiftsim_mpi.a $(MPI_LIBS) $(EXTRA_LIBS)
swift_fixdt_mpi_SOURCES = main.c
swift_fixdt_mpi_CFLAGS = $(MYFLAGS) $(AM_CFLAGS) $(MPI_FLAGS) -DENGINE_POLICY="engine_policy_fixdt | engine_policy_keep $(ENGINE_POLICY_SETAFFINITY)"
swift_fixdt_mpi_LDADD = ../src/.libs/libswiftsim_mpi.a $(HDF5_LDFLAGS) $(HDF5_LIBS) $(FFTW_LIBS) $(MPI_LIBS)
swift_fixdt_mpi_LDADD = ../src/.libs/libswiftsim_mpi.a $(MPI_LIBS) $(EXTRA_LIBS)
# Scripts to generate ICs
EXTRA_DIST = UniformBox/makeIC.py UniformBox/run.sh UniformBox/uniformBox.yml \
......
......@@ -74,6 +74,8 @@ void print_help_message() {
printf(" %2s %8s %s\n", "-g", "",
"Run with an external gravitational potential");
printf(" %2s %8s %s\n", "-G", "", "Run with self-gravity");
printf(" %2s %8s %s\n", "-n", "{int}",
"Execute a fixed number of time steps");
printf(" %2s %8s %s\n", "-s", "", "Run with SPH");
printf(" %2s %8s %s\n", "-t", "{int}",
"The number of threads to use on each MPI rank. Defaults to 1 if not "
......@@ -138,6 +140,7 @@ int main(int argc, char *argv[]) {
int with_aff = 0;
int dry_run = 0;
int dump_tasks = 0;
int nsteps = -2;
int with_cosmology = 0;
int with_external_gravity = 0;
int with_self_gravity = 0;
......@@ -150,7 +153,7 @@ int main(int argc, char *argv[]) {
/* Parse the parameters */
int c;
while ((c = getopt(argc, argv, "acdef:gGhst:v:y:")) != -1) switch (c) {
while ((c = getopt(argc, argv, "acdef:gGhn:st:v:y:")) != -1) switch (c) {
case 'a':
with_aff = 1;
break;
......@@ -179,6 +182,13 @@ int main(int argc, char *argv[]) {
case 'h':
if (myrank == 0) print_help_message();
return 0;
case 'n':
if (sscanf(optarg, "%d", &nsteps) != 1) {
if (myrank == 0) printf("Error parsing fixed number of steps.\n");
if (myrank == 0) print_help_message();
return 1;
}
break;
case 's':
with_hydro = 1;
break;
......@@ -322,18 +332,21 @@ int main(int argc, char *argv[]) {
size_t Ngas = 0, Ngpart = 0;
double dim[3] = {0., 0., 0.};
int periodic = 0;
int flag_entropy_ICs = 0;
if (myrank == 0) clocks_gettime(&tic);
#if defined(WITH_MPI)
#if defined(HAVE_PARALLEL_HDF5)
read_ic_parallel(ICfileName, dim, &parts, &gparts, &Ngas, &Ngpart, &periodic,
myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL, dry_run);
&flag_entropy_ICs, myrank, nr_nodes, MPI_COMM_WORLD,
MPI_INFO_NULL, dry_run);
#else
read_ic_serial(ICfileName, dim, &parts, &gparts, &Ngas, &Ngpart, &periodic,
myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL, dry_run);
&flag_entropy_ICs, myrank, nr_nodes, MPI_COMM_WORLD,
MPI_INFO_NULL, dry_run);
#endif
#else
read_ic_single(ICfileName, dim, &parts, &gparts, &Ngas, &Ngpart, &periodic,
dry_run);
&flag_entropy_ICs, dry_run);
#endif
if (myrank == 0) {
clocks_gettime(&toc);
......@@ -466,7 +479,7 @@ int main(int argc, char *argv[]) {
#endif
/* Initialise the particles */
engine_init_particles(&e);
engine_init_particles(&e, flag_entropy_ICs);
/* Legend */
if (myrank == 0)
......@@ -474,7 +487,7 @@ int main(int argc, char *argv[]) {
"Updates", "g-Updates", "Wall-clock time", clocks_getunit());
/* Main simulation loop */
for (int j = 0; !engine_is_done(&e); j++) {
for (int j = 0; !engine_is_done(&e) && e.step != nsteps; j++) {
/* Repartition the space amongst the nodes? */
#ifdef WITH_MPI
......
......@@ -24,6 +24,13 @@ AM_LDFLAGS = $(HDF5_LDFLAGS) $(FFTW_LIBS) -version-info 0:0:0
# The git command, if available.
GIT_CMD = @GIT_CMD@
# Additional dependencies for shared libraries.
EXTRA_LIBS = $(HDF5_LIBS) $(PROFILER_LIBS) $(TCMALLOC_LIBS)
# MPI libraries.
MPI_LIBS = $(METIS_LIBS) $(MPI_THREAD_LIBS)
MPI_FLAGS = -DWITH_MPI $(METIS_INCS)
# Build the libswiftsim library
lib_LTLIBRARIES = libswiftsim.la
# Build a MPI-enabled version too?
......@@ -66,11 +73,13 @@ nobase_noinst_HEADERS = approx_math.h atomic.h cycle.h error.h inline.h kernel_h
# Sources and flags for regular library
libswiftsim_la_SOURCES = $(AM_SOURCES)
libswiftsim_la_CFLAGS = $(AM_CFLAGS)
libswiftsim_la_LDFLAGS = $(AM_LDFLAGS) $(EXTRA_LIBS)
# Sources and flags for MPI library
libswiftsim_mpi_la_SOURCES = $(AM_SOURCES)
libswiftsim_mpi_la_CFLAGS = $(AM_CFLAGS) -DWITH_MPI $(METIS_INCS)
libswiftsim_mpi_la_LDFLAGS = $(AM_LDFLAGS) -DWITH_MPI $(METIS_LIBS)
libswiftsim_mpi_la_CFLAGS = $(AM_CFLAGS) $(MPI_FLAGS)
libswiftsim_mpi_la_LDFLAGS = $(AM_LDFLAGS) $(MPI_LIBS) $(EXTRA_LIBS)
libswiftsim_mpi_la_SHORTNAME = mpi
......
......@@ -128,6 +128,7 @@ int cell_unpack(struct pcell *pc, struct cell *c, struct space *s) {
}
/* Return the total number of unpacked cells. */
c->pcell_size = count;
return count;
}
......@@ -213,6 +214,39 @@ int cell_pack(struct cell *c, struct pcell *pc) {
pc->progeny[k] = -1;
/* Return the number of packed cells used. */
c->pcell_size = count;
return count;
}
int cell_pack_ti_ends(struct cell *c, int *ti_ends) {
/* Pack this cell's data. */
ti_ends[0] = c->ti_end_min;
/* Fill in the progeny, depth-first recursion. */
int count = 1;
for (int k = 0; k < 8; k++)
if (c->progeny[k] != NULL) {
count += cell_pack_ti_ends(c->progeny[k], &ti_ends[count]);
}
/* Return the number of packed values. */
return count;
}
int cell_unpack_ti_ends(struct cell *c, int *ti_ends) {
/* Unpack this cell's data. */
c->ti_end_min = ti_ends[0];
/* Fill in the progeny, depth-first recursion. */
int count = 1;
for (int k = 0; k < 8; k++)
if (c->progeny[k] != NULL) {
count += cell_unpack_ti_ends(c->progeny[k], &ti_ends[count]);
}
/* Return the number of packed values. */
return count;
}
......
......@@ -125,7 +125,10 @@ struct cell {
struct task *ghost, *init, *drift, *kick;
/* Task receiving data. */
struct task *recv_xv, *recv_rho;
struct task *recv_xv, *recv_rho, *recv_ti;
/* Task send data. */
struct link *send_xv, *send_rho, *send_ti;
/* Tasks for gravity tree. */
struct task *grav_up, *grav_down;
......@@ -185,6 +188,8 @@ int cell_glocktree(struct cell *c);
void cell_gunlocktree(struct cell *c);
int cell_pack(struct cell *c, struct pcell *pc);
int cell_unpack(struct pcell *pc, struct cell *c, struct space *s);
int cell_pack_ti_ends(struct cell *c, int *ti_ends);
int cell_unpack_ti_ends(struct cell *c, int *ti_ends);
int cell_getsize(struct cell *c);
int cell_link_parts(struct cell *c, struct part *parts);
int cell_link_gparts(struct cell *c, struct gpart *gparts);
......
......@@ -675,44 +675,63 @@ void engine_addtasks_grav(struct engine *e, struct cell *c, struct task *up,
*
* @param e The #engine.
* @param ci The sending #cell.
* @param cj The receiving #cell
* @param cj Dummy cell containing the nodeID of the receiving node.
* @param t_xv The send_xv #task, if it has already been created.
* @param t_rho The send_rho #task, if it has already been created.
* @param t_ti The send_ti #task, if required and has already been created.
*/
void engine_addtasks_send(struct engine *e, struct cell *ci, struct cell *cj) {
void engine_addtasks_send(struct engine *e, struct cell *ci, struct cell *cj,
struct task *t_xv, struct task *t_rho,
struct task *t_ti) {
#ifdef WITH_MPI
struct link *l = NULL;
struct scheduler *s = &e->sched;
const int nodeID = cj->nodeID;
/* Check if any of the density tasks are for the target node. */
for (l = ci->density; l != NULL; l = l->next)
if (l->t->ci->nodeID == cj->nodeID ||
(l->t->cj != NULL && l->t->cj->nodeID == cj->nodeID))
if (l->t->ci->nodeID == nodeID ||
(l->t->cj != NULL && l->t->cj->nodeID == nodeID))
break;
/* If so, attach send tasks. */
if (l != NULL) {
/* Create the tasks. */
struct task *t_xv = scheduler_addtask(s, task_type_send, task_subtype_none,
2 * ci->tag, 0, ci, cj, 0);
struct task *t_rho = scheduler_addtask(s, task_type_send, task_subtype_none,
2 * ci->tag + 1, 0, ci, cj, 0);
/* Create the tasks and their dependencies? */
if (t_xv == NULL) {
t_xv = scheduler_addtask(s, task_type_send, task_subtype_none,
3 * ci->tag, 0, ci, cj, 0);
t_rho = scheduler_addtask(s, task_type_send, task_subtype_none,
3 * ci->tag + 1, 0, ci, cj, 0);
if (!(e->policy & engine_policy_fixdt))
t_ti = scheduler_addtask(s, task_type_send, task_subtype_tend,
3 * ci->tag + 2, 0, ci, cj, 0);
/* The send_rho task depends on the cell's ghost task. */
scheduler_addunlock(s, ci->super->ghost, t_rho);
/* The send_rho task depends on the cell's ghost task. */
scheduler_addunlock(s, ci->super->ghost, t_rho);
/* The send_rho task should unlock the super-cell's kick task. */
scheduler_addunlock(s, t_rho, ci->super->kick);
/* The send_rho task should unlock the super-cell's kick task. */
scheduler_addunlock(s, t_rho, ci->super->kick);
/* The send_xv task should unlock the super-cell's ghost task. */
scheduler_addunlock(s, t_xv, ci->super->ghost);
/* The send_xv task should unlock the super-cell's ghost task. */
scheduler_addunlock(s, t_xv, ci->super->ghost);
/* The super-cell's kick task should unlock the send_ti task. */
if (t_ti != NULL) scheduler_addunlock(s, ci->super->kick, t_ti);
}
/* Add them to the local cell. */
ci->send_xv = engine_addlink(e, ci->send_xv, t_xv);
ci->send_rho = engine_addlink(e, ci->send_rho, t_rho);
if (t_ti != NULL) ci->send_ti = engine_addlink(e, ci->send_ti, t_ti);
}
/* Recurse? */
else if (ci->split)
if (ci->split)
for (int k = 0; k < 8; k++)
if (ci->progeny[k] != NULL) engine_addtasks_send(e, ci->progeny[k], cj);
if (ci->progeny[k] != NULL)
engine_addtasks_send(e, ci->progeny[k], cj, t_xv, t_rho, t_ti);
#else
error("SWIFT was not compiled with MPI support.");
......@@ -723,40 +742,51 @@ void engine_addtasks_send(struct engine *e, struct cell *ci, struct cell *cj) {
* @brief Add recv tasks to a hierarchy of cells.
*
* @param e The #engine.
* @param c The #cell.
* @param c The foreign #cell.
* @param t_xv The recv_xv #task, if it has already been created.
* @param t_rho The recv_rho #task, if it has already been created.
* @param t_ti The recv_ti #task, if required and has already been created.
*/
void engine_addtasks_recv(struct engine *e, struct cell *c, struct task *t_xv,
struct task *t_rho) {
struct task *t_rho, struct task *t_ti) {
#ifdef WITH_MPI
struct scheduler *s = &e->sched;
/* Do we need to construct a recv task? */
if (t_xv == NULL && c->nr_density > 0) {
/* Do we need to construct a recv task?
Note that since c is a foreign cell, all its density tasks will involve
only the current rank, and thus we don't have to check them.*/
if (t_xv == NULL && c->density != NULL) {
/* Create the tasks. */
t_xv = c->recv_xv = scheduler_addtask(s, task_type_recv, task_subtype_none,
2 * c->tag, 0, c, NULL, 0);
t_rho = c->recv_rho = scheduler_addtask(
s, task_type_recv, task_subtype_none, 2 * c->tag + 1, 0, c, NULL, 0);
}
t_xv = scheduler_addtask(s, task_type_recv, task_subtype_none, 3 * c->tag,
0, c, NULL, 0);
t_rho = scheduler_addtask(s, task_type_recv, task_subtype_none,
3 * c->tag + 1, 0, c, NULL, 0);
if (!(e->policy & engine_policy_fixdt))
t_ti = scheduler_addtask(s, task_type_recv, task_subtype_tend,
3 * c->tag + 2, 0, c, NULL, 0);
}
c->recv_xv = t_xv;
c->recv_rho = t_rho;
c->recv_ti = t_ti;
/* Add dependencies. */
for (struct link *l = c->density; l != NULL; l = l->next) {
scheduler_addunlock(s, t_xv, l->t);
scheduler_addunlock(s, l->t, t_rho);
}
for (struct link *l = c->force; l != NULL; l = l->next)
for (struct link *l = c->force; l != NULL; l = l->next) {
scheduler_addunlock(s, t_rho, l->t);
if (t_ti != NULL) scheduler_addunlock(s, l->t, t_ti);
}
if (c->sorts != NULL) scheduler_addunlock(s, t_xv, c->sorts);
/* Recurse? */
if (c->split)
for (int k = 0; k < 8; k++)
if (c->progeny[k] != NULL)
engine_addtasks_recv(e, c->progeny[k], t_xv, t_rho);
engine_addtasks_recv(e, c->progeny[k], t_xv, t_rho, t_ti);
#else
error("SWIFT was not compiled with MPI support.");
......@@ -1782,10 +1812,9 @@ int engine_marktasks(struct engine *e) {
for (int k = 0; k < nr_tasks; k++) {
/* Get a handle on the kth task. */
struct task *t = &tasks[ind[k]];
struct task *t = &tasks[k];
/* Sort-task? Note that due to the task ranking, the sorts
will all come before the pairs. */
/* Sort-task? */
if (t->type == task_type_sort) {
/* Re-set the flags. */
......@@ -1794,6 +1823,24 @@ int engine_marktasks(struct engine *e) {
}
/* Send/recv-task? */
else if (t->type == task_type_send || t->type == task_type_recv) {
t->skip = 1;
}
}
/* Run through the tasks and mark as skip or not. */
for (int k = 0; k < nr_tasks; k++) {
/* Get a handle on the kth task. */
struct task *t = &tasks[k];
/* Skip sorts, sends, and recvs. */
if (t->type == task_type_sort || t->type == task_type_send ||
t->type == task_type_recv) {
continue;
}
/* Single-cell task? */
else if (t->type == task_type_self || t->type == task_type_ghost ||
t->type == task_type_sub_self) {
......@@ -1809,9 +1856,6 @@ int engine_marktasks(struct engine *e) {
const struct cell *ci = t->ci;
const struct cell *cj = t->cj;
/* Set this task's skip. */
t->skip = (ci->ti_end_min > ti_end && cj->ti_end_min > ti_end);
/* Too much particle movement? */
if (t->tight &&
(fmaxf(ci->h_max, cj->h_max) + ci->dx_max + cj->dx_max > cj->dmin ||
......@@ -1819,8 +1863,13 @@ int engine_marktasks(struct engine *e) {
cj->dx_max > space_maxreldx * cj->h_max))
return 1;
/* Set this task's skip. */
if ((t->skip = (ci->ti_end_min > ti_end && cj->ti_end_min > ti_end)) ==
1)
continue;
/* Set the sort flags. */
if (!t->skip && t->type == task_type_pair &&
if (t->type == task_type_pair &&
t->subtype != task_subtype_grav) {
if (!(ci->sorted & (1 << t->flags))) {
ci->sorts->flags |= (1 << t->flags);
......@@ -1832,6 +1881,68 @@ int engine_marktasks(struct engine *e) {
}
}
/* Activate the send/recv flags. */
if (ci->nodeID != e->nodeID) {
/* Activate the tasks to recv foreign cell ci's data. */
ci->recv_xv->skip = 0;
ci->recv_rho->skip = 0;
ci->recv_ti->skip = 0;
/* Look for the local cell cj's send tasks. */
struct link *l = NULL;
for (l = cj->send_xv; l != NULL && l->t->cj->nodeID != ci->nodeID;
l = l->next)
;
if (l == NULL) {
abort();
error("Missing link to send_xv task.");
}
l->t->skip = 0;
for (l = cj->send_rho; l != NULL && l->t->cj->nodeID != ci->nodeID;
l = l->next)
;
if (l == NULL) error("Missing link to send_rho task.");
l->t->skip = 0;
for (l = cj->send_ti; l != NULL && l->t->cj->nodeID != ci->nodeID;
l = l->next)
;
if (l == NULL) error("Missing link to send_ti task.");
l->t->skip = 0;
} else if (cj->nodeID != e->nodeID) {
/* Activate the tasks to recv foreign cell cj's data. */
cj->recv_xv->skip = 0;
cj->recv_rho->skip = 0;
cj->recv_ti->skip = 0;
/* Look for the local cell ci's send tasks. */
struct link *l = NULL;
for (l = ci->send_xv; l != NULL && l->t->cj->nodeID != cj->nodeID;
l = l->next)
;
if (l == NULL) {
abort();
error("Missing link to send_xv task.");
}
l->t->skip = 0;
for (l = ci->send_rho; l != NULL && l->t->cj->nodeID != cj->nodeID;
l = l->next)
;
if (l == NULL) error("Missing link to send_rho task.");
l->t->skip = 0;
for (l = ci->send_ti; l != NULL && l->t->cj->nodeID != cj->nodeID;
l = l->next)
;
if (l == NULL) error("Missing link to send_ti task.");
l->t->skip = 0;
}