Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
SWIFT
SWIFTsim
Commits
e7b9d012
Commit
e7b9d012
authored
May 02, 2019
by
Matthieu Schaller
Browse files
Added MPI tasks for the BH density and feedback.
parent
447086d3
Changes
13
Hide whitespace changes
Inline
Side-by-side
examples/main.c
View file @
e7b9d012
...
...
@@ -511,7 +511,6 @@ int main(int argc, char *argv[]) {
if
(
with_star_formation
&&
with_feedback
)
error
(
"Can't run with star formation and feedback over MPI (yet)"
);
if
(
with_limiter
)
error
(
"Can't run with time-step limiter over MPI (yet)"
);
if
(
with_black_holes
)
error
(
"Can't run with black holes over MPI (yet)"
);
#endif
/* Temporary early aborts for modes not supported with hand-vec. */
...
...
src/active.h
View file @
e7b9d012
...
...
@@ -372,7 +372,7 @@ __attribute__((always_inline)) INLINE static int bpart_is_active(
if
(
ti_end
<
ti_current
)
error
(
"
s
-particle in an impossible time-zone! bp->ti_end=%lld "
"
b
-particle in an impossible time-zone! bp->ti_end=%lld "
"e->ti_current=%lld"
,
ti_end
,
ti_current
);
#endif
...
...
src/cell.c
View file @
e7b9d012
...
...
@@ -497,12 +497,15 @@ int cell_pack(struct cell *restrict c, struct pcell *restrict pc,
/* Start by packing the data of the current cell. */
pc
->
hydro
.
h_max
=
c
->
hydro
.
h_max
;
pc
->
stars
.
h_max
=
c
->
stars
.
h_max
;
pc
->
black_holes
.
h_max
=
c
->
black_holes
.
h_max
;
pc
->
hydro
.
ti_end_min
=
c
->
hydro
.
ti_end_min
;
pc
->
hydro
.
ti_end_max
=
c
->
hydro
.
ti_end_max
;
pc
->
grav
.
ti_end_min
=
c
->
grav
.
ti_end_min
;
pc
->
grav
.
ti_end_max
=
c
->
grav
.
ti_end_max
;
pc
->
stars
.
ti_end_min
=
c
->
stars
.
ti_end_min
;
pc
->
stars
.
ti_end_max
=
c
->
stars
.
ti_end_max
;
pc
->
black_holes
.
ti_end_min
=
c
->
black_holes
.
ti_end_min
;
pc
->
black_holes
.
ti_end_max
=
c
->
black_holes
.
ti_end_max
;
pc
->
hydro
.
ti_old_part
=
c
->
hydro
.
ti_old_part
;
pc
->
grav
.
ti_old_part
=
c
->
grav
.
ti_old_part
;
pc
->
grav
.
ti_old_multipole
=
c
->
grav
.
ti_old_multipole
;
...
...
@@ -510,6 +513,7 @@ int cell_pack(struct cell *restrict c, struct pcell *restrict pc,
pc
->
hydro
.
count
=
c
->
hydro
.
count
;
pc
->
grav
.
count
=
c
->
grav
.
count
;
pc
->
stars
.
count
=
c
->
stars
.
count
;
pc
->
black_holes
.
count
=
c
->
black_holes
.
count
;
pc
->
maxdepth
=
c
->
maxdepth
;
/* Copy the Multipole related information */
...
...
@@ -602,19 +606,24 @@ int cell_unpack(struct pcell *restrict pc, struct cell *restrict c,
/* Unpack the current pcell. */
c
->
hydro
.
h_max
=
pc
->
hydro
.
h_max
;
c
->
stars
.
h_max
=
pc
->
stars
.
h_max
;
c
->
black_holes
.
h_max
=
pc
->
black_holes
.
h_max
;
c
->
hydro
.
ti_end_min
=
pc
->
hydro
.
ti_end_min
;
c
->
hydro
.
ti_end_max
=
pc
->
hydro
.
ti_end_max
;
c
->
grav
.
ti_end_min
=
pc
->
grav
.
ti_end_min
;
c
->
grav
.
ti_end_max
=
pc
->
grav
.
ti_end_max
;
c
->
stars
.
ti_end_min
=
pc
->
stars
.
ti_end_min
;
c
->
stars
.
ti_end_max
=
pc
->
stars
.
ti_end_max
;
c
->
black_holes
.
ti_end_min
=
pc
->
black_holes
.
ti_end_min
;
c
->
black_holes
.
ti_end_max
=
pc
->
black_holes
.
ti_end_max
;
c
->
hydro
.
ti_old_part
=
pc
->
hydro
.
ti_old_part
;
c
->
grav
.
ti_old_part
=
pc
->
grav
.
ti_old_part
;
c
->
grav
.
ti_old_multipole
=
pc
->
grav
.
ti_old_multipole
;
c
->
stars
.
ti_old_part
=
pc
->
stars
.
ti_old_part
;
c
->
black_holes
.
ti_old_part
=
pc
->
black_holes
.
ti_old_part
;
c
->
hydro
.
count
=
pc
->
hydro
.
count
;
c
->
grav
.
count
=
pc
->
grav
.
count
;
c
->
stars
.
count
=
pc
->
stars
.
count
;
c
->
black_holes
.
count
=
pc
->
black_holes
.
count
;
c
->
maxdepth
=
pc
->
maxdepth
;
#ifdef SWIFT_DEBUG_CHECKS
...
...
@@ -664,6 +673,7 @@ int cell_unpack(struct pcell *restrict pc, struct cell *restrict c,
temp
->
hydro
.
dx_max_sort
=
0
.
f
;
temp
->
stars
.
dx_max_part
=
0
.
f
;
temp
->
stars
.
dx_max_sort
=
0
.
f
;
temp
->
black_holes
.
dx_max_part
=
0
.
f
;
temp
->
nodeID
=
c
->
nodeID
;
temp
->
parent
=
c
;
c
->
progeny
[
k
]
=
temp
;
...
...
@@ -2452,7 +2462,6 @@ void cell_activate_stars_sorts_up(struct cell *c, struct scheduler *s) {
#endif
scheduler_activate
(
s
,
c
->
stars
.
sorts
);
if
(
c
->
nodeID
==
engine_rank
)
{
// MATTHIEU: to do: do we actually need both drifts here?
cell_activate_drift_spart
(
c
,
s
);
}
}
else
{
...
...
@@ -3610,22 +3619,18 @@ int cell_unskip_black_holes_tasks(struct cell *c, struct scheduler *s) {
scheduler_activate
(
s
,
ci
->
mpi
.
hydro
.
recv_rho
);
/* If the local cell is active, more stuff will be needed. */
error
(
"MATTHIEU: needs implementing"
);
// scheduler_activate_send(s, cj->mpi.black_holes.send, ci_nodeID);
scheduler_activate_send
(
s
,
cj
->
mpi
.
black_holes
.
send
,
ci_nodeID
);
cell_activate_drift_bpart
(
cj
,
s
);
/* If the local cell is active, send its ti_end values. */
error
(
"MATTHIEU: needs implementing"
);
// scheduler_activate_send(s, cj->mpi.black_holes.send_ti, ci_nodeID);
scheduler_activate_send
(
s
,
cj
->
mpi
.
black_holes
.
send_ti
,
ci_nodeID
);
}
if
(
ci_active
)
{
error
(
"MATTHIEU: needs implementing"
);
// scheduler_activate(s, ci->mpi.black_holes.recv);
scheduler_activate
(
s
,
ci
->
mpi
.
black_holes
.
recv
);
/* If the foreign cell is active, we want its ti_end values. */
error
(
"MATTHIEU: needs implementing"
);
// scheduler_activate(s, ci->mpi.black_holes.recv_ti);
scheduler_activate
(
s
,
ci
->
mpi
.
black_holes
.
recv_ti
);
/* Is the foreign cell active and will need stuff from us? */
scheduler_activate_send
(
s
,
cj
->
mpi
.
hydro
.
send_xv
,
ci_nodeID
);
...
...
@@ -3643,22 +3648,18 @@ int cell_unskip_black_holes_tasks(struct cell *c, struct scheduler *s) {
scheduler_activate
(
s
,
cj
->
mpi
.
hydro
.
recv_rho
);
/* If the local cell is active, more stuff will be needed. */
error
(
"MATTHIEU: needs implementing"
);
// scheduler_activate_send(s, ci->mpi.black_holes.send, cj_nodeID);
scheduler_activate_send
(
s
,
ci
->
mpi
.
black_holes
.
send
,
cj_nodeID
);
cell_activate_drift_bpart
(
ci
,
s
);
/* If the local cell is active, send its ti_end values. */
error
(
"MATTHIEU: needs implementing"
);
// scheduler_activate_send(s, ci->mpi.black_holes.send_ti, cj_nodeID);
scheduler_activate_send
(
s
,
ci
->
mpi
.
black_holes
.
send_ti
,
cj_nodeID
);
}
if
(
cj_active
)
{
error
(
"MATTHIEU: needs implementing"
);
// scheduler_activate(s, cj->mpi.black_holes.recv);
scheduler_activate
(
s
,
cj
->
mpi
.
black_holes
.
recv
);
/* If the foreign cell is active, we want its ti_end values. */
error
(
"MATTHIEU: needs implementing"
);
// scheduler_activate(s, cj->mpi.black_holes.recv_ti);
scheduler_activate
(
s
,
cj
->
mpi
.
black_holes
.
recv_ti
);
/* Is the foreign cell active and will need stuff from us? */
scheduler_activate_send
(
s
,
ci
->
mpi
.
hydro
.
send_xv
,
cj_nodeID
);
...
...
@@ -4573,7 +4574,8 @@ void cell_check_timesteps(struct cell *c) {
#ifdef SWIFT_DEBUG_CHECKS
if
(
c
->
hydro
.
ti_end_min
==
0
&&
c
->
grav
.
ti_end_min
==
0
&&
c
->
stars
.
ti_end_min
==
0
&&
c
->
nr_tasks
>
0
)
c
->
stars
.
ti_end_min
==
0
&&
c
->
black_holes
.
ti_end_min
==
0
&&
c
->
nr_tasks
>
0
)
error
(
"Cell without assigned time-step"
);
if
(
c
->
split
)
{
...
...
src/cell.h
View file @
e7b9d012
...
...
@@ -172,6 +172,26 @@ struct pcell {
}
stars
;
/*! Black hole variables */
struct
{
/*! Number of #spart in this cell. */
int
count
;
/*! Maximal smoothing length. */
double
h_max
;
/*! Minimal integer end-of-timestep in this cell for black hole tasks */
integertime_t
ti_end_min
;
/*! Maximal integer end-of-timestep in this cell for black hole tasks */
integertime_t
ti_end_max
;
/*! Integer time of the last drift of the #spart in this cell */
integertime_t
ti_old_part
;
}
black_holes
;
/*! Maximal depth in that part of the tree */
int
maxdepth
;
...
...
src/engine.c
View file @
e7b9d012
...
...
@@ -2875,9 +2875,7 @@ void engine_collect_end_of_step_recurse_black_holes(struct cell *c,
/* Skip super-cells (Their values are already set) */
#ifdef WITH_MPI
// MATTHIEU
if
(
c
->
timestep
!=
NULL
)
return
;
// || c->mpi.black_holes.recv_ti != NULL) return;
if
(
c
->
timestep
!=
NULL
||
c
->
mpi
.
black_holes
.
recv_ti
!=
NULL
)
return
;
#else
if
(
c
->
timestep
!=
NULL
)
return
;
#endif
/* WITH_MPI */
...
...
@@ -3305,8 +3303,9 @@ void engine_skip_force_and_kick(struct engine *e) {
/* Skip everything that updates the particles */
if
(
t
->
type
==
task_type_drift_part
||
t
->
type
==
task_type_drift_gpart
||
t
->
type
==
task_type_drift_spart
||
t
->
type
==
task_type_kick1
||
t
->
type
==
task_type_kick2
||
t
->
type
==
task_type_timestep
||
t
->
type
==
task_type_drift_spart
||
t
->
type
==
task_type_drift_bpart
||
t
->
type
==
task_type_kick1
||
t
->
type
==
task_type_kick2
||
t
->
type
==
task_type_timestep
||
t
->
type
==
task_type_timestep_limiter
||
t
->
subtype
==
task_subtype_force
||
t
->
subtype
==
task_subtype_limiter
||
t
->
subtype
==
task_subtype_grav
||
...
...
@@ -3324,6 +3323,7 @@ void engine_skip_force_and_kick(struct engine *e) {
t
->
subtype
==
task_subtype_tend_part
||
t
->
subtype
==
task_subtype_tend_gpart
||
t
->
subtype
==
task_subtype_tend_spart
||
t
->
subtype
==
task_subtype_tend_bpart
||
t
->
subtype
==
task_subtype_rho
||
t
->
subtype
==
task_subtype_gpart
)
t
->
skip
=
1
;
}
...
...
src/engine_maketasks.c
View file @
e7b9d012
...
...
@@ -292,6 +292,75 @@ void engine_addtasks_send_stars(struct engine *e, struct cell *ci,
#endif
}
/**
* @brief Add send tasks for the black holes pairs to a hierarchy of cells.
*
* @param e The #engine.
* @param ci The sending #cell.
* @param cj Dummy cell containing the nodeID of the receiving node.
* @param t_feedback The send_feed #task, if it has already been created.
* @param t_ti The recv_ti_end #task, if it has already been created.
*/
void
engine_addtasks_send_black_holes
(
struct
engine
*
e
,
struct
cell
*
ci
,
struct
cell
*
cj
,
struct
task
*
t_feedback
,
struct
task
*
t_ti
)
{
#ifdef WITH_MPI
struct
link
*
l
=
NULL
;
struct
scheduler
*
s
=
&
e
->
sched
;
const
int
nodeID
=
cj
->
nodeID
;
/* Check if any of the density tasks are for the target node. */
for
(
l
=
ci
->
black_holes
.
density
;
l
!=
NULL
;
l
=
l
->
next
)
if
(
l
->
t
->
ci
->
nodeID
==
nodeID
||
(
l
->
t
->
cj
!=
NULL
&&
l
->
t
->
cj
->
nodeID
==
nodeID
))
break
;
/* If so, attach send tasks. */
if
(
l
!=
NULL
)
{
if
(
t_feedback
==
NULL
)
{
/* Make sure this cell is tagged. */
cell_ensure_tagged
(
ci
);
/* Create the tasks and their dependencies? */
t_feedback
=
scheduler_addtask
(
s
,
task_type_send
,
task_subtype_bpart
,
ci
->
mpi
.
tag
,
0
,
ci
,
cj
);
t_ti
=
scheduler_addtask
(
s
,
task_type_send
,
task_subtype_tend_bpart
,
ci
->
mpi
.
tag
,
0
,
ci
,
cj
);
/* The send_black_holes task should unlock the super_cell's kick task. */
scheduler_addunlock
(
s
,
t_feedback
,
ci
->
hydro
.
super
->
black_holes
.
black_holes_out
);
/* Ghost before you send */
scheduler_addunlock
(
s
,
ci
->
hydro
.
super
->
black_holes
.
ghost
,
t_feedback
);
/* Drift before you send */
scheduler_addunlock
(
s
,
ci
->
hydro
.
super
->
black_holes
.
drift
,
t_feedback
);
scheduler_addunlock
(
s
,
ci
->
super
->
timestep
,
t_ti
);
}
engine_addlink
(
e
,
&
ci
->
mpi
.
black_holes
.
send
,
t_feedback
);
engine_addlink
(
e
,
&
ci
->
mpi
.
black_holes
.
send_ti
,
t_ti
);
}
/* Recurse? */
if
(
ci
->
split
)
for
(
int
k
=
0
;
k
<
8
;
k
++
)
if
(
ci
->
progeny
[
k
]
!=
NULL
)
engine_addtasks_send_black_holes
(
e
,
ci
->
progeny
[
k
],
cj
,
t_feedback
,
t_ti
);
#else
error
(
"SWIFT was not compiled with MPI support."
);
#endif
}
/**
* @brief Add recv tasks for hydro pairs to a hierarchy of cells.
*
...
...
@@ -445,6 +514,64 @@ void engine_addtasks_recv_stars(struct engine *e, struct cell *c,
#endif
}
/**
* @brief Add recv tasks for black_holes pairs to a hierarchy of cells.
*
* @param e The #engine.
* @param c The foreign #cell.
* @param t_feedback The recv_feed #task, if it has already been created.
* @param t_ti The recv_ti_end #task, if it has already been created.
*/
void
engine_addtasks_recv_black_holes
(
struct
engine
*
e
,
struct
cell
*
c
,
struct
task
*
t_feedback
,
struct
task
*
t_ti
)
{
#ifdef WITH_MPI
struct
scheduler
*
s
=
&
e
->
sched
;
/* Have we reached a level where there are any black_holes tasks ? */
if
(
t_feedback
==
NULL
&&
c
->
black_holes
.
density
!=
NULL
)
{
#ifdef SWIFT_DEBUG_CHECKS
/* Make sure this cell has a valid tag. */
if
(
c
->
mpi
.
tag
<
0
)
error
(
"Trying to receive from untagged cell."
);
#endif // SWIFT_DEBUG_CHECKS
/* Create the tasks. */
t_feedback
=
scheduler_addtask
(
s
,
task_type_recv
,
task_subtype_bpart
,
c
->
mpi
.
tag
,
0
,
c
,
NULL
);
t_ti
=
scheduler_addtask
(
s
,
task_type_recv
,
task_subtype_tend_bpart
,
c
->
mpi
.
tag
,
0
,
c
,
NULL
);
}
c
->
mpi
.
black_holes
.
recv
=
t_feedback
;
c
->
mpi
.
black_holes
.
recv_ti
=
t_ti
;
#ifdef SWIFT_DEBUG_CHECKS
if
(
c
->
nodeID
==
e
->
nodeID
)
error
(
"Local cell!"
);
#endif
for
(
struct
link
*
l
=
c
->
black_holes
.
density
;
l
!=
NULL
;
l
=
l
->
next
)
{
scheduler_addunlock
(
s
,
l
->
t
,
t_feedback
);
}
for
(
struct
link
*
l
=
c
->
black_holes
.
feedback
;
l
!=
NULL
;
l
=
l
->
next
)
{
scheduler_addunlock
(
s
,
t_feedback
,
l
->
t
);
scheduler_addunlock
(
s
,
l
->
t
,
t_ti
);
}
/* Recurse? */
if
(
c
->
split
)
for
(
int
k
=
0
;
k
<
8
;
k
++
)
if
(
c
->
progeny
[
k
]
!=
NULL
)
engine_addtasks_recv_black_holes
(
e
,
c
->
progeny
[
k
],
t_feedback
,
t_ti
);
#else
error
(
"SWIFT was not compiled with MPI support."
);
#endif
}
/**
* @brief Add recv tasks for gravity pairs to a hierarchy of cells.
*
...
...
@@ -1502,7 +1629,7 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
}
/* The black hole feedback tasks */
if
(
with_
feedback
)
{
if
(
with_
black_holes
)
{
t_bh_density
=
scheduler_addtask
(
sched
,
task_type_self
,
task_subtype_bh_density
,
flags
,
0
,
ci
,
NULL
);
t_bh_feedback
=
...
...
@@ -1563,7 +1690,7 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
ci
->
hydro
.
super
->
stars
.
stars_out
);
}
if
(
with_
feedback
)
{
if
(
with_
black_holes
)
{
scheduler_addunlock
(
sched
,
ci
->
hydro
.
super
->
black_holes
.
drift
,
t_bh_density
);
...
...
@@ -1959,7 +2086,7 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
}
/* The black hole feedback tasks */
if
(
with_
feedback
)
{
if
(
with_
black_holes
)
{
t_bh_density
=
scheduler_addtask
(
sched
,
task_type_sub_pair
,
task_subtype_bh_density
,
flags
,
0
,
ci
,
cj
);
...
...
@@ -2107,7 +2234,7 @@ void engine_make_extra_hydroloop_tasks_mapper(void *map_data, int num_elements,
cj
->
hydro
.
super
->
stars
.
stars_out
);
}
if
(
with_
feedback
)
{
if
(
with_
black_holes
)
{
scheduler_addunlock
(
sched
,
cj
->
hydro
.
super
->
black_holes
.
drift
,
t_bh_density
);
...
...
@@ -2160,6 +2287,7 @@ void engine_make_hydroloop_tasks_mapper(void *map_data, int num_elements,
const
int
periodic
=
e
->
s
->
periodic
;
const
int
with_feedback
=
(
e
->
policy
&
engine_policy_feedback
);
const
int
with_stars
=
(
e
->
policy
&
engine_policy_stars
);
const
int
with_black_holes
=
(
e
->
policy
&
engine_policy_black_holes
);
struct
space
*
s
=
e
->
s
;
struct
scheduler
*
sched
=
&
e
->
sched
;
...
...
@@ -2182,7 +2310,8 @@ void engine_make_hydroloop_tasks_mapper(void *map_data, int num_elements,
struct
cell
*
ci
=
&
cells
[
cid
];
/* Skip cells without hydro or star particles */
if
((
ci
->
hydro
.
count
==
0
)
&&
(
!
with_stars
||
ci
->
stars
.
count
==
0
))
if
((
ci
->
hydro
.
count
==
0
)
&&
(
!
with_stars
||
ci
->
stars
.
count
==
0
)
&&
(
!
with_black_holes
||
ci
->
black_holes
.
count
==
0
))
continue
;
/* If the cell is local build a self-interaction */
...
...
@@ -2212,7 +2341,8 @@ void engine_make_hydroloop_tasks_mapper(void *map_data, int num_elements,
/* Is that neighbour local and does it have gas or star particles ? */
if
((
cid
>=
cjd
)
||
((
cj
->
hydro
.
count
==
0
)
&&
(
!
with_feedback
||
cj
->
stars
.
count
==
0
))
||
(
!
with_feedback
||
cj
->
stars
.
count
==
0
)
&&
(
!
with_black_holes
||
cj
->
black_holes
.
count
==
0
))
||
(
ci
->
nodeID
!=
nodeID
&&
cj
->
nodeID
!=
nodeID
))
continue
;
...
...
@@ -2287,9 +2417,6 @@ void engine_addtasks_send_mapper(void *map_data, int num_elements,
struct
cell
*
cj
=
cell_type_pairs
[
k
].
cj
;
const
int
type
=
cell_type_pairs
[
k
].
type
;
/* Add the send task for the particle timesteps. */
// engine_addtasks_send_timestep(e, ci, cj, NULL, NULL, with_limiter);
/* Add the send tasks for the cells in the proxy that have a hydro
* connection. */
if
((
e
->
policy
&
engine_policy_hydro
)
&&
(
type
&
proxy_cell_type_hydro
))
...
...
@@ -2302,6 +2429,13 @@ void engine_addtasks_send_mapper(void *map_data, int num_elements,
if
((
e
->
policy
&
engine_policy_feedback
)
&&
(
type
&
proxy_cell_type_hydro
))
engine_addtasks_send_stars
(
e
,
ci
,
cj
,
/*t_feedback=*/
NULL
,
/*t_ti=*/
NULL
);
/* Add the send tasks for the cells in the proxy that have a black holes
* connection. */
if
((
e
->
policy
&
engine_policy_black_holes
)
&&
(
type
&
proxy_cell_type_hydro
))
engine_addtasks_send_black_holes
(
e
,
ci
,
cj
,
/*t_feedback=*/
NULL
,
/*t_ti=*/
NULL
);
/* Add the send tasks for the cells in the proxy that have a gravity
* connection. */
if
((
e
->
policy
&
engine_policy_self_gravity
)
&&
...
...
@@ -2321,9 +2455,6 @@ void engine_addtasks_recv_mapper(void *map_data, int num_elements,
struct
cell
*
ci
=
cell_type_pairs
[
k
].
ci
;
const
int
type
=
cell_type_pairs
[
k
].
type
;
/* Add the recv task for the particle timesteps. */
// engine_addtasks_recv_timestep(e, ci, NULL, NULL, with_limiter);
/* Add the recv tasks for the cells in the proxy that have a hydro
* connection. */
if
((
e
->
policy
&
engine_policy_hydro
)
&&
(
type
&
proxy_cell_type_hydro
))
...
...
@@ -2334,6 +2465,11 @@ void engine_addtasks_recv_mapper(void *map_data, int num_elements,
if
((
e
->
policy
&
engine_policy_feedback
)
&&
(
type
&
proxy_cell_type_hydro
))
engine_addtasks_recv_stars
(
e
,
ci
,
NULL
,
NULL
);
/* Add the recv tasks for the cells in the proxy that have a black holes
* connection. */
if
((
e
->
policy
&
engine_policy_feedback
)
&&
(
type
&
proxy_cell_type_hydro
))
engine_addtasks_recv_black_holes
(
e
,
ci
,
NULL
,
NULL
);
/* Add the recv tasks for the cells in the proxy that have a gravity
* connection. */
if
((
e
->
policy
&
engine_policy_self_gravity
)
&&
...
...
src/engine_marktasks.c
View file @
e7b9d012
...
...
@@ -637,23 +637,18 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
scheduler_activate
(
s
,
ci
->
mpi
.
hydro
.
recv_rho
);
/* If the local cell is active, more stuff will be needed. */
error
(
"MATTHIEU: needs implementing"
);
// scheduler_activate_send(s, cj->mpi.black_holes.send, ci_nodeID);
scheduler_activate_send
(
s
,
cj
->
mpi
.
black_holes
.
send
,
ci_nodeID
);
cell_activate_drift_bpart
(
cj
,
s
);
/* If the local cell is active, send its ti_end values. */
error
(
"MATTHIEU: needs implementing"
);
// scheduler_activate_send(s, cj->mpi.black_holes.send_ti,
// ci_nodeID);
scheduler_activate_send
(
s
,
cj
->
mpi
.
black_holes
.
send_ti
,
ci_nodeID
);
}
if
(
ci_active_black_holes
)
{
error
(
"MATTHIEU: needs implementing"
);
// scheduler_activate(s, ci->mpi.black_holes.recv);
scheduler_activate
(
s
,
ci
->
mpi
.
black_holes
.
recv
);
/* If the foreign cell is active, we want its ti_end values. */
error
(
"MATTHIEU: needs implementing"
);
// scheduler_activate(s, ci->mpi.black_holes.recv_ti);
scheduler_activate
(
s
,
ci
->
mpi
.
black_holes
.
recv_ti
);
/* Is the foreign cell active and will need stuff from us? */
scheduler_activate_send
(
s
,
cj
->
mpi
.
hydro
.
send_xv
,
ci_nodeID
);
...
...
@@ -672,23 +667,18 @@ void engine_marktasks_mapper(void *map_data, int num_elements,
scheduler_activate
(
s
,
cj
->
mpi
.
hydro
.
recv_rho
);
/* If the local cell is active, more stuff will be needed. */
error
(
"MATTHIEU: needs implementing"
);
// scheduler_activate_send(s, ci->mpi.black_holes.send, cj_nodeID);
scheduler_activate_send
(
s
,
ci
->
mpi
.
black_holes
.
send
,
cj_nodeID
);
cell_activate_drift_bpart
(
ci
,
s
);
/* If the local cell is active, send its ti_end values. */
error
(
"MATTHIEU: needs implementing"
);
// scheduler_activate_send(s, ci->mpi.black_holes.send_ti,
// cj_nodeID);
scheduler_activate_send
(
s
,
ci
->
mpi
.
black_holes
.
send_ti
,
cj_nodeID
);
}
if
(
cj_active_black_holes
)
{
error
(
"MATTHIEU: needs implementing"
);
// scheduler_activate(s, cj->mpi.black_holes.recv);
scheduler_activate
(
s
,
cj
->
mpi
.
black_holes
.
recv
);
/* If the foreign cell is active, we want its ti_end values. */
error
(
"MATTHIEU: needs implementing"
);
// scheduler_activate(s, cj->mpi.black_holes.recv_ti);
scheduler_activate
(
s
,
cj
->
mpi
.
black_holes
.
recv_ti
);
/* Is the foreign cell active and will need stuff from us? */
scheduler_activate_send
(
s
,
ci
->
mpi
.
hydro
.
send_xv
,
cj_nodeID
);
...
...
src/error.h
View file @
e7b9d012
...
...
@@ -64,7 +64,7 @@ extern int engine_rank;
clocks_get_timesincestart(), __FILE__, __FUNCTION__, __LINE__, \
##__VA_ARGS__); \
memdump(engine_rank); \
MPI_Abort(MPI_COMM_WORLD, -1);
\
swift_abort(-1);
\
})
#else
extern
int
engine_rank
;
...
...
src/runner.c
View file @
e7b9d012
...
...
@@ -3845,6 +3845,92 @@ void runner_do_recv_spart(struct runner *r, struct cell *c, int clear_sorts,
#endif
}
/**
* @brief Construct the cell properties from the received #bpart.
*
* Note that we do not need to clear the sorts since we do not sort
* the black holes.
*
* @param r The runner thread.
* @param c The cell.
* @param clear_sorts Should we clear the sort flag and hence trigger a sort ?
* @param timer Are we timing this ?
*/
void
runner_do_recv_bpart
(
struct
runner
*
r
,
struct
cell
*
c
,
int
clear_sorts
,
int
timer
)
{
#ifdef WITH_MPI
struct
bpart
*
restrict
bparts
=
c
->
black_holes
.
parts
;
const
size_t
nr_bparts
=
c
->
black_holes
.
count
;
const
integertime_t
ti_current
=
r
->
e
->
ti_current
;
TIMER_TIC
;
integertime_t
ti_black_holes_end_min
=
max_nr_timesteps
;
integertime_t
ti_black_holes_end_max
=
0
;
timebin_t
time_bin_min
=
num_time_bins
;
timebin_t
time_bin_max
=
0
;
float
h_max
=
0
.
f
;
#ifdef SWIFT_DEBUG_CHECKS
if
(
c
->
nodeID
==
engine_rank
)
error
(
"Updating a local cell!"
);
#endif
/* If this cell is a leaf, collect the particle data. */
if
(
!
c
->
split
)
{
/* Collect everything... */
for
(
size_t
k
=
0
;
k
<
nr_bparts
;
k
++
)
{
#ifdef DEBUG_INTERACTIONS_BLACK_HOLES
bparts
[
k
].
num_ngb_force
=
0
;
#endif
if
(
bparts
[
k
].
time_bin
==
time_bin_inhibited
)
continue
;
time_bin_min
=
min
(
time_bin_min
,
bparts
[
k
].
time_bin
);
time_bin_max
=
max
(
time_bin_max
,
bparts
[
k
].
time_bin
);
h_max
=
max
(
h_max
,
bparts
[
k
].
h
);
}
/* Convert into a time */
ti_black_holes_end_min
=
get_integer_time_end
(
ti_current
,
time_bin_min
);
ti_black_holes_end_max
=
get_integer_time_end
(
ti_current
,
time_bin_max
);
}
/* Otherwise, recurse and collect. */
else
{
for
(
int
k
=
0
;
k
<
8
;
k
++
)
{
if
(
c
->
progeny
[
k
]
!=
NULL
&&
c
->
progeny
[
k
]
->
black_holes
.
count
>
0
)
{
runner_do_recv_bpart
(
r
,
c
->
progeny
[
k
],
clear_sorts
,
0
);
ti_black_holes_end_min
=
min
(
ti_black_holes_end_min
,
c
->
progeny
[
k
]
->
black_holes
.
ti_end_min
);
ti_black_holes_end_max
=
max
(
ti_black_holes_end_max
,
c
->
progeny
[
k
]
->
grav
.
ti_end_max
);
h_max
=
max
(
h_max
,
c
->
progeny
[
k
]
->
black_holes
.
h_max
);
}
}
}
#ifdef SWIFT_DEBUG_CHECKS
if
(
ti_black_holes_end_min
<
ti_current
)
error
(
"Received a cell at an incorrect time c->ti_end_min=%lld, "
"e->ti_current=%lld."
,
ti_black_holes_end_min
,
ti_current
);
#endif
/* ... and store. */
// c->grav.ti_end_min = ti_gravity_end_min;
// c->grav.ti_end_max = ti_gravity_end_max;
c
->
black_holes
.
ti_old_part
=
ti_current
;
c
->
black_holes
.
h_max
=
h_max
;
if
(
timer
)
TIMER_TOC
(
timer_dorecv_bpart
);
#else
error
(
"SWIFT was not compiled with MPI support."
);
#endif
}
/**
* @brief The #runner main thread routine.
*
...
...
@@ -4113,6 +4199,8 @@ void *runner_main(void *data) {
runner_do_recv_gpart
(
r
,
ci
,
1
);
}
else
if
(
t
->
subtype
==
task_subtype_spart
)
{
runner_do_recv_spart
(
r
,
ci
,
1
,
1
);
}
else
if
(
t
->
subtype
==
task_subtype_bpart
)
{
runner_do_recv_bpart
(
r
,
ci
,
1
,
1
);
}
else
if
(
t
->
subtype
==
task_subtype_multipole
)
{
cell_unpack_multipoles
(
ci
,
(
struct
gravity_tensors
*
)
t
->
buff
);
free
(
t
->
buff
);