Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
SWIFT
SWIFTsim
Commits
43682ecb
Commit
43682ecb
authored
Apr 11, 2019
by
Matthieu Schaller
Browse files
Also allow black hole particles when running over MPI.
parent
75e14a5d
Changes
18
Expand all
Hide whitespace changes
Inline
Side-by-side
examples/main.c
View file @
43682ecb
...
...
@@ -499,7 +499,6 @@ int main(int argc, char *argv[]) {
if
(
with_star_formation
&&
with_feedback
)
error
(
"Can't run with star formation and feedback over MPI (yet)"
);
if
(
with_limiter
)
error
(
"Can't run with time-step limiter over MPI (yet)"
);
if
(
with_black_holes
)
error
(
"Can't run with black holes over MPI (yet)"
);
#endif
/* Temporary early aborts for modes not supported with hand-vec. */
...
...
@@ -799,16 +798,18 @@ int main(int argc, char *argv[]) {
#if defined(HAVE_HDF5)
#if defined(WITH_MPI)
#if defined(HAVE_PARALLEL_HDF5)
read_ic_parallel
(
ICfileName
,
&
us
,
dim
,
&
parts
,
&
gparts
,
&
sparts
,
&
Ngas
,
&
Ngpart
,
&
Nspart
,
&
flag_entropy_ICs
,
with_hydro
,
read_ic_parallel
(
ICfileName
,
&
us
,
dim
,
&
parts
,
&
gparts
,
&
sparts
,
&
bparts
,
&
Ngas
,
&
Ngpart
,
&
Nspart
,
&
Nbpart
,
&
flag_entropy_ICs
,
with_hydro
,
(
with_external_gravity
||
with_self_gravity
),
with_stars
,
with_black_holes
,
cleanup_h
,
cleanup_sqrt_a
,
cosmo
.
h
,
cosmo
.
a
,
myrank
,
nr_nodes
,
MPI_COMM_WORLD
,
MPI_INFO_NULL
,
nr_threads
,
dry_run
);
#else
read_ic_serial
(
ICfileName
,
&
us
,
dim
,
&
parts
,
&
gparts
,
&
sparts
,
&
Ngas
,
&
Ngpart
,
&
Nspart
,
&
flag_entropy_ICs
,
with_hydro
,
read_ic_serial
(
ICfileName
,
&
us
,
dim
,
&
parts
,
&
gparts
,
&
sparts
,
&
bparts
,
&
Ngas
,
&
Ngpart
,
&
Nspart
,
&
Nbpart
,
&
flag_entropy_ICs
,
with_hydro
,
(
with_external_gravity
||
with_self_gravity
),
with_stars
,
with_black_holes
,
cleanup_h
,
cleanup_sqrt_a
,
cosmo
.
h
,
cosmo
.
a
,
myrank
,
nr_nodes
,
MPI_COMM_WORLD
,
MPI_INFO_NULL
,
nr_threads
,
dry_run
);
...
...
@@ -834,6 +835,10 @@ int main(int argc, char *argv[]) {
for
(
size_t
k
=
0
;
k
<
Ngpart
;
++
k
)
if
(
gparts
[
k
].
type
==
swift_type_stars
)
error
(
"Linking problem"
);
}
if
(
!
with_black_holes
&&
!
dry_run
)
{
for
(
size_t
k
=
0
;
k
<
Ngpart
;
++
k
)
if
(
gparts
[
k
].
type
==
swift_type_black_hole
)
error
(
"Linking problem"
);
}
if
(
!
with_hydro
&&
!
dry_run
)
{
for
(
size_t
k
=
0
;
k
<
Ngpart
;
++
k
)
if
(
gparts
[
k
].
type
==
swift_type_gas
)
error
(
"Linking problem"
);
...
...
@@ -1058,9 +1063,9 @@ int main(int argc, char *argv[]) {
/* Legend */
if
(
myrank
==
0
)
{
printf
(
"# %6s %14s %12s %12s %14s %9s %12s %12s %12s %16s [%s] %6s
\n
"
,
printf
(
"# %6s %14s %12s %12s %14s %9s %12s %12s %12s
%12s
%16s [%s] %6s
\n
"
,
"Step"
,
"Time"
,
"Scale-factor"
,
"Redshift"
,
"Time-step"
,
"Time-bins"
,
"Updates"
,
"g-Updates"
,
"s-Updates"
,
"Wall-clock time"
,
"Updates"
,
"g-Updates"
,
"s-Updates"
,
"b-Updates"
,
"Wall-clock time"
,
clocks_getunit
(),
"Props"
);
fflush
(
stdout
);
}
...
...
src/cell.c
View file @
43682ecb
...
...
@@ -188,6 +188,39 @@ int cell_link_sparts(struct cell *c, struct spart *sparts) {
return
c
->
stars
.
count
;
}
/**
* @brief Link the cells recursively to the given #bpart array.
*
* @param c The #cell.
* @param bparts The #bpart array.
*
* @return The number of particles linked.
*/
int
cell_link_bparts
(
struct
cell
*
c
,
struct
bpart
*
bparts
)
{
#ifdef SWIFT_DEBUG_CHECKS
if
(
c
->
nodeID
==
engine_rank
)
error
(
"Linking foreign particles in a local cell!"
);
if
(
c
->
black_holes
.
parts
!=
NULL
)
error
(
"Linking bparts into a cell that was already linked"
);
#endif
c
->
black_holes
.
parts
=
bparts
;
/* Fill the progeny recursively, depth-first. */
if
(
c
->
split
)
{
int
offset
=
0
;
for
(
int
k
=
0
;
k
<
8
;
k
++
)
{
if
(
c
->
progeny
[
k
]
!=
NULL
)
offset
+=
cell_link_bparts
(
c
->
progeny
[
k
],
&
bparts
[
offset
]);
}
}
/* Return the total number of linked particles. */
return
c
->
black_holes
.
count
;
}
/**
* @brief Recurse down foreign cells until reaching one with hydro
* tasks; then trigger the linking of the #part array from that
...
...
src/cell.h
View file @
43682ecb
...
...
@@ -824,6 +824,7 @@ int cell_getsize(struct cell *c);
int
cell_link_parts
(
struct
cell
*
c
,
struct
part
*
parts
);
int
cell_link_gparts
(
struct
cell
*
c
,
struct
gpart
*
gparts
);
int
cell_link_sparts
(
struct
cell
*
c
,
struct
spart
*
sparts
);
int
cell_link_bparts
(
struct
cell
*
c
,
struct
bpart
*
bparts
);
int
cell_link_foreign_parts
(
struct
cell
*
c
,
struct
part
*
parts
);
int
cell_link_foreign_gparts
(
struct
cell
*
c
,
struct
gpart
*
gparts
);
int
cell_count_parts_for_tasks
(
const
struct
cell
*
c
);
...
...
src/collectgroup.c
View file @
43682ecb
...
...
@@ -36,17 +36,20 @@
/* Local collections for MPI reduces. */
struct
mpicollectgroup1
{
long
long
updated
,
g_updated
,
s_updated
;
long
long
inhibited
,
g_inhibited
,
s_inhibited
;
long
long
updated
,
g_updated
,
s_updated
,
b_updated
;
long
long
inhibited
,
g_inhibited
,
s_inhibited
,
b_inhibited
;
integertime_t
ti_hydro_end_min
;
integertime_t
ti_gravity_end_min
;
integertime_t
ti_stars_end_min
;
integertime_t
ti_black_holes_end_min
;
integertime_t
ti_hydro_end_max
;
integertime_t
ti_gravity_end_max
;
integertime_t
ti_stars_end_max
;
integertime_t
ti_black_holes_end_max
;
integertime_t
ti_hydro_beg_max
;
integertime_t
ti_gravity_beg_max
;
integertime_t
ti_stars_beg_max
;
integertime_t
ti_black_holes_beg_max
;
int
forcerebuild
;
long
long
total_nr_cells
;
long
long
total_nr_tasks
;
...
...
@@ -96,18 +99,23 @@ void collectgroup1_apply(struct collectgroup1 *grp1, struct engine *e) {
e
->
ti_stars_end_min
=
grp1
->
ti_stars_end_min
;
e
->
ti_stars_end_max
=
grp1
->
ti_stars_end_max
;
e
->
ti_stars_beg_max
=
grp1
->
ti_stars_beg_max
;
e
->
ti_black_holes_end_min
=
grp1
->
ti_black_holes_end_min
;
e
->
ti_black_holes_end_max
=
grp1
->
ti_black_holes_end_max
;
e
->
ti_black_holes_beg_max
=
grp1
->
ti_black_holes_beg_max
;
e
->
ti_end_min
=
min
3
(
e
->
ti_hydro_end_min
,
e
->
ti_gravity_end_min
,
e
->
ti_stars_end_min
);
min
4
(
e
->
ti_hydro_end_min
,
e
->
ti_gravity_end_min
,
e
->
ti_stars_end_min
,
e
->
ti_black_holes_end_min
);
e
->
ti_end_max
=
max
3
(
e
->
ti_hydro_end_max
,
e
->
ti_gravity_end_max
,
e
->
ti_stars_end_max
);
max
4
(
e
->
ti_hydro_end_max
,
e
->
ti_gravity_end_max
,
e
->
ti_stars_end_max
,
e
->
ti_black_holes_end_max
);
e
->
ti_beg_max
=
max
3
(
e
->
ti_hydro_beg_max
,
e
->
ti_gravity_beg_max
,
e
->
ti_stars_beg_max
);
max
4
(
e
->
ti_hydro_beg_max
,
e
->
ti_gravity_beg_max
,
e
->
ti_stars_beg_max
,
e
->
ti_black_holes_beg_max
);
e
->
updates
=
grp1
->
updated
;
e
->
g_updates
=
grp1
->
g_updated
;
e
->
s_updates
=
grp1
->
s_updated
;
e
->
b_updates
=
grp1
->
b_updated
;
e
->
nr_inhibited_parts
=
grp1
->
inhibited
;
e
->
nr_inhibited_gparts
=
grp1
->
g_inhibited
;
e
->
nr_inhibited_sparts
=
grp1
->
s_inhibited
;
e
->
nr_inhibited_bparts
=
grp1
->
b_inhibited
;
e
->
forcerebuild
=
grp1
->
forcerebuild
;
e
->
total_nr_cells
=
grp1
->
total_nr_cells
;
e
->
total_nr_tasks
=
grp1
->
total_nr_tasks
;
...
...
@@ -153,20 +161,24 @@ void collectgroup1_apply(struct collectgroup1 *grp1, struct engine *e) {
*/
void
collectgroup1_init
(
struct
collectgroup1
*
grp1
,
size_t
updated
,
size_t
g_updated
,
size_t
s_updated
,
size_t
inhibited
,
size_t
g_inhibited
,
size_t
s_inhibited
,
size_t
s_updated
,
size_t
b_updated
,
size_t
inhibited
,
size_t
g_inhibited
,
size_t
s_inhibited
,
size_t
b_inhibited
,
integertime_t
ti_hydro_end_min
,
integertime_t
ti_hydro_end_max
,
integertime_t
ti_hydro_beg_max
,
integertime_t
ti_gravity_end_min
,
integertime_t
ti_gravity_end_max
,
integertime_t
ti_gravity_beg_max
,
integertime_t
ti_stars_end_min
,
integertime_t
ti_stars_end_max
,
integertime_t
ti_stars_beg_max
,
int
forcerebuild
,
long
long
total_nr_cells
,
integertime_t
ti_stars_beg_max
,
integertime_t
ti_black_holes_end_min
,
integertime_t
ti_black_holes_end_max
,
integertime_t
ti_black_holes_beg_max
,
int
forcerebuild
,
long
long
total_nr_cells
,
long
long
total_nr_tasks
,
float
tasks_per_cell
)
{
grp1
->
updated
=
updated
;
grp1
->
g_updated
=
g_updated
;
grp1
->
s_updated
=
s_updated
;
grp1
->
b_updated
=
b_updated
;
grp1
->
inhibited
=
inhibited
;
grp1
->
g_inhibited
=
g_inhibited
;
grp1
->
s_inhibited
=
s_inhibited
;
grp1
->
b_inhibited
=
b_inhibited
;
grp1
->
ti_hydro_end_min
=
ti_hydro_end_min
;
grp1
->
ti_hydro_end_max
=
ti_hydro_end_max
;
grp1
->
ti_hydro_beg_max
=
ti_hydro_beg_max
;
...
...
@@ -176,6 +188,9 @@ void collectgroup1_init(
grp1
->
ti_stars_end_min
=
ti_stars_end_min
;
grp1
->
ti_stars_end_max
=
ti_stars_end_max
;
grp1
->
ti_stars_beg_max
=
ti_stars_beg_max
;
grp1
->
ti_black_holes_end_min
=
ti_black_holes_end_min
;
grp1
->
ti_black_holes_end_max
=
ti_black_holes_end_max
;
grp1
->
ti_black_holes_beg_max
=
ti_black_holes_beg_max
;
grp1
->
forcerebuild
=
forcerebuild
;
grp1
->
total_nr_cells
=
total_nr_cells
;
grp1
->
total_nr_tasks
=
total_nr_tasks
;
...
...
@@ -199,18 +214,23 @@ void collectgroup1_reduce(struct collectgroup1 *grp1) {
mpigrp11
.
updated
=
grp1
->
updated
;
mpigrp11
.
g_updated
=
grp1
->
g_updated
;
mpigrp11
.
s_updated
=
grp1
->
s_updated
;
mpigrp11
.
b_updated
=
grp1
->
b_updated
;
mpigrp11
.
inhibited
=
grp1
->
inhibited
;
mpigrp11
.
g_inhibited
=
grp1
->
g_inhibited
;
mpigrp11
.
s_inhibited
=
grp1
->
s_inhibited
;
mpigrp11
.
b_inhibited
=
grp1
->
b_inhibited
;
mpigrp11
.
ti_hydro_end_min
=
grp1
->
ti_hydro_end_min
;
mpigrp11
.
ti_gravity_end_min
=
grp1
->
ti_gravity_end_min
;
mpigrp11
.
ti_stars_end_min
=
grp1
->
ti_stars_end_min
;
mpigrp11
.
ti_black_holes_end_min
=
grp1
->
ti_black_holes_end_min
;
mpigrp11
.
ti_hydro_end_max
=
grp1
->
ti_hydro_end_max
;
mpigrp11
.
ti_gravity_end_max
=
grp1
->
ti_gravity_end_max
;
mpigrp11
.
ti_stars_end_max
=
grp1
->
ti_stars_end_max
;
mpigrp11
.
ti_black_holes_end_max
=
grp1
->
ti_black_holes_end_max
;
mpigrp11
.
ti_hydro_beg_max
=
grp1
->
ti_hydro_beg_max
;
mpigrp11
.
ti_gravity_beg_max
=
grp1
->
ti_gravity_beg_max
;
mpigrp11
.
ti_stars_beg_max
=
grp1
->
ti_stars_beg_max
;
mpigrp11
.
ti_black_holes_beg_max
=
grp1
->
ti_black_holes_beg_max
;
mpigrp11
.
forcerebuild
=
grp1
->
forcerebuild
;
mpigrp11
.
total_nr_cells
=
grp1
->
total_nr_cells
;
mpigrp11
.
total_nr_tasks
=
grp1
->
total_nr_tasks
;
...
...
@@ -225,18 +245,23 @@ void collectgroup1_reduce(struct collectgroup1 *grp1) {
grp1
->
updated
=
mpigrp12
.
updated
;
grp1
->
g_updated
=
mpigrp12
.
g_updated
;
grp1
->
s_updated
=
mpigrp12
.
s_updated
;
grp1
->
b_updated
=
mpigrp12
.
b_updated
;
grp1
->
inhibited
=
mpigrp12
.
inhibited
;
grp1
->
g_inhibited
=
mpigrp12
.
g_inhibited
;
grp1
->
s_inhibited
=
mpigrp12
.
s_inhibited
;
grp1
->
b_inhibited
=
mpigrp12
.
b_inhibited
;
grp1
->
ti_hydro_end_min
=
mpigrp12
.
ti_hydro_end_min
;
grp1
->
ti_gravity_end_min
=
mpigrp12
.
ti_gravity_end_min
;
grp1
->
ti_stars_end_min
=
mpigrp12
.
ti_stars_end_min
;
grp1
->
ti_black_holes_end_min
=
mpigrp12
.
ti_black_holes_end_min
;
grp1
->
ti_hydro_end_max
=
mpigrp12
.
ti_hydro_end_max
;
grp1
->
ti_gravity_end_max
=
mpigrp12
.
ti_gravity_end_max
;
grp1
->
ti_stars_end_max
=
mpigrp12
.
ti_stars_end_max
;
grp1
->
ti_black_holes_end_max
=
mpigrp12
.
ti_black_holes_end_max
;
grp1
->
ti_hydro_beg_max
=
mpigrp12
.
ti_hydro_beg_max
;
grp1
->
ti_gravity_beg_max
=
mpigrp12
.
ti_gravity_beg_max
;
grp1
->
ti_stars_beg_max
=
mpigrp12
.
ti_stars_beg_max
;
grp1
->
ti_black_holes_beg_max
=
mpigrp12
.
ti_black_holes_beg_max
;
grp1
->
forcerebuild
=
mpigrp12
.
forcerebuild
;
grp1
->
total_nr_cells
=
mpigrp12
.
total_nr_cells
;
grp1
->
total_nr_tasks
=
mpigrp12
.
total_nr_tasks
;
...
...
@@ -260,11 +285,13 @@ static void doreduce1(struct mpicollectgroup1 *mpigrp11,
mpigrp11
->
updated
+=
mpigrp12
->
updated
;
mpigrp11
->
g_updated
+=
mpigrp12
->
g_updated
;
mpigrp11
->
s_updated
+=
mpigrp12
->
s_updated
;
mpigrp11
->
b_updated
+=
mpigrp12
->
b_updated
;
/* Sum of inhibited */
mpigrp11
->
inhibited
+=
mpigrp12
->
inhibited
;
mpigrp11
->
g_inhibited
+=
mpigrp12
->
g_inhibited
;
mpigrp11
->
s_inhibited
+=
mpigrp12
->
s_inhibited
;
mpigrp11
->
b_inhibited
+=
mpigrp12
->
b_inhibited
;
/* Minimum end time. */
mpigrp11
->
ti_hydro_end_min
=
...
...
@@ -273,6 +300,8 @@ static void doreduce1(struct mpicollectgroup1 *mpigrp11,
min
(
mpigrp11
->
ti_gravity_end_min
,
mpigrp12
->
ti_gravity_end_min
);
mpigrp11
->
ti_stars_end_min
=
min
(
mpigrp11
->
ti_stars_end_min
,
mpigrp12
->
ti_stars_end_min
);
mpigrp11
->
ti_black_holes_end_min
=
min
(
mpigrp11
->
ti_black_holes_end_min
,
mpigrp12
->
ti_black_holes_end_min
);
/* Maximum end time. */
mpigrp11
->
ti_hydro_end_max
=
...
...
@@ -281,6 +310,8 @@ static void doreduce1(struct mpicollectgroup1 *mpigrp11,
max
(
mpigrp11
->
ti_gravity_end_max
,
mpigrp12
->
ti_gravity_end_max
);
mpigrp11
->
ti_stars_end_max
=
max
(
mpigrp11
->
ti_stars_end_max
,
mpigrp12
->
ti_stars_end_max
);
mpigrp11
->
ti_black_holes_end_max
=
max
(
mpigrp11
->
ti_black_holes_end_max
,
mpigrp12
->
ti_black_holes_end_max
);
/* Maximum beg time. */
mpigrp11
->
ti_hydro_beg_max
=
...
...
@@ -289,6 +320,8 @@ static void doreduce1(struct mpicollectgroup1 *mpigrp11,
max
(
mpigrp11
->
ti_gravity_beg_max
,
mpigrp12
->
ti_gravity_beg_max
);
mpigrp11
->
ti_stars_beg_max
=
max
(
mpigrp11
->
ti_stars_beg_max
,
mpigrp12
->
ti_stars_beg_max
);
mpigrp11
->
ti_black_holes_beg_max
=
max
(
mpigrp11
->
ti_black_holes_beg_max
,
mpigrp12
->
ti_black_holes_beg_max
);
/* Everyone must agree to not rebuild. */
if
(
mpigrp11
->
forcerebuild
||
mpigrp12
->
forcerebuild
)
...
...
src/collectgroup.h
View file @
43682ecb
...
...
@@ -35,15 +35,16 @@ struct engine;
struct
collectgroup1
{
/* Number of particles updated */
long
long
updated
,
g_updated
,
s_updated
;
long
long
updated
,
g_updated
,
s_updated
,
b_updated
;
/* Number of particles inhibited */
long
long
inhibited
,
g_inhibited
,
s_inhibited
;
long
long
inhibited
,
g_inhibited
,
s_inhibited
,
b_inhibited
;
/* Times for the time-step */
integertime_t
ti_hydro_end_min
,
ti_hydro_end_max
,
ti_hydro_beg_max
;
integertime_t
ti_gravity_end_min
,
ti_gravity_end_max
,
ti_gravity_beg_max
;
integertime_t
ti_stars_end_min
,
ti_stars_end_max
,
ti_stars_beg_max
;
integertime_t
ti_black_holes_end_min
,
ti_black_holes_end_max
,
ti_black_holes_beg_max
;
/* Force the engine to rebuild? */
int
forcerebuild
;
...
...
@@ -59,14 +60,16 @@ struct collectgroup1 {
void
collectgroup_init
(
void
);
void
collectgroup1_apply
(
struct
collectgroup1
*
grp1
,
struct
engine
*
e
);
void
collectgroup1_init
(
struct
collectgroup1
*
grp1
,
size_t
updated
,
size_t
g_updated
,
size_t
s_updated
,
size_t
inhibited
,
size_t
g_inhibited
,
size_t
s_inhibited
,
struct
collectgroup1
*
grp1
,
size_t
updated
,
size_t
g_updated
,
size_t
s_updated
,
size_t
b_updated
,
size_t
inhibited
,
size_t
g_inhibited
,
size_t
s_inhibited
,
size_t
b_inhibited
,
integertime_t
ti_hydro_end_min
,
integertime_t
ti_hydro_end_max
,
integertime_t
ti_hydro_beg_max
,
integertime_t
ti_gravity_end_min
,
integertime_t
ti_gravity_end_max
,
integertime_t
ti_gravity_beg_max
,
integertime_t
ti_stars_end_min
,
integertime_t
ti_stars_end_max
,
integertime_t
ti_stars_beg_max
,
int
forcerebuild
,
long
long
total_nr_cells
,
long
long
total_nr_tasks
,
float
tasks_per_cell
);
integertime_t
ti_stars_beg_max
,
integertime_t
ti_black_holes_end_min
,
integertime_t
ti_black_holes_end_max
,
integertime_t
ti_black_holes_beg_max
,
int
forcerebuild
,
long
long
total_nr_cells
,
long
long
total_nr_tasks
,
float
tasks_per_cell
);
void
collectgroup1_reduce
(
struct
collectgroup1
*
grp1
);
#endif
/* SWIFT_COLLECTGROUP_H */
src/engine.c
View file @
43682ecb
This diff is collapsed.
Click to expand it.
src/engine.h
View file @
43682ecb
...
...
@@ -204,6 +204,15 @@ struct engine {
/* Maximal stars ti_beg for the next time-step */
integertime_t
ti_stars_beg_max
;
/* Minimal black holes ti_end for the next time-step */
integertime_t
ti_black_holes_end_min
;
/* Maximal black holes ti_end for the next time-step */
integertime_t
ti_black_holes_end_max
;
/* Maximal black holes ti_beg for the next time-step */
integertime_t
ti_black_holes_beg_max
;
/* Minimal overall ti_end for the next time-step */
integertime_t
ti_end_min
;
...
...
@@ -473,7 +482,8 @@ void engine_exchange_strays(struct engine *e, const size_t offset_parts,
const
int
*
ind_part
,
size_t
*
Npart
,
const
size_t
offset_gparts
,
const
int
*
ind_gpart
,
size_t
*
Ngpart
,
const
size_t
offset_sparts
,
const
int
*
ind_spart
,
size_t
*
Nspart
);
const
int
*
ind_spart
,
size_t
*
Nspart
,
const
size_t
offset_bparts
,
const
int
*
ind_bpart
,
size_t
*
Nbpart
);
void
engine_rebuild
(
struct
engine
*
e
,
int
redistributed
,
int
clean_h_values
);
void
engine_repartition
(
struct
engine
*
e
);
void
engine_repartition_trigger
(
struct
engine
*
e
);
...
...
src/parallel_io.c
View file @
43682ecb
...
...
@@ -37,6 +37,7 @@
#include
"parallel_io.h"
/* Local includes. */
#include
"black_holes_io.h"
#include
"chemistry_io.h"
#include
"common_io.h"
#include
"cooling_io.h"
...
...
@@ -651,14 +652,17 @@ void writeArray(struct engine* e, hid_t grp, char* fileName,
* @param parts (output) The array of #part read from the file.
* @param gparts (output) The array of #gpart read from the file.
* @param sparts (output) The array of #spart read from the file.
* @param bparts (output) The array of #bpart read from the file.
* @param Ngas (output) The number of particles read from the file.
* @param Ngparts (output) The number of particles read from the file.
* @param Nstars (output) The number of particles read from the file.
* @param Nblackholes (output) The number of particles read from the file.
* @param flag_entropy (output) 1 if the ICs contained Entropy in the
* InternalEnergy field
* @param with_hydro Are we running with hydro ?
* @param with_gravity Are we running with gravity ?
* @param with_stars Are we running with stars ?
* @param with_black_holes Are we running with black holes ?
* @param cleanup_h Are we cleaning-up h-factors from the quantities we read?
* @param cleanup_sqrt_a Are we cleaning-up the sqrt(a) factors in the Gadget
* IC velocities?
...
...
@@ -674,9 +678,9 @@ void writeArray(struct engine* e, hid_t grp, char* fileName,
*/
void
read_ic_parallel
(
char
*
fileName
,
const
struct
unit_system
*
internal_units
,
double
dim
[
3
],
struct
part
**
parts
,
struct
gpart
**
gparts
,
struct
spart
**
sparts
,
size_t
*
Ngas
,
size_t
*
Ngparts
,
size_t
*
Nstars
,
int
*
flag_entropy
,
int
with_hydro
,
int
with_gravity
,
int
with_stars
,
int
cleanup_h
,
struct
spart
**
sparts
,
struct
bpart
**
bparts
,
size_t
*
Ngas
,
size_t
*
Ngparts
,
size_t
*
Nstars
,
size_t
*
Nblackholes
,
int
*
flag_entropy
,
int
with_hydro
,
int
with_gravity
,
int
with_stars
,
int
with_black_holes
,
int
cleanup_h
,
int
cleanup_sqrt_a
,
double
h
,
double
a
,
int
mpi_rank
,
int
mpi_size
,
MPI_Comm
comm
,
MPI_Info
info
,
int
n_threads
,
int
dry_run
)
{
...
...
@@ -692,6 +696,9 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units,
int
dimension
=
3
;
/* Assume 3D if nothing is specified */
size_t
Ndm
=
0
;
/* Initialise counters */
*
Ngas
=
0
,
*
Ngparts
=
0
,
*
Nstars
=
0
,
*
Nblackholes
=
0
;
/* Open file */
/* message("Opening file '%s' as IC.", fileName); */
hid_t
h_plist_id
=
H5Pcreate
(
H5P_FILE_ACCESS
);
...
...
@@ -831,12 +838,22 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units,
bzero
(
*
sparts
,
*
Nstars
*
sizeof
(
struct
spart
));
}
/* Allocate memory to store black hole particles */
if
(
with_black_holes
)
{
*
Nblackholes
=
N
[
swift_type_black_hole
];
if
(
swift_memalign
(
"sparts"
,
(
void
**
)
bparts
,
bpart_align
,
*
Nblackholes
*
sizeof
(
struct
bpart
))
!=
0
)
error
(
"Error while allocating memory for black_holes particles"
);
bzero
(
*
bparts
,
*
Nblackholes
*
sizeof
(
struct
bpart
));
}
/* Allocate memory to store gravity particles */
if
(
with_gravity
)
{
Ndm
=
N
[
1
];
*
Ngparts
=
(
with_hydro
?
N
[
swift_type_gas
]
:
0
)
+
N
[
swift_type_dark_matter
]
+
(
with_stars
?
N
[
swift_type_stars
]
:
0
);
(
with_stars
?
N
[
swift_type_stars
]
:
0
)
+
(
with_black_holes
?
N
[
swift_type_black_hole
]
:
0
);
if
(
swift_memalign
(
"gparts"
,
(
void
**
)
gparts
,
gpart_align
,
*
Ngparts
*
sizeof
(
struct
gpart
))
!=
0
)
error
(
"Error while allocating memory for gravity particles"
);
...
...
@@ -892,6 +909,13 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units,
}
break
;
case
swift_type_black_hole
:
if
(
with_black_holes
)
{
Nparticles
=
*
Nblackholes
;
black_holes_read_particles
(
*
bparts
,
list
,
&
num_fields
);
}
break
;
default:
if
(
mpi_rank
==
0
)
message
(
"Particle Type %d not yet supported. Particles ignored"
,
...
...
@@ -925,6 +949,10 @@ void read_ic_parallel(char* fileName, const struct unit_system* internal_units,
if
(
with_stars
)
io_duplicate_stars_gparts
(
&
tp
,
*
sparts
,
*
gparts
,
*
Nstars
,
Ndm
+
*
Ngas
);
/* Duplicate the stars particles into gparts */
if
(
with_black_holes
)
io_duplicate_black_holes_gparts
(
&
tp
,
*
bparts
,
*
gparts
,
*
Nblackholes
,
Ndm
+
*
Ngas
+
*
Nstars
);
threadpool_clean
(
&
tp
);
}
...
...
@@ -957,6 +985,7 @@ void prepare_file(struct engine* e, const char* baseName, long long N_total[6],
const
struct
xpart
*
xparts
=
e
->
s
->
xparts
;
const
struct
gpart
*
gparts
=
e
->
s
->
gparts
;
const
struct
spart
*
sparts
=
e
->
s
->
sparts
;
const
struct
bpart
*
bparts
=
e
->
s
->
bparts
;
struct
swift_params
*
params
=
e
->
parameter_file
;
const
int
with_cosmology
=
e
->
policy
&
engine_policy_cosmology
;
const
int
with_cooling
=
e
->
policy
&
engine_policy_cooling
;
...
...
@@ -1182,6 +1211,13 @@ void prepare_file(struct engine* e, const char* baseName, long long N_total[6],
}
break
;
case
swift_type_black_hole
:
black_holes_write_particles
(
bparts
,
list
,
&
num_fields
);
if
(
with_stf
)
{
num_fields
+=
velociraptor_write_bparts
(
bparts
,
list
+
num_fields
);
}
break
;
default:
error
(
"Particle Type %d not yet supported. Aborting"
,
ptype
);
}
...
...
@@ -1245,6 +1281,7 @@ void write_output_parallel(struct engine* e, const char* baseName,
const
struct
xpart
*
xparts
=
e
->
s
->
xparts
;
const
struct
gpart
*
gparts
=
e
->
s
->
gparts
;
const
struct
spart
*
sparts
=
e
->
s
->
sparts
;
const
struct
bpart
*
bparts
=
e
->
s
->
bparts
;
struct
swift_params
*
params
=
e
->
parameter_file
;
const
int
with_cosmology
=
e
->
policy
&
engine_policy_cosmology
;
const
int
with_cooling
=
e
->
policy
&
engine_policy_cooling
;
...
...
@@ -1260,6 +1297,7 @@ void write_output_parallel(struct engine* e, const char* baseName,
const
size_t
Ntot
=
e
->
s
->
nr_gparts
;
const
size_t
Ngas
=
e
->
s
->
nr_parts
;
const
size_t
Nstars
=
e
->
s
->
nr_sparts
;
const
size_t
Nblackholes
=
e
->
s
->
nr_bparts
;
// const size_t Nbaryons = Ngas + Nstars;
// const size_t Ndm = Ntot > 0 ? Ntot - Nbaryons : 0;
...
...
@@ -1270,6 +1308,8 @@ void write_output_parallel(struct engine* e, const char* baseName,
e
->
s
->
nr_parts
-
e
->
s
->
nr_inhibited_parts
-
e
->
s
->
nr_extra_parts
;
const
size_t
Nstars_written
=
e
->
s
->
nr_sparts
-
e
->
s
->
nr_inhibited_sparts
-
e
->
s
->
nr_extra_sparts
;
const
size_t
Nblackholes_written
=
e
->
s
->
nr_bparts
-
e
->
s
->
nr_inhibited_bparts
-
e
->
s
->
nr_extra_bparts
;
const
size_t
Nbaryons_written
=
Ngas_written
+
Nstars_written
;
const
size_t
Ndm_written
=
Ntot_written
>
0
?
Ntot_written
-
Nbaryons_written
:
0
;
...
...
@@ -1458,6 +1498,7 @@ void write_output_parallel(struct engine* e, const char* baseName,
struct
gpart
*
gparts_written
=
NULL
;
struct
velociraptor_gpart_data
*
gpart_group_data_written
=
NULL
;
struct
spart
*
sparts_written
=
NULL
;
struct
bpart
*
bparts_written
=
NULL
;
/* Write particle fields from the particle structure */
switch
(
ptype
)
{
...
...
@@ -1608,6 +1649,39 @@ void write_output_parallel(struct engine* e, const char* baseName,
}
}
break
;
case
swift_type_black_hole
:
{
if
(
Nblackholes
==
Nblackholes_written
)
{
/* No inhibted particles: easy case */
Nparticles
=
Nblackholes
;
black_holes_write_particles
(
bparts
,
list
,
&
num_fields
);
if
(
with_stf
)
{
num_fields
+=
velociraptor_write_bparts
(
bparts
,
list
+
num_fields
);
}
}
else
{
/* Ok, we need to fish out the particles we want */
Nparticles
=
Nblackholes_written
;
/* Allocate temporary arrays */
if
(
swift_memalign
(
"bparts_written"
,
(
void
**
)
&
bparts_written
,
bpart_align
,
Nblackholes_written
*
sizeof
(
struct
bpart
))
!=
0
)
error
(
"Error while allocating temporart memory for bparts"
);
/* Collect the particles we want to write */
io_collect_bparts_to_write
(
bparts
,
bparts_written
,
Nblackholes
,
Nblackholes_written
);
/* Select the fields to write */
black_holes_write_particles
(
bparts_written
,
list
,
&
num_fields
);
if
(
with_stf
)
{
num_fields
+=
velociraptor_write_bparts
(
bparts_written
,
list
+
num_fields
);
}
}
}
break
;
default:
error
(
"Particle Type %d not yet supported. Aborting"
,
ptype
);
}
...
...
@@ -1634,6 +1708,7 @@ void write_output_parallel(struct engine* e, const char* baseName,
if
(
gpart_group_data_written
)
swift_free
(
"gpart_group_written"
,
gpart_group_data_written
);
if
(
sparts_written
)
swift_free
(
"sparts_written"
,
sparts_written
);
if
(
bparts_written
)
swift_free
(
"bparts_written"
,
bparts_written
);
#ifdef IO_SPEED_MEASUREMENT
MPI_Barrier
(
MPI_COMM_WORLD
);
...
...
src/parallel_io.h
View file @
43682ecb
...
...
@@ -35,9 +35,10 @@
void
read_ic_parallel
(
char
*
fileName
,
const
struct
unit_system
*
internal_units
,
double
dim
[
3
],
struct
part
**
parts
,
struct
gpart
**
gparts
,
struct
spart
**
sparts
,
size_t
*
Ngas
,
size_t
*
Ngparts
,
size_t
*
Nsparts
,
int
*
flag_entropy
,
int
with_hydro
,
int
with_gravity
,
int
with_stars
,
int
cleanup_h
,
struct
spart
**
sparts
,
struct
bpart
**
bparts
,
size_t
*
Ngas
,
size_t
*
Ngparts
,
size_t
*
Nsparts
,
size_t
*
Nbparts
,
int
*
flag_entropy
,
int
with_hydro
,
int
with_gravity
,
int
with_stars
,
int
with_black_holes
,
int
cleanup_h
,
int
cleanup_sqrt_a
,
double
h
,
double
a
,
int
mpi_rank
,
int
mpi_size
,
MPI_Comm
comm
,
MPI_Info
info
,
int
nr_threads
,
int
dry_run
);
...
...
@@ -47,13 +48,6 @@ void write_output_parallel(struct engine* e, const char* baseName,
const
struct
unit_system
*
snapshot_units
,
int
mpi_rank
,
int
mpi_size
,
MPI_Comm
comm
,
MPI_Info
info
);
void
writeArray
(
struct
engine
*
e
,
hid_t
grp
,
char
*
fileName
,
char
*
partTypeGroupName
,
struct
io_props
props
,
size_t
N
,
long
long
N_total
,
int
mpi_rank
,
long
long
offset
,
const
struct
unit_system
*
internal_units
,
const
struct
unit_system
*
snapshot_units
);
#endif
#endif
/* SWIFT_PARALLEL_IO_H */
src/part.c
View file @
43682ecb
...
...
@@ -64,6 +64,22 @@ void part_relink_gparts_to_sparts(struct spart *sparts, size_t N,
}
}
/**
* @brief Re-link the #gpart%s associated with the list of #bpart%s.
*
* @param bparts The list of #bpart.
* @param N The number of s-particles to re-link;
* @param offset The offset of #bpart%s relative to the global bparts list.
*/
void
part_relink_gparts_to_bparts
(
struct
bpart
*
bparts
,
size_t
N
,
ptrdiff_t
offset
)
{
for
(
size_t
k
=
0
;
k
<
N
;
k
++
)
{
if
(
bparts
[
k
].
gpart
)
{
bparts
[
k
].
gpart
->
id_or_neg_offset
=
-
(
k
+
offset
);
}
}
}
/**
* @brief Re-link the #part%s associated with the list of #gpart%s.
*
...
...
@@ -243,7 +259,7 @@ void part_verify_links(struct part *parts, struct gpart *gparts,
gparts
[
k
].
x
[
2
]
!=
bpart
->
x
[
2
])
error
(
"Linked particles are not at the same position !
\n
"
"gp->x=[%e %e %e]
s
p->x=[%e %e %e] diff=[%e %e %e]"
,
"gp->x=[%e %e %e]
b
p->x=[%e %e %e] diff=[%e %e %e]"
,
gparts
[
k
].
x
[
0
],
gparts
[
k
].
x
[
1
],
gparts
[
k
].
x
[
2
],
bpart
->
x
[
0
],
bpart
->
x
[
1
],
bpart
->
x
[
2
],
gparts
[
k
].
x
[
0
]
-
bpart
->
x
[
0
],
gparts
[
k
].
x
[
1
]
-
bpart
->
x
[
1
],
gparts
[
k
].
x
[
2
]
-
bpart
->
x
[
2
]);
...
...
src/part.h
View file @
43682ecb
...
...
@@ -115,6 +115,8 @@ void part_relink_gparts_to_parts(struct part *parts, size_t N,
ptrdiff_t
offset
);
void
part_relink_gparts_to_sparts
(
struct
spart
*
sparts
,
size_t
N
,
ptrdiff_t
offset
);
void
part_relink_gparts_to_bparts
(
struct
bpart
*
bparts
,
size_t
N
,
ptrdiff_t
offset
);
void
part_relink_parts_to_gparts
(
struct
gpart
*
gparts
,
size_t
N
,
struct
part
*
parts
);
void
part_relink_sparts_to_gparts
(
struct
gpart
*
gparts
,
size_t
N
,
...
...
src/proxy.c
View file @
43682ecb
...
...
@@ -584,7 +584,8 @@ void proxy_parts_exchange_first(struct proxy *p) {
p
->
buff_out
[
0
]
=
p
->
nr_parts_out