Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
SWIFTsim
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Deploy
Releases
Model registry
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
SWIFT
SWIFTsim
Commits
1cf52206
Commit
1cf52206
authored
9 years ago
by
Matthieu Schaller
Browse files
Options
Downloads
Patches
Plain Diff
Fixed the >1G particle i/o issue in serial mode
parent
e351174e
No related branches found
No related tags found
1 merge request
!48
Io fixes
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
src/serial_io.c
+43
-24
43 additions, 24 deletions
src/serial_io.c
src/single_io.c
+9
-8
9 additions, 8 deletions
src/single_io.c
with
52 additions
and
32 deletions
src/serial_io.c
+
43
−
24
View file @
1cf52206
...
...
@@ -87,13 +87,15 @@ void readArrayBackEnd(hid_t grp, char* name, enum DATA_TYPE type, int N,
if
(
importance
==
COMPULSORY
)
{
error
(
"Compulsory data set '%s' not present in the file."
,
name
);
}
else
{
for
(
i
=
0
;
i
<
N
;
++
i
)
memset
(
part_c
+
i
*
partSize
,
0
,
copySize
);
for
(
i
=
0
;
i
<
N
;
++
i
)
memset
(
part_c
+
i
*
partSize
,
0
,
copySize
);
return
;
}
}
/* message( "Reading %s '%s' array...", importance == COMPULSORY ?
* "compulsory": "optional ", name); */
/* message( "Reading %s '%s' array...", importance == COMPULSORY ? */
/* "compulsory": "optional ", name); */
/* fflush(stdout); */
/* Open data space */
h_data
=
H5Dopen1
(
grp
,
name
);
...
...
@@ -117,9 +119,9 @@ void readArrayBackEnd(hid_t grp, char* name, enum DATA_TYPE type, int N,
offsets
[
0
]
=
offset
;
offsets
[
1
]
=
0
;
}
else
{
rank
=
1
;
rank
=
2
;
shape
[
0
]
=
N
;
shape
[
1
]
=
0
;
shape
[
1
]
=
1
;
offsets
[
0
]
=
offset
;
offsets
[
1
]
=
0
;
}
...
...
@@ -131,6 +133,20 @@ void readArrayBackEnd(hid_t grp, char* name, enum DATA_TYPE type, int N,
h_filespace
=
H5Dget_space
(
h_data
);
H5Sselect_hyperslab
(
h_filespace
,
H5S_SELECT_SET
,
offsets
,
NULL
,
shape
,
NULL
);
/* int rank_memspace = H5Sget_simple_extent_ndims(h_memspace); */
/* int rank_filespace = H5Sget_simple_extent_ndims(h_filespace); */
/* message("Memspace rank: %d", rank_memspace); */
/* message("Filespace rank: %d", rank_filespace); */
/* fflush(stdout); */
/* hsize_t dims_memspace[2], max_dims_memspace[2]; */
/* hsize_t dims_filespace[2], max_dims_filespace[2]; */
/* H5Sget_simple_extent_dims(h_memspace, dims_memspace, max_dims_memspace); */
/* H5Sget_simple_extent_dims(h_filespace, dims_filespace, max_dims_filespace);
*/
/* Read HDF5 dataspace in temporary buffer */
/* Dirty version that happens to work for vectors but should be improved */
/* Using HDF5 dataspaces would be better */
...
...
@@ -196,11 +212,11 @@ void read_ic_serial(char* fileName, double dim[3], struct part** parts, int* N,
int
*
periodic
,
int
mpi_rank
,
int
mpi_size
,
MPI_Comm
comm
,
MPI_Info
info
)
{
hid_t
h_file
=
0
,
h_grp
=
0
;
double
boxSize
[
3
]
=
{
0
.
0
,
-
1
.
0
,
-
1
.
0
};
/* GADGET has only cubic boxes (in cosmological mode) */
int
numParticles
[
6
]
=
{
0
};
/* GADGET has 6 particle types. We only keep the type 0*/
int
numParticles_highWord
[
6
]
=
{
0
};
double
boxSize
[
3
]
=
{
0
.
0
,
-
1
.
0
,
-
1
.
0
};
/* GADGET has only cubic boxes (in cosmological mode) */
int
numParticles
[
6
]
=
{
0
};
/* GADGET has 6 particle types. We only keep the type 0*/
int
numParticles_highWord
[
6
]
=
{
0
};
long
long
offset
=
0
;
long
long
N_total
=
0
;
int
rank
;
...
...
@@ -235,14 +251,17 @@ void read_ic_serial(char* fileName, double dim[3], struct part** parts, int* N,
readAttribute
(
h_grp
,
"NumPart_Total"
,
UINT
,
numParticles
);
readAttribute
(
h_grp
,
"NumPart_Total_HighWord"
,
UINT
,
numParticles_highWord
);
N_total
=
((
long
long
)
numParticles
[
0
])
+
((
long
long
)
numParticles_highWord
[
0
]
<<
32
);
N_total
=
((
long
long
)
numParticles
[
0
])
+
((
long
long
)
numParticles_highWord
[
0
]
<<
32
);
dim
[
0
]
=
boxSize
[
0
];
dim
[
1
]
=
(
boxSize
[
1
]
<
0
)
?
boxSize
[
0
]
:
boxSize
[
1
];
dim
[
2
]
=
(
boxSize
[
2
]
<
0
)
?
boxSize
[
0
]
:
boxSize
[
2
];
/* message("Found %d particles in a %speriodic box of size [%f %f %f].", */
/* *N, (periodic ? "": "non-"), dim[0], dim[1], dim[2]); */
/* message("Found %lld particles in a %speriodic box of size [%f %f %f].",
*/
/* N_total, (periodic ? "": "non-"), dim[0], dim[1], dim[2]); */
fflush
(
stdout
);
/* Close header */
H5Gclose
(
h_grp
);
...
...
@@ -264,8 +283,8 @@ void read_ic_serial(char* fileName, double dim[3], struct part** parts, int* N,
if
(
posix_memalign
((
void
*
)
parts
,
part_align
,
(
*
N
)
*
sizeof
(
struct
part
))
!=
0
)
error
(
"Error while allocating memory for particles"
);
bzero
(
*
parts
,
*
N
*
sizeof
(
struct
part
));
/* message("Allocated %8.2f MB for particles.", *N * sizeof(struct part) /
*
(1024.*1024.)); */
/* message("Allocated %8.2f MB for particles.", *N * sizeof(struct part) /
*/
/*
(1024.*1024.)); */
/* Now loop over ranks and read the data */
for
(
rank
=
0
;
rank
<
mpi_size
;
++
rank
)
{
...
...
@@ -499,9 +518,9 @@ void write_output_serial(struct engine* e, struct UnitSystem* us, int mpi_rank,
hid_t
h_file
=
0
,
h_grp
=
0
;
int
N
=
e
->
s
->
nr_parts
;
int
periodic
=
e
->
s
->
periodic
;
int
numParticles
[
6
]
=
{
N
,
0
};
int
numParticlesHighWord
[
6
]
=
{
0
};
unsigned
int
flagEntropy
[
6
]
=
{
0
};
int
numParticles
[
6
]
=
{
N
,
0
};
int
numParticlesHighWord
[
6
]
=
{
0
};
unsigned
int
flagEntropy
[
6
]
=
{
0
};
long
long
N_total
=
0
,
offset
=
0
;
double
offset_d
=
0
.,
N_d
=
0
.,
N_total_d
=
0
.;
int
numFiles
=
1
;
...
...
@@ -516,7 +535,7 @@ void write_output_serial(struct engine* e, struct UnitSystem* us, int mpi_rank,
/* Compute offset in the file and total number of particles */
/* Done using double to allow for up to 2^50=10^15 particles */
N_d
=
(
double
)
N
;
N_d
=
(
double
)
N
;
MPI_Exscan
(
&
N_d
,
&
offset_d
,
1
,
MPI_DOUBLE
,
MPI_SUM
,
comm
);
N_total_d
=
offset_d
+
N_d
;
MPI_Bcast
(
&
N_total_d
,
1
,
MPI_DOUBLE
,
mpi_size
-
1
,
comm
);
...
...
@@ -524,8 +543,8 @@ void write_output_serial(struct engine* e, struct UnitSystem* us, int mpi_rank,
error
(
"Error while computing the offest for parallel output: Simulation has "
"more than 10^15 particles.
\n
"
);
N_total
=
(
long
long
)
N_total_d
;
offset
=
(
long
long
)
offset_d
;
N_total
=
(
long
long
)
N_total_d
;
offset
=
(
long
long
)
offset_d
;
/* Do common stuff first */
if
(
mpi_rank
==
0
)
{
...
...
@@ -568,13 +587,13 @@ void write_output_serial(struct engine* e, struct UnitSystem* us, int mpi_rank,
writeAttribute
(
h_grp
,
"Time"
,
DOUBLE
,
&
dblTime
,
1
);
/* GADGET-2 legacy values */
numParticles
[
0
]
=
(
unsigned
int
)
N_total
;
numParticles
[
0
]
=
(
unsigned
int
)
N_total
;
writeAttribute
(
h_grp
,
"NumPart_ThisFile"
,
UINT
,
numParticles
,
6
);
writeAttribute
(
h_grp
,
"NumPart_Total"
,
UINT
,
numParticles
,
6
);
numParticlesHighWord
[
0
]
=
(
unsigned
int
)(
N_total
>>
32
);
writeAttribute
(
h_grp
,
"NumPart_Total_HighWord"
,
UINT
,
numParticlesHighWord
,
6
);
double
MassTable
[
6
]
=
{
0
.,
0
.,
0
.,
0
.,
0
.,
0
.};
double
MassTable
[
6
]
=
{
0
.,
0
.,
0
.,
0
.,
0
.,
0
.
};
writeAttribute
(
h_grp
,
"MassTable"
,
DOUBLE
,
MassTable
,
6
);
writeAttribute
(
h_grp
,
"Flag_Entropy_ICs"
,
UINT
,
flagEntropy
,
6
);
writeAttribute
(
h_grp
,
"NumFilesPerSnapshot"
,
INT
,
&
numFiles
,
1
);
...
...
This diff is collapsed.
Click to expand it.
src/single_io.c
+
9
−
8
View file @
1cf52206
...
...
@@ -83,7 +83,8 @@ void readArrayBackEnd(hid_t grp, char* name, enum DATA_TYPE type, int N,
/* message("Optional data set '%s' not present. Zeroing this particle
* field...", name); */
for
(
i
=
0
;
i
<
N
;
++
i
)
memset
(
part_c
+
i
*
partSize
,
0
,
copySize
);
for
(
i
=
0
;
i
<
N
;
++
i
)
memset
(
part_c
+
i
*
partSize
,
0
,
copySize
);
return
;
}
...
...
@@ -166,10 +167,10 @@ void readArrayBackEnd(hid_t grp, char* name, enum DATA_TYPE type, int N,
void
read_ic_single
(
char
*
fileName
,
double
dim
[
3
],
struct
part
**
parts
,
int
*
N
,
int
*
periodic
)
{
hid_t
h_file
=
0
,
h_grp
=
0
;
double
boxSize
[
3
]
=
{
0
.
0
,
-
1
.
0
,
-
1
.
0
};
/* GADGET has only cubic boxes (in cosmological mode) */
int
numParticles
[
6
]
=
{
0
};
/* GADGET has 6 particle types. We only keep the type 0*/
double
boxSize
[
3
]
=
{
0
.
0
,
-
1
.
0
,
-
1
.
0
};
/* GADGET has only cubic boxes (in cosmological mode) */
int
numParticles
[
6
]
=
{
0
};
/* GADGET has 6 particle types. We only keep the type 0*/
/* Open file */
/* message("Opening file '%s' as IC.", fileName); */
...
...
@@ -384,8 +385,8 @@ void write_output_single(struct engine* e, struct UnitSystem* us) {
hid_t
h_file
=
0
,
h_grp
=
0
;
int
N
=
e
->
s
->
nr_parts
;
int
periodic
=
e
->
s
->
periodic
;
int
numParticles
[
6
]
=
{
N
,
0
};
int
numParticlesHighWord
[
6
]
=
{
0
};
int
numParticles
[
6
]
=
{
N
,
0
};
int
numParticlesHighWord
[
6
]
=
{
0
};
int
numFiles
=
1
;
struct
part
*
parts
=
e
->
s
->
parts
;
FILE
*
xmfFile
=
0
;
...
...
@@ -437,7 +438,7 @@ void write_output_single(struct engine* e, struct UnitSystem* us) {
writeAttribute
(
h_grp
,
"NumPart_Total"
,
UINT
,
numParticles
,
6
);
writeAttribute
(
h_grp
,
"NumPart_Total_HighWord"
,
UINT
,
numParticlesHighWord
,
6
);
double
MassTable
[
6
]
=
{
0
.,
0
.,
0
.,
0
.,
0
.,
0
.};
double
MassTable
[
6
]
=
{
0
.,
0
.,
0
.,
0
.,
0
.,
0
.
};
writeAttribute
(
h_grp
,
"MassTable"
,
DOUBLE
,
MassTable
,
6
);
writeAttribute
(
h_grp
,
"Flag_Entropy_ICs"
,
UINT
,
numParticlesHighWord
,
6
);
writeAttribute
(
h_grp
,
"NumFilesPerSnapshot"
,
INT
,
&
numFiles
,
1
);
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment