diff --git a/src/common_io.c b/src/common_io.c
index 17c6e17c2cbfa453497225d0a7a8919c1b96c6f2..47e04b868b9d610983b977bd6455c1c78d70bb41 100644
--- a/src/common_io.c
+++ b/src/common_io.c
@@ -409,99 +409,102 @@ void io_write_code_description(hid_t h_file) {
/**
* @brief Mapper function to copy #part or #gpart fields into a buffer.
*/
-void io_copy_mapper(void *restrict temp, int N,
- void *restrict extra_data) {
+void io_copy_mapper(void* restrict temp, int N, void* restrict extra_data) {
- const struct io_props props = *((const struct io_props*) extra_data);
+ const struct io_props props = *((const struct io_props*)extra_data);
const size_t typeSize = io_sizeof_type(props.type);
const size_t copySize = typeSize * props.dimension;
/* How far are we with this chunk? */
- char *restrict temp_c = temp;
+ char* restrict temp_c = temp;
const ptrdiff_t delta = (temp_c - props.start_temp_c) / copySize;
- for(int k = 0; k < N; k++) {
- memcpy(&temp_c[k * copySize], props.field + (delta + k) * props.partSize, copySize);
+ for (int k = 0; k < N; k++) {
+ memcpy(&temp_c[k * copySize], props.field + (delta + k) * props.partSize,
+ copySize);
}
}
/**
- * @brief Mapper function to copy #part into a buffer of floats using a conversion function.
+ * @brief Mapper function to copy #part into a buffer of floats using a
+ * conversion function.
*/
-void io_convert_part_f_mapper(void *restrict temp, int N,
- void *restrict extra_data) {
+void io_convert_part_f_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
- const struct io_props props = *((const struct io_props*) extra_data);
- const struct part *restrict parts = props.parts;
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct part* restrict parts = props.parts;
const struct engine* e = props.e;
const size_t dim = props.dimension;
/* How far are we with this chunk? */
- float *restrict temp_f = temp;
+ float* restrict temp_f = temp;
const ptrdiff_t delta = (temp_f - props.start_temp_f) / dim;
-
+
for (int i = 0; i < N; i++)
props.convert_part_f(e, parts + delta + i, &temp_f[i * dim]);
}
/**
- * @brief Mapper function to copy #part into a buffer of doubles using a conversion function.
+ * @brief Mapper function to copy #part into a buffer of doubles using a
+ * conversion function.
*/
-void io_convert_part_d_mapper(void *restrict temp, int N,
- void *restrict extra_data) {
+void io_convert_part_d_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
- const struct io_props props = *((const struct io_props*) extra_data);
- const struct part *restrict parts = props.parts;
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct part* restrict parts = props.parts;
const struct engine* e = props.e;
const size_t dim = props.dimension;
/* How far are we with this chunk? */
- double *restrict temp_d = temp;
+ double* restrict temp_d = temp;
const ptrdiff_t delta = (temp_d - props.start_temp_d) / dim;
-
+
for (int i = 0; i < N; i++)
props.convert_part_d(e, parts + delta + i, &temp_d[i * dim]);
}
/**
- * @brief Mapper function to copy #gpart into a buffer of floats using a conversion function.
+ * @brief Mapper function to copy #gpart into a buffer of floats using a
+ * conversion function.
*/
-void io_convert_gpart_f_mapper(void *restrict temp, int N,
- void *restrict extra_data) {
+void io_convert_gpart_f_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
- const struct io_props props = *((const struct io_props*) extra_data);
- const struct gpart *restrict gparts = props.gparts;
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct gpart* restrict gparts = props.gparts;
const struct engine* e = props.e;
const size_t dim = props.dimension;
/* How far are we with this chunk? */
- float *restrict temp_f = temp;
+ float* restrict temp_f = temp;
const ptrdiff_t delta = (temp_f - props.start_temp_f) / dim;
-
+
for (int i = 0; i < N; i++)
props.convert_gpart_f(e, gparts + delta + i, &temp_f[i * dim]);
}
/**
- * @brief Mapper function to copy #gpart into a buffer of doubles using a conversion function.
+ * @brief Mapper function to copy #gpart into a buffer of doubles using a
+ * conversion function.
*/
-void io_convert_gpart_d_mapper(void *restrict temp, int N,
- void *restrict extra_data) {
+void io_convert_gpart_d_mapper(void* restrict temp, int N,
+ void* restrict extra_data) {
- const struct io_props props = *((const struct io_props*) extra_data);
- const struct gpart *restrict gparts = props.gparts;
+ const struct io_props props = *((const struct io_props*)extra_data);
+ const struct gpart* restrict gparts = props.gparts;
const struct engine* e = props.e;
const size_t dim = props.dimension;
/* How far are we with this chunk? */
- double *restrict temp_d = temp;
+ double* restrict temp_d = temp;
const ptrdiff_t delta = (temp_d - props.start_temp_d) / dim;
-
+
for (int i = 0; i < N; i++)
props.convert_gpart_d(e, gparts + delta + i, &temp_d[i * dim]);
}
-
/**
* @brief Copy the particle data into a temporary buffer ready for i/o.
*
@@ -530,54 +533,58 @@ void io_copy_temp_buffer(void* temp, const struct engine* e,
props.start_temp_c = temp_c;
/* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*) &e->threadpool, io_copy_mapper,
- temp_c, N, copySize, 0, (void*) &props);
-
+ threadpool_map((struct threadpool*)&e->threadpool, io_copy_mapper, temp_c,
+ N, copySize, 0, (void*)&props);
+
} else { /* Converting particle to data */
if (props.convert_part_f != NULL) {
/* Prepare some parameters */
- float *temp_f = temp;
+ float* temp_f = temp;
props.start_temp_f = temp;
props.e = e;
/* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*) &e->threadpool, io_convert_part_f_mapper,
- temp_f, N, copySize, 0, (void*) &props);
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_part_f_mapper, temp_f, N, copySize, 0,
+ (void*)&props);
} else if (props.convert_part_d != NULL) {
-
+
/* Prepare some parameters */
- double *temp_d = temp;
+ double* temp_d = temp;
props.start_temp_d = temp;
props.e = e;
/* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*) &e->threadpool, io_convert_part_d_mapper,
- temp_d, N, copySize, 0, (void*) &props);
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_part_d_mapper, temp_d, N, copySize, 0,
+ (void*)&props);
} else if (props.convert_gpart_f != NULL) {
/* Prepare some parameters */
- float *temp_f = temp;
+ float* temp_f = temp;
props.start_temp_f = temp;
props.e = e;
/* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*) &e->threadpool, io_convert_gpart_f_mapper,
- temp_f, N, copySize, 0, (void*) &props);
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_gpart_f_mapper, temp_f, N, copySize, 0,
+ (void*)&props);
} else if (props.convert_gpart_d != NULL) {
/* Prepare some parameters */
- double *temp_d = temp;
+ double* temp_d = temp;
props.start_temp_d = temp;
props.e = e;
/* Copy the whole thing into a buffer */
- threadpool_map((struct threadpool*) &e->threadpool, io_convert_gpart_d_mapper,
- temp_d, N, copySize, 0, (void*) &props);
+ threadpool_map((struct threadpool*)&e->threadpool,
+ io_convert_gpart_d_mapper, temp_d, N, copySize, 0,
+ (void*)&props);
} else {
error("Missing conversion function");
diff --git a/src/parallel_io.c b/src/parallel_io.c
index 847ad69a4e6d07039f8876c0fc987f2803bc1411..79b35223d0adbb4e28ae183b399785ff961ff607 100644
--- a/src/parallel_io.c
+++ b/src/parallel_io.c
@@ -266,16 +266,16 @@ void writeArray_chunk(struct engine* e, hid_t h_data, hid_t h_plist_id,
num_elements * typeSize) != 0)
error("Unable to allocate temporary i/o buffer");
- MPI_Barrier(MPI_COMM_WORLD);
- ticks tic = getticks();
+ /* MPI_Barrier(MPI_COMM_WORLD); */
+ /* ticks tic = getticks(); */
/* Copy the particle data to the temporary buffer */
io_copy_temp_buffer(temp, e, props, N, internal_units, snapshot_units);
- MPI_Barrier(MPI_COMM_WORLD);
- if(engine_rank == 0)
- message( "Copying for '%s' took %.3f %s." , props.name,
- clocks_from_ticks(getticks() - tic), clocks_getunit());
+ /* MPI_Barrier(MPI_COMM_WORLD); */
+ /* if(engine_rank == 0) */
+ /* message( "Copying for '%s' took %.3f %s." , props.name, */
+ /* clocks_from_ticks(getticks() - tic), clocks_getunit()); */
/* Create data space */
const hid_t h_memspace = H5Screate(H5S_SIMPLE);
@@ -322,8 +322,8 @@ void writeArray_chunk(struct engine* e, hid_t h_data, hid_t h_plist_id,
/* N, props.name, N * props.dimension, N * props.dimension * typeSize, */
/* (int)(N * props.dimension * typeSize), offset); */
- MPI_Barrier(MPI_COMM_WORLD);
- tic = getticks();
+ /* MPI_Barrier(MPI_COMM_WORLD); */
+ /* tic = getticks(); */
/* Write temporary buffer to HDF5 dataspace */
h_err = H5Dwrite(h_data, io_hdf5_type(props.type), h_memspace, h_filespace,
@@ -332,17 +332,17 @@ void writeArray_chunk(struct engine* e, hid_t h_data, hid_t h_plist_id,
error("Error while writing data array '%s'.", props.name);
}
- MPI_Barrier(MPI_COMM_WORLD);
- ticks toc = getticks();
- float ms = clocks_from_ticks(toc - tic);
- int megaBytes = N * props.dimension * typeSize / (1024 * 1024);
- int total = 0;
- MPI_Reduce(&megaBytes, &total, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
-
- if(engine_rank == 0)
- message( "H5Dwrite for '%s' (%d MB) took %.3f %s (speed = %f MB/s)." , props.name, total,
- ms, clocks_getunit(), total / (ms / 1000.));
-
+ /* MPI_Barrier(MPI_COMM_WORLD); */
+ /* ticks toc = getticks(); */
+ /* float ms = clocks_from_ticks(toc - tic); */
+ /* int megaBytes = N * props.dimension * typeSize / (1024 * 1024); */
+ /* int total = 0; */
+ /* MPI_Reduce(&megaBytes, &total, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD); */
+
+ if (engine_rank == 0)
+ message("H5Dwrite for '%s' (%d MB) took %.3f %s (speed = %f MB/s).",
+ props.name, total, ms, clocks_getunit(), total / (ms / 1000.));
+
/* Free and close everything */
free(temp);
H5Sclose(h_memspace);
@@ -373,7 +373,7 @@ void writeArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile,
const struct unit_system* snapshot_units) {
const size_t typeSize = io_sizeof_type(props.type);
- const ticks tic = getticks();
+ /* const ticks tic = getticks(); */
/* Work out properties of the array in the file */
int rank;
@@ -488,10 +488,10 @@ void writeArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile,
H5Dclose(h_data);
H5Pclose(h_plist_id);
- MPI_Barrier(MPI_COMM_WORLD);
- if(engine_rank == 0)
- message( "'%s' took %.3f %s." , props.name,
- clocks_from_ticks(getticks() - tic), clocks_getunit());
+ /* MPI_Barrier(MPI_COMM_WORLD); */
+ /* if(engine_rank == 0) */
+ /* message( "'%s' took %.3f %s." , props.name, */
+ /* clocks_from_ticks(getticks() - tic), clocks_getunit()); */
}
/**
@@ -822,38 +822,42 @@ void write_output_parallel(struct engine* e, const char* baseName,
/* Prepare the XMF file for the new entry */
if (mpi_rank == 0) xmfFile = xmf_prepare_file(baseName);
- /* Open HDF5 file */
+ /* Prepare some file-access properties */
hid_t plist_id = H5Pcreate(H5P_FILE_ACCESS);
- //MPI_Info_set(info, "IBM_largeblock_io", "true");
+
+ /* Set some MPI-IO parameters */
+ // MPI_Info_set(info, "IBM_largeblock_io", "true");
MPI_Info_set(info, "romio_cb_write", "enable");
MPI_Info_set(info, "romio_ds_write", "disable");
- H5Pset_fapl_mpio(plist_id, comm, info);
- hid_t h_err = H5Pset_alignment(plist_id, IO_BUFFER_ALIGNMENT, 4096);
- if(h_err < 0)
- error("Error setting Hdf5 alignment");
+
+ /* Activate parallel i/o */
+ hid_t h_err = H5Pset_fapl_mpio(plist_id, comm, info);
+ if (h_err < 0) error("Error setting parallel i/o");
+
+ /* Align on 4k pages. */
+ h_err = H5Pset_alignment(plist_id, 1024, 4096);
+ if (h_err < 0) error("Error setting Hdf5 alignment");
/* Disable meta-data cache eviction */
H5AC_cache_config_t mdc_config;
mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION;
h_err = H5Pget_mdc_config(plist_id, &mdc_config);
- if( h_err < 0)
- error("Error getting the MDC config");
+ if (h_err < 0) error("Error getting the MDC config");
mdc_config.evictions_enabled = 0; /* false */
mdc_config.incr_mode = H5C_incr__off;
mdc_config.decr_mode = H5C_decr__off;
- mdc_config.flash_incr_mode = H5C_flash_incr__off;
+ mdc_config.flash_incr_mode = H5C_flash_incr__off;
h_err = H5Pset_mdc_config(plist_id, &mdc_config);
- if( h_err < 0)
- error("Error setting the MDC config");
+ if (h_err < 0) error("Error setting the MDC config");
+ /* Open HDF5 file with the chosen parameters */
h_file = H5Fcreate(fileName, H5F_ACC_TRUNC, H5P_DEFAULT, plist_id);
if (h_file < 0) {
error("Error while opening file '%s'.", fileName);
}
- /* Compute offset in the file and total number of
- * particles */
+ /* Compute offset in the file and total number of particles */
size_t N[swift_type_count] = {Ngas, Ndm, 0, 0, Nstars, 0};
long long N_total[swift_type_count] = {0};
long long offset[swift_type_count] = {0};
@@ -866,8 +870,7 @@ void write_output_parallel(struct engine* e, const char* baseName,
MPI_Bcast(&N_total, 6, MPI_LONG_LONG_INT, mpi_size - 1, comm);
/* Now everybody konws its offset and the total number of
- * particles of each
- * type */
+ * particles of each type */
/* Write the part of the XMF file corresponding to this
* specific output */