diff --git a/src/common_io.c b/src/common_io.c index 7ad518923ddbd47032ef56105a473cbf957b7007..74bc4ea657811c17d12987a7b8bd8d65efe46146 100644 --- a/src/common_io.c +++ b/src/common_io.c @@ -415,7 +415,6 @@ void io_write_cell_offsets(hid_t h_grp, const int cdim[3], offset_spart[0] = 0; /* Collect the cell information of *local* cells */ - int count_local_cells = 0; long long local_offset_part = 0; long long local_offset_gpart = 0; long long local_offset_spart = 0; @@ -443,8 +442,6 @@ void io_write_cell_offsets(hid_t h_grp, const int cdim[3], offset_gpart[i] = local_offset_gpart + global_offsets[swift_type_dark_matter]; offset_spart[i] = local_offset_spart + global_offsets[swift_type_stars]; - ++count_local_cells; - local_offset_part += count_part[i]; local_offset_gpart += count_gpart[i]; local_offset_spart += count_spart[i]; @@ -468,7 +465,7 @@ void io_write_cell_offsets(hid_t h_grp, const int cdim[3], } #ifdef WITH_MPI - /* Now, reduce all the arrays. Note that we use a bit-by-bit OR here. This + /* Now, reduce all the arrays. Note that we use a bit-wise OR here. This is safe as we made sure only local cells have non-zero values. */ if (nodeID == 0) { MPI_Reduce(MPI_IN_PLACE, count_part, nr_cells, MPI_LONG_LONG_INT, MPI_BOR, @@ -513,8 +510,7 @@ void io_write_cell_offsets(hid_t h_grp, const int cdim[3], 0, MPI_COMM_WORLD); } - - /* For the centres we use a sum as MPI does not like bitwise operations + /* For the centres we use a sum as MPI does not like bit-wise operations on floating point numbers */ if (nodeID == 0) { MPI_Reduce(MPI_IN_PLACE, centres, 3 * nr_cells, MPI_DOUBLE, MPI_SUM, @@ -552,7 +548,6 @@ void io_write_cell_offsets(hid_t h_grp, const int cdim[3], H5Dclose(h_data); H5Sclose(h_space); - /* Group containing the offsets for each particle type */ h_subgrp = H5Gcreate(h_grp, "Offsets", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); if (h_subgrp < 0) error("Error while creating offsets sub-group"); diff --git a/src/parallel_io.c b/src/parallel_io.c index c307a008cca8256642fef70848ce8ae1495463f8..8cb5fbb691bfeb321a21c2cec7a836c26d59bd44 100644 --- a/src/parallel_io.c +++ b/src/parallel_io.c @@ -1282,22 +1282,27 @@ void write_output_parallel(struct engine* e, const char* baseName, snprintf(fileName, FILENAME_BUFFER_SIZE, "%s_%04i.hdf5", baseName, e->snapshot_output_count); - hid_t h_file_cells, h_grp_cells; - if (e->nodeID == 0) { + /* Now write the top-level cell structure */ + hid_t h_file_cells = 0, h_grp_cells = 0; + if (mpi_rank == 0) { + + /* Open the snapshot on rank 0 */ h_file_cells = H5Fopen(fileName, H5F_ACC_RDWR, H5P_DEFAULT); if (h_file_cells < 0) error("Error while opening file '%s' on rank %d.", fileName, mpi_rank); + + /* Create the group we want in the file */ h_grp_cells = H5Gcreate(h_file_cells, "/Cells", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); if (h_grp_cells < 0) error("Error while creating cells group"); - } else { - h_file_cells = 0; } + /* Write the location of the particles in the arrays */ io_write_cell_offsets(h_grp_cells, e->s->cdim, e->s->cells_top, e->s->nr_cells, e->s->width, mpi_rank, N_total, offset); - if (e->nodeID == 0) { + /* Close everything */ + if (mpi_rank == 0) { H5Gclose(h_grp_cells); H5Fclose(h_file_cells); } diff --git a/src/serial_io.c b/src/serial_io.c index a273d033323702ad22a0ac63532f5f81765d5fca..3a94d3cb920852abdb1381ca3769b761b35e3a05 100644 --- a/src/serial_io.c +++ b/src/serial_io.c @@ -1028,6 +1028,31 @@ void write_output_serial(struct engine* e, const char* baseName, H5Fclose(h_file); } + /* Now write the top-level cell structure */ + hid_t h_file_cells = 0, h_grp_cells = 0; + if (mpi_rank == 0) { + + /* Open the snapshot on rank 0 */ + h_file_cells = H5Fopen(fileName, H5F_ACC_RDWR, H5P_DEFAULT); + if (h_file_cells < 0) + error("Error while opening file '%s' on rank %d.", fileName, mpi_rank); + + /* Create the group we want in the file */ + h_grp_cells = H5Gcreate(h_file_cells, "/Cells", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if (h_grp_cells < 0) + error("Error while creating cells group"); + } + + /* Write the location of the particles in the arrays */ + io_write_cell_offsets(h_grp_cells, e->s->cdim, e->s->cells_top, e->s->nr_cells, + e->s->width, mpi_rank, N_total, offset); + + /* Close everything */ + if (mpi_rank == 0) { + H5Gclose(h_grp_cells); + H5Fclose(h_file_cells); + } + /* Now loop over ranks and write the data */ for (int rank = 0; rank < mpi_size; ++rank) { diff --git a/src/single_io.c b/src/single_io.c index c96d505fc2367f4ca0b06eddf84a380886a845a2..1c9df2178193e0feb06e240e55294301cfe559f3 100644 --- a/src/single_io.c +++ b/src/single_io.c @@ -826,10 +826,12 @@ void write_output_single(struct engine* e, const char* baseName, /* Print the system of Units used internally */ io_write_unit_system(h_file, internal_units, "InternalCodeUnits"); - /* Write the location of the particles in the arrays */ + /* Now write the top-level cell structure */ long long global_offsets[swift_type_count] = {0}; h_grp = H5Gcreate(h_file, "/Cells", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); if (h_grp < 0) error("Error while creating cells group"); + + /* Write the location of the particles in the arrays */ io_write_cell_offsets(h_grp, e->s->cdim, e->s->cells_top, e->s->nr_cells, e->s->width, e->nodeID, N_total, global_offsets); H5Gclose(h_grp);