From de3cc54dcb6c1cd91c19c8bca85998dd5f69d9eb Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Sat, 2 Jun 2018 23:55:49 +0200 Subject: [PATCH 1/2] Make the code crash if a compressed field is read with parallel-hdf5 version 1.10.2. --- src/parallel_io.c | 48 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/parallel_io.c b/src/parallel_io.c index 3c073ce71..9e9ba6aed 100644 --- a/src/parallel_io.c +++ b/src/parallel_io.c @@ -206,6 +206,54 @@ void readArray(hid_t grp, struct io_props props, size_t N, long long N_total, const hid_t h_data = H5Dopen2(grp, props.name, H5P_DEFAULT); if (h_data < 0) error("Error while opening data space '%s'.", props.name); +/* Parallel-HDF5 1.10.2 incorrectly reads data that was compressed */ +/* We detect this here and crash with an error message instead of */ +/* continuing with garbage data. */ +#if H5_VERSION_LE(1, 10, 2) && H5_VERSION_GE(1, 10, 2) + if (mpi_rank == 0) { + + /* Recover the list of filters that were applied to the data */ + const hid_t h_plist = H5Dget_create_plist(h_data); + if (h_plist < 0) + error("Error getting property list for data set '%s'", props.name); + + /* Recover the number of filters in the list */ + const int n_filters = H5Pget_nfilters(props); + + for (int n = 0; n < n_filters; ++n) { + + unsigned int flag; + size_t cd_nelmts = 10; + unsigned int* cd_values = malloc(cd_nelmts * sizeof(unsigned int)); + size_t namelen = 256; + char* name = calloc(namelen * sizeof(char)); + unsigned int filter_config; + + /* Recover the n^th filter in the list */ + const H5Z_filter_t filter = + H5Pget_filter(props, n, &flag, &cd_nelmts, cd_values, namelen, name, + &filter_config); + if (filter < 0) + error("Error retrieving %d^th (%d) filter for data set '%s'", n, + f_filters, props.name); + + /* Now check whether the deflate filter had been applied */ + if (filter == H5Z_FILTER_DEFLATE) + error( + "HDF5 1.10.2 cannot correctly read data that was compressed with " + "the 'deflate' filter.\nThe field '%s' has had this filter applied " + "and the code would silently read garbage into the particle arrays " + "so we'd rather stop here.You can:\n - Recompile the code with an " + "earlier or older version of HDF5.\n - Use the 'h5repack' tool to " + "remove the filter from the ICs.", + props.name); + + free(name); + free(cd_values); + } + } +#endif + /* Create property list for collective dataset read. */ const hid_t h_plist_id = H5Pcreate(H5P_DATASET_XFER); H5Pset_dxpl_mpio(h_plist_id, H5FD_MPIO_COLLECTIVE); -- GitLab From b75cc7513976b2b3d1e23ddb9528e4088672b0ee Mon Sep 17 00:00:00 2001 From: Matthieu Schaller Date: Sun, 3 Jun 2018 00:38:11 +0200 Subject: [PATCH 2/2] Fix compiling error. Improve error message and better value for the input array sizes to the H5Pget_filter function. --- src/parallel_io.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/parallel_io.c b/src/parallel_io.c index 9e9ba6aed..ab856f42c 100644 --- a/src/parallel_io.c +++ b/src/parallel_io.c @@ -218,24 +218,24 @@ void readArray(hid_t grp, struct io_props props, size_t N, long long N_total, error("Error getting property list for data set '%s'", props.name); /* Recover the number of filters in the list */ - const int n_filters = H5Pget_nfilters(props); + const int n_filters = H5Pget_nfilters(h_plist); for (int n = 0; n < n_filters; ++n) { unsigned int flag; - size_t cd_nelmts = 10; + size_t cd_nelmts = 32; unsigned int* cd_values = malloc(cd_nelmts * sizeof(unsigned int)); size_t namelen = 256; - char* name = calloc(namelen * sizeof(char)); + char* name = calloc(namelen, sizeof(char)); unsigned int filter_config; /* Recover the n^th filter in the list */ const H5Z_filter_t filter = - H5Pget_filter(props, n, &flag, &cd_nelmts, cd_values, namelen, name, + H5Pget_filter(h_plist, n, &flag, &cd_nelmts, cd_values, namelen, name, &filter_config); if (filter < 0) error("Error retrieving %d^th (%d) filter for data set '%s'", n, - f_filters, props.name); + n_filters, props.name); /* Now check whether the deflate filter had been applied */ if (filter == H5Z_FILTER_DEFLATE) @@ -243,14 +243,17 @@ void readArray(hid_t grp, struct io_props props, size_t N, long long N_total, "HDF5 1.10.2 cannot correctly read data that was compressed with " "the 'deflate' filter.\nThe field '%s' has had this filter applied " "and the code would silently read garbage into the particle arrays " - "so we'd rather stop here.You can:\n - Recompile the code with an " + "so we'd rather stop here. You can:\n - Recompile the code with an " "earlier or older version of HDF5.\n - Use the 'h5repack' tool to " - "remove the filter from the ICs.", + "remove the filter from the ICs (e.g. h5repack -f NONE -i in_file " + "-o out_file).\n", props.name); free(name); free(cd_values); } + + H5Pclose(h_plist); } #endif -- GitLab