Commit 0fdbda15 authored by Peter W. Draper's avatar Peter W. Draper
Browse files

Merge branch 'updated_io' into 'master'

Compressed i/o

This adds gzip compression to the datasets within the HDF5 outputs. That can save a lot of storage space for arrays that are constant (mass,...) or similar/regular (IDs, h, ...).

We can play with the chunk sizes and compression levels later on, when we have "real" runs and a parameter file.

It "solves" issue #106.

Feel free to remove the source branch.

See merge request !98
parents d3f5f664 4b7c35b6
......@@ -247,7 +247,8 @@ void writeArrayBackEnd(hid_t grp, char* fileName, FILE* xmfFile, char* name,
}
/* Create dataset */
h_data = H5Dcreate1(grp, name, hdf5Type(type), h_filespace, H5P_DEFAULT);
h_data = H5Dcreate(grp, name, hdf5Type(type), h_filespace, H5P_DEFAULT,
H5P_DEFAULT, H5P_DEFAULT);
if (h_data < 0) {
error("Error while creating dataset '%s'.", name);
}
......@@ -378,7 +379,7 @@ void read_ic_parallel(char* fileName, double dim[3], struct part** parts,
/* Open header to read simulation properties */
/* message("Reading runtime parameters..."); */
h_grp = H5Gopen1(h_file, "/RuntimePars");
h_grp = H5Gopen(h_file, "/RuntimePars", H5P_DEFAULT);
if (h_grp < 0) error("Error while opening runtime parameters\n");
/* Read the relevant information */
......@@ -389,7 +390,7 @@ void read_ic_parallel(char* fileName, double dim[3], struct part** parts,
/* Open header to read simulation properties */
/* message("Reading file header..."); */
h_grp = H5Gopen1(h_file, "/Header");
h_grp = H5Gopen(h_file, "/Header", H5P_DEFAULT);
if (h_grp < 0) error("Error while opening file header\n");
/* Read the relevant information and print status */
......@@ -423,7 +424,7 @@ void read_ic_parallel(char* fileName, double dim[3], struct part** parts,
/* Open SPH particles group */
/* message("Reading particle arrays..."); */
h_grp = H5Gopen1(h_file, "/PartType0");
h_grp = H5Gopen(h_file, "/PartType0", H5P_DEFAULT);
if (h_grp < 0) error("Error while opening particle group.\n");
/* Read particle fields into the particle structure */
......@@ -508,7 +509,8 @@ void write_output_parallel(struct engine* e, struct UnitSystem* us,
/* Open header to write simulation properties */
/* message("Writing runtime parameters..."); */
h_grp = H5Gcreate1(h_file, "/RuntimePars", 0);
h_grp =
H5Gcreate(h_file, "/RuntimePars", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (h_grp < 0) error("Error while creating runtime parameters group\n");
/* Write the relevant information */
......@@ -519,7 +521,7 @@ void write_output_parallel(struct engine* e, struct UnitSystem* us,
/* Open header to write simulation properties */
/* message("Writing file header..."); */
h_grp = H5Gcreate1(h_file, "/Header", 0);
h_grp = H5Gcreate(h_file, "/Header", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (h_grp < 0) error("Error while creating file header\n");
/* Print the relevant information and print status */
......@@ -546,7 +548,7 @@ void write_output_parallel(struct engine* e, struct UnitSystem* us,
writeCodeDescription(h_file);
/* Print the SPH parameters */
h_grpsph = H5Gcreate1(h_file, "/SPH", 0);
h_grpsph = H5Gcreate(h_file, "/SPH", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (h_grpsph < 0) error("Error while creating SPH group");
writeSPHflavour(h_grpsph);
H5Gclose(h_grpsph);
......@@ -556,7 +558,8 @@ void write_output_parallel(struct engine* e, struct UnitSystem* us,
/* Create SPH particles group */
/* message("Writing particle arrays..."); */
h_grp = H5Gcreate1(h_file, "/PartType0", 0);
h_grp =
H5Gcreate(h_file, "/PartType0", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (h_grp < 0) error("Error while creating particle group.\n");
/* Write particle fields from the particle structure */
......
......@@ -97,7 +97,7 @@ void readArrayBackEnd(hid_t grp, char* name, enum DATA_TYPE type, int N,
/* fflush(stdout); */
/* Open data space */
h_data = H5Dopen1(grp, name);
h_data = H5Dopen(grp, name, H5P_DEFAULT);
if (h_data < 0) error("Error while opening data space '%s'.", name);
/* Check data type */
......@@ -175,9 +175,10 @@ void readArrayBackEnd(hid_t grp, char* name, enum DATA_TYPE type, int N,
void prepareArray(hid_t grp, char* fileName, FILE* xmfFile, char* name,
enum DATA_TYPE type, long long N_total, int dim,
struct UnitSystem* us, enum UnitConversionFactor convFactor) {
hid_t h_data = 0, h_err = 0, h_space = 0;
hid_t h_data = 0, h_err = 0, h_space = 0, h_prop = 0;
int rank = 0;
hsize_t shape[2];
hsize_t chunk_shape[2];
char buffer[150];
/* Create data space */
......@@ -190,10 +191,14 @@ void prepareArray(hid_t grp, char* fileName, FILE* xmfFile, char* name,
rank = 2;
shape[0] = N_total;
shape[1] = dim;
chunk_shape[0] = 1 << 16; /* Just a guess...*/
chunk_shape[1] = dim;
} else {
rank = 1;
shape[0] = N_total;
shape[1] = 0;
chunk_shape[0] = 1 << 16; /* Just a guess...*/
chunk_shape[1] = 0;
}
/* Change shape of data space */
......@@ -202,8 +207,25 @@ void prepareArray(hid_t grp, char* fileName, FILE* xmfFile, char* name,
error("Error while changing data space shape for field '%s'.", name);
}
/* Dataset properties */
h_prop = H5Pcreate(H5P_DATASET_CREATE);
/* Set chunk size */
h_err = H5Pset_chunk(h_prop, rank, chunk_shape);
if (h_err < 0) {
error("Error while setting chunk size (%lld, %lld) for field '%s'.",
chunk_shape[0], chunk_shape[1], name);
}
/* Impose data compression */
h_err = H5Pset_deflate(h_prop, 4);
if (h_err < 0) {
error("Error while setting compression options for field '%s'.", name);
}
/* Create dataset */
h_data = H5Dcreate1(grp, name, hdf5Type(type), h_space, H5P_DEFAULT);
h_data = H5Dcreate(grp, name, hdf5Type(type), h_space, H5P_DEFAULT, h_prop,
H5P_DEFAULT);
if (h_data < 0) {
error("Error while creating dataspace '%s'.", name);
}
......@@ -219,6 +241,7 @@ void prepareArray(hid_t grp, char* fileName, FILE* xmfFile, char* name,
writeAttribute_f(h_data, "a-scale exponent", aFactor(us, convFactor));
writeAttribute_s(h_data, "Conversion factor", buffer);
H5Pclose(h_prop);
H5Dclose(h_data);
H5Sclose(h_space);
}
......@@ -408,7 +431,7 @@ void read_ic_serial(char* fileName, double dim[3], struct part** parts, int* N,
/* Open header to read simulation properties */
/* message("Reading runtime parameters..."); */
h_grp = H5Gopen1(h_file, "/RuntimePars");
h_grp = H5Gopen(h_file, "/RuntimePars", H5P_DEFAULT);
if (h_grp < 0) error("Error while opening runtime parameters\n");
/* Read the relevant information */
......@@ -419,7 +442,7 @@ void read_ic_serial(char* fileName, double dim[3], struct part** parts, int* N,
/* Open header to read simulation properties */
/* message("Reading file header..."); */
h_grp = H5Gopen1(h_file, "/Header");
h_grp = H5Gopen(h_file, "/Header", H5P_DEFAULT);
if (h_grp < 0) error("Error while opening file header\n");
/* Read the relevant information and print status */
......@@ -474,7 +497,7 @@ void read_ic_serial(char* fileName, double dim[3], struct part** parts, int* N,
/* Open SPH particles group */
/* message("Reading particle arrays..."); */
h_grp = H5Gopen1(h_file, "/PartType0");
h_grp = H5Gopen(h_file, "/PartType0", H5P_DEFAULT);
if (h_grp < 0)
error("Error while opening particle group on rank %d.\n", mpi_rank);
......@@ -563,7 +586,8 @@ void write_output_serial(struct engine* e, struct UnitSystem* us, int mpi_rank,
/* Open header to write simulation properties */
/* message("Writing runtime parameters..."); */
h_grp = H5Gcreate1(h_file, "/RuntimePars", 0);
h_grp = H5Gcreate(h_file, "/RuntimePars", H5P_DEFAULT, H5P_DEFAULT,
H5P_DEFAULT);
if (h_grp < 0) error("Error while creating runtime parameters group\n");
/* Write the relevant information */
......@@ -574,7 +598,7 @@ void write_output_serial(struct engine* e, struct UnitSystem* us, int mpi_rank,
/* Open header to write simulation properties */
/* message("Writing file header..."); */
h_grp = H5Gcreate1(h_file, "/Header", 0);
h_grp = H5Gcreate(h_file, "/Header", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (h_grp < 0) error("Error while creating file header\n");
/* Print the relevant information and print status */
......@@ -601,7 +625,7 @@ void write_output_serial(struct engine* e, struct UnitSystem* us, int mpi_rank,
writeCodeDescription(h_file);
/* Print the SPH parameters */
h_grpsph = H5Gcreate1(h_file, "/SPH", 0);
h_grpsph = H5Gcreate(h_file, "/SPH", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (h_grpsph < 0) error("Error while creating SPH group");
writeSPHflavour(h_grpsph);
H5Gclose(h_grpsph);
......@@ -611,7 +635,8 @@ void write_output_serial(struct engine* e, struct UnitSystem* us, int mpi_rank,
/* Create SPH particles group */
/* message("Writing particle arrays..."); */
h_grp = H5Gcreate1(h_file, "/PartType0", 0);
h_grp =
H5Gcreate(h_file, "/PartType0", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (h_grp < 0) error("Error while creating particle group.\n");
/* Close particle group */
......@@ -633,7 +658,7 @@ void write_output_serial(struct engine* e, struct UnitSystem* us, int mpi_rank,
/* Open SPH particles group */
/* message("Reading particle arrays..."); */
h_grp = H5Gopen1(h_file, "/PartType0");
h_grp = H5Gopen(h_file, "/PartType0", H5P_DEFAULT);
if (h_grp < 0)
error("Error while opening particle group on rank %d.\n", mpi_rank);
......
......@@ -94,7 +94,7 @@ void readArrayBackEnd(hid_t grp, char* name, enum DATA_TYPE type, int N,
* "compulsory": "optional ", name); */
/* Open data space */
h_data = H5Dopen1(grp, name);
h_data = H5Dopen(grp, name, H5P_DEFAULT);
if (h_data < 0) {
error("Error while opening data space '%s'.", name);
}
......@@ -157,7 +157,7 @@ void writeArrayBackEnd(hid_t grp, char* fileName, FILE* xmfFile, char* name,
enum DATA_TYPE type, int N, int dim, char* part_c,
struct UnitSystem* us,
enum UnitConversionFactor convFactor) {
hid_t h_data = 0, h_err = 0, h_space = 0;
hid_t h_data = 0, h_err = 0, h_space = 0, h_prop = 0;
void* temp = 0;
int i = 0, rank = 0;
const size_t typeSize = sizeOfType(type);
......@@ -165,6 +165,7 @@ void writeArrayBackEnd(hid_t grp, char* fileName, FILE* xmfFile, char* name,
const size_t partSize = sizeof(struct part);
char* temp_c = 0;
hsize_t shape[2];
hsize_t chunk_shape[2];
char buffer[150];
/* message("Writing '%s' array...", name); */
......@@ -188,10 +189,14 @@ void writeArrayBackEnd(hid_t grp, char* fileName, FILE* xmfFile, char* name,
rank = 2;
shape[0] = N;
shape[1] = dim;
chunk_shape[0] = 1 << 16; /* Just a guess...*/
chunk_shape[1] = dim;
} else {
rank = 1;
shape[0] = N;
shape[1] = 0;
chunk_shape[0] = 1 << 16; /* Just a guess...*/
chunk_shape[1] = 0;
}
/* Change shape of data space */
......@@ -200,8 +205,25 @@ void writeArrayBackEnd(hid_t grp, char* fileName, FILE* xmfFile, char* name,
error("Error while changing data space shape for field '%s'.", name);
}
/* Dataset properties */
h_prop = H5Pcreate(H5P_DATASET_CREATE);
/* Set chunk size */
h_err = H5Pset_chunk(h_prop, rank, chunk_shape);
if (h_err < 0) {
error("Error while setting chunk size (%lld, %lld) for field '%s'.",
chunk_shape[0], chunk_shape[1], name);
}
/* Impose data compression */
h_err = H5Pset_deflate(h_prop, 4);
if (h_err < 0) {
error("Error while setting compression options for field '%s'.", name);
}
/* Create dataset */
h_data = H5Dcreate1(grp, name, hdf5Type(type), h_space, H5P_DEFAULT);
h_data = H5Dcreate(grp, name, hdf5Type(type), h_space, H5P_DEFAULT, h_prop,
H5P_DEFAULT);
if (h_data < 0) {
error("Error while creating dataspace '%s'.", name);
}
......@@ -225,6 +247,7 @@ void writeArrayBackEnd(hid_t grp, char* fileName, FILE* xmfFile, char* name,
/* Free and close everything */
free(temp);
H5Pclose(h_prop);
H5Dclose(h_data);
H5Sclose(h_space);
}
......@@ -313,7 +336,7 @@ void read_ic_single(char* fileName, double dim[3], struct part** parts, int* N,
/* Open header to read simulation properties */
/* message("Reading runtime parameters..."); */
h_grp = H5Gopen1(h_file, "/RuntimePars");
h_grp = H5Gopen(h_file, "/RuntimePars", H5P_DEFAULT);
if (h_grp < 0) error("Error while opening runtime parameters\n");
/* Read the relevant information */
......@@ -324,7 +347,7 @@ void read_ic_single(char* fileName, double dim[3], struct part** parts, int* N,
/* Open header to read simulation properties */
/* message("Reading file header..."); */
h_grp = H5Gopen1(h_file, "/Header");
h_grp = H5Gopen(h_file, "/Header", H5P_DEFAULT);
if (h_grp < 0) error("Error while opening file header\n");
/* Read the relevant information and print status */
......@@ -352,7 +375,7 @@ void read_ic_single(char* fileName, double dim[3], struct part** parts, int* N,
/* Open SPH particles group */
/* message("Reading particle arrays..."); */
h_grp = H5Gopen1(h_file, "/PartType0");
h_grp = H5Gopen(h_file, "/PartType0", H5P_DEFAULT);
if (h_grp < 0) error("Error while opening particle group.\n");
/* Read particle fields into the particle structure */
......@@ -415,7 +438,8 @@ void write_output_single(struct engine* e, struct UnitSystem* us) {
/* Open header to write simulation properties */
/* message("Writing runtime parameters..."); */
h_grp = H5Gcreate1(h_file, "/RuntimePars", 0);
h_grp =
H5Gcreate(h_file, "/RuntimePars", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (h_grp < 0) error("Error while creating runtime parameters group\n");
/* Write the relevant information */
......@@ -426,7 +450,7 @@ void write_output_single(struct engine* e, struct UnitSystem* us) {
/* Open header to write simulation properties */
/* message("Writing file header..."); */
h_grp = H5Gcreate1(h_file, "/Header", 0);
h_grp = H5Gcreate(h_file, "/Header", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (h_grp < 0) error("Error while creating file header\n");
/* Print the relevant information and print status */
......@@ -451,7 +475,7 @@ void write_output_single(struct engine* e, struct UnitSystem* us) {
writeCodeDescription(h_file);
/* Print the SPH parameters */
h_grpsph = H5Gcreate1(h_file, "/SPH", 0);
h_grpsph = H5Gcreate(h_file, "/SPH", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (h_grpsph < 0) error("Error while creating SPH group");
writeSPHflavour(h_grpsph);
H5Gclose(h_grpsph);
......@@ -461,7 +485,8 @@ void write_output_single(struct engine* e, struct UnitSystem* us) {
/* Create SPH particles group */
/* message("Writing particle arrays..."); */
h_grp = H5Gcreate1(h_file, "/PartType0", 0);
h_grp =
H5Gcreate(h_file, "/PartType0", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
if (h_grp < 0) error("Error while creating particle group.\n");
/* Write particle fields from the particle structure */
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment