From 729bbd31607c842e48c10f98a80f7c81d8b665bd Mon Sep 17 00:00:00 2001
From: Alexei Borissov <alexei.borissov@ed.ac.uk>
Date: Wed, 1 Jul 2020 13:00:28 +0100
Subject: [PATCH] Check if compression turned on before deciding on IO chunk
size
---
src/distributed_io.c | 7 +++++--
src/line_of_sight.c | 7 +++++--
src/serial_io.c | 7 +++++--
src/single_io.c | 7 +++++--
4 files changed, 20 insertions(+), 8 deletions(-)
diff --git a/src/distributed_io.c b/src/distributed_io.c
index abcf637160..4a1b9865bb 100644
--- a/src/distributed_io.c
+++ b/src/distributed_io.c
@@ -109,6 +109,9 @@ void write_distributed_array(const struct engine* e, hid_t grp,
if (h_space < 0)
error("Error while creating data space for field '%s'.", props.name);
+ /* Decide what chunk size to use based on compression */
+ int log2_chunk_size = e->snapshot_compression > 0 ? 12 : 18;
+
int rank;
hsize_t shape[2];
hsize_t chunk_shape[2];
@@ -117,13 +120,13 @@ void write_distributed_array(const struct engine* e, hid_t grp,
rank = 2;
shape[0] = N;
shape[1] = props.dimension;
- chunk_shape[0] = 1 << 20; /* Just a guess...*/
+ chunk_shape[0] = 1 << log2_chunk_size;
chunk_shape[1] = props.dimension;
} else {
rank = 1;
shape[0] = N;
shape[1] = 0;
- chunk_shape[0] = 1 << 20; /* Just a guess...*/
+ chunk_shape[0] = 1 << log2_chunk_size;
chunk_shape[1] = 0;
}
diff --git a/src/line_of_sight.c b/src/line_of_sight.c
index 4c1af849b0..32d37c84a9 100644
--- a/src/line_of_sight.c
+++ b/src/line_of_sight.c
@@ -286,6 +286,9 @@ void write_los_hdf5_dataset(const struct io_props props, const size_t N,
if (h_space < 0)
error("Error while creating data space for field '%s'.", props.name);
+ /* Decide what chunk size to use based on compression */
+ int log2_chunk_size = e->snapshot_compression > 0 ? 12 : 18;
+
int rank = 0;
hsize_t shape[2];
hsize_t chunk_shape[2];
@@ -293,13 +296,13 @@ void write_los_hdf5_dataset(const struct io_props props, const size_t N,
rank = 2;
shape[0] = N;
shape[1] = props.dimension;
- chunk_shape[0] = 1 << 20; /* Just a guess...*/
+ chunk_shape[0] = 1 << log2_chunk_size;
chunk_shape[1] = props.dimension;
} else {
rank = 1;
shape[0] = N;
shape[1] = 0;
- chunk_shape[0] = 1 << 20; /* Just a guess...*/
+ chunk_shape[0] = 1 << log2_chunk_size;
chunk_shape[1] = 0;
}
diff --git a/src/serial_io.c b/src/serial_io.c
index 8dcbcb56b8..57289c15fb 100644
--- a/src/serial_io.c
+++ b/src/serial_io.c
@@ -250,6 +250,9 @@ void prepare_array_serial(const struct engine* e, hid_t grp, char* fileName,
if (h_space < 0)
error("Error while creating data space for field '%s'.", props.name);
+ /* Decide what chunk size to use based on compression */
+ int log2_chunk_size = e->snapshot_compression > 0 ? 12 : 18;
+
int rank = 0;
hsize_t shape[2];
hsize_t chunk_shape[2];
@@ -257,13 +260,13 @@ void prepare_array_serial(const struct engine* e, hid_t grp, char* fileName,
rank = 2;
shape[0] = N_total;
shape[1] = props.dimension;
- chunk_shape[0] = 1 << 20; /* Just a guess...*/
+ chunk_shape[0] = 1 << log2_chunk_size;
chunk_shape[1] = props.dimension;
} else {
rank = 1;
shape[0] = N_total;
shape[1] = 0;
- chunk_shape[0] = 1 << 20; /* Just a guess...*/
+ chunk_shape[0] = 1 << log2_chunk_size;
chunk_shape[1] = 0;
}
diff --git a/src/single_io.c b/src/single_io.c
index 3086547fd8..45e79169c9 100644
--- a/src/single_io.c
+++ b/src/single_io.c
@@ -255,6 +255,9 @@ void write_array_single(const struct engine* e, hid_t grp, char* fileName,
if (h_space < 0)
error("Error while creating data space for field '%s'.", props.name);
+ /* Decide what chunk size to use based on compression */
+ int log2_chunk_size = e->snapshot_compression > 0 ? 12 : 18;
+
int rank;
hsize_t shape[2];
hsize_t chunk_shape[2];
@@ -263,13 +266,13 @@ void write_array_single(const struct engine* e, hid_t grp, char* fileName,
rank = 2;
shape[0] = N;
shape[1] = props.dimension;
- chunk_shape[0] = 1 << 20; /* Just a guess...*/
+ chunk_shape[0] = 1 << log2_chunk_size;
chunk_shape[1] = props.dimension;
} else {
rank = 1;
shape[0] = N;
shape[1] = 0;
- chunk_shape[0] = 1 << 20; /* Just a guess...*/
+ chunk_shape[0] = 1 << log2_chunk_size;
chunk_shape[1] = 0;
}
--
GitLab