Skip to content

GitLab

Commit bc4eb419 authored by Loic Hausammann's avatar Loic Hausammann
Browse files

Logger: Move towards using directly the map and not the offset

parent 8bb5ae6c
Hide whitespace changes
Inline Side-by-side
logger/Makefile.am
View file @ bc4eb419
......@@ -28,7 +28,8 @@ BIN_LDFLAGS = -version-info 0:0:0
GIT_CMD = @GIT_CMD@
# Additional dependencies for shared libraries.
EXTRA_LIBS = $(PROFILER_LIBS) $(TCMALLOC_LIBS) $(JEMALLOC_LIBS) $(TBBMALLOC_LIBS) $(HDF5_LIBS) $(FFTW_LIBS) $(GRACKLE_LIBS) $(VELOCIRAPTOR_LIBS) $(GSL_LIBS)
EXTRA_LIBS = $(PROFILER_LIBS) $(TCMALLOC_LIBS) $(JEMALLOC_LIBS) $(TBBMALLOC_LIBS) $(HDF5_LIBS) $(FFTW_LIBS) $(GRACKLE_LIBS) \
$(VELOCIRAPTOR_LIBS) $(GSL_LIBS) -L../src/.libs -lswiftsim
# MPI libraries.
# MPI_LIBS = $(MPI_THREAD_LIBS)
......
logger/logger_header.c
View file @ bc4eb419
......@@ -92,7 +92,7 @@ void header_change_offset_direction(struct header *h, enum logger_offset_directi
/* Skip file format and version numbers */
size_t offset = LOGGER_VERSION_SIZE + 2 * sizeof(int);
logger_io_write_data(h->log->log.map, LOGGER_NUMBER_SIZE, &new_value, offset);
logger_io_write_data(h->log->log.map + offset, LOGGER_NUMBER_SIZE, &new_value);
}
/**
......@@ -102,7 +102,6 @@ void header_change_offset_direction(struct header *h, enum logger_offset_directi
* @param log The #logger_logfile.
*/
void header_read(struct header *h, struct logger_logfile *log) {
size_t offset = 0;
void *map = log->log.map;
/* Set pointer to log */
......@@ -110,39 +109,39 @@ void header_read(struct header *h, struct logger_logfile *log) {
/* read file format */
char file_format[STRING_SIZE];
offset = logger_io_read_data(map, LOGGER_VERSION_SIZE, &file_format, offset);
map = logger_io_read_data(map, LOGGER_VERSION_SIZE, &file_format);
if (strcmp(file_format, "SWIFT_LOGGER"))
error("Wrong file format (%s)", file_format);
/* Read major version number */
offset = logger_io_read_data(map, sizeof(int), &h->major_version, offset);
map = logger_io_read_data(map, sizeof(int), &h->major_version);
/* Read minor version number */
offset = logger_io_read_data(map, sizeof(int), &h->minor_version, offset);
map = logger_io_read_data(map, sizeof(int), &h->minor_version);
if (h->log->reader->verbose > 0)
message("File version %i.%i", h->major_version, h->minor_version);
/* read the offset directions */
offset = logger_io_read_data(map, LOGGER_NUMBER_SIZE, &h->offset_direction, offset);
map = logger_io_read_data(map, LOGGER_NUMBER_SIZE, &h->offset_direction);
if (!header_is_forward(h) && !header_is_backward(h) &&
!header_is_corrupted(h))
error("Wrong offset value in the header (%i)", h->offset_direction);
/* read offset to first record */
offset = logger_io_read_data(map, LOGGER_OFFSET_SIZE, &h->offset_first_record, offset);
map = logger_io_read_data(map, LOGGER_OFFSET_SIZE, &h->offset_first_record);
/* read the size of the strings */
offset = logger_io_read_data(map, LOGGER_NUMBER_SIZE, &h->string_length, offset);
map = logger_io_read_data(map, LOGGER_NUMBER_SIZE, &h->string_length);
/* check if value defined in this file is large enough. */
if (STRING_SIZE < h->string_length) {
error("Name too large in log file");
error("Name too large in log file %li", h->string_length);
}
/* read the number of masks. */
offset = logger_io_read_data(map, LOGGER_NUMBER_SIZE, &h->number_mask, offset);
map = logger_io_read_data(map, LOGGER_NUMBER_SIZE, &h->number_mask);
/* allocate the masks memory. */
h->masks = malloc(sizeof(struct mask_data) * h->number_mask);
......@@ -150,18 +149,19 @@ void header_read(struct header *h, struct logger_logfile *log) {
/* loop over all masks. */
for (size_t i = 0; i < h->number_mask; i++) {
/* read the mask name. */
offset = logger_io_read_data(map, h->string_length, h->masks[i].name, offset);
map = logger_io_read_data(map, h->string_length, h->masks[i].name);
/* set the mask value. */
h->masks[i].mask = 1 << i;
/* read the mask data size. */
offset = logger_io_read_data(map, LOGGER_NUMBER_SIZE, &h->masks[i].size, offset);
map = logger_io_read_data(map, LOGGER_NUMBER_SIZE, &h->masks[i].size);
}
/* Check the logfile header's size */
if (offset != h->offset_first_record) {
if (map != log->log.map + h->offset_first_record) {
header_print(h);
size_t offset = map - log->log.map;
error("Wrong header size (in header %zi, current %zi)", h->offset_first_record,
offset);
}
......
logger/logger_index.c
View file @ bc4eb419
......@@ -42,16 +42,16 @@ void logger_index_init(struct logger_index *index, struct logger_reader *reader,
/* Open file. */
index->data = logger_io_mmap_file(filename, &index->file_size);
/* Read the double time. */
size_t offset = 0;
offset = logger_io_read_data(index->data, sizeof(double), &index->time, offset);
/* /\* Read the double time. *\/ */
/* size_t offset = 0; */
/* offset = logger_io_read_data(index->data, sizeof(double), &index->time, offset); */
/* Read the integer time. */
offset = logger_io_read_data(index->data, sizeof(integertime_t), &index->int_time, offset);
/* /\* Read the integer time. *\/ */
/* offset = logger_io_read_data(index->data, sizeof(integertime_t), &index->int_time, offset); */
/* Read the number of particles. */
offset = logger_io_read_data(index->data, swift_type_count * sizeof(long long),
&index->number_particles, offset);
/* /\* Read the number of particles. *\/ */
/* offset = logger_io_read_data(index->data, swift_type_count * sizeof(long long), */
/* &index->number_particles, offset); */
/* Count total number of particles. */
long long N = 0;
......
logger/logger_io.h
View file @ bc4eb419
......@@ -38,47 +38,44 @@ void logger_io_munmap_file(void *map, size_t file_size);
*
* @param h #header file structure.
* @param data Pointer to the data to read.
* @param offset position in the file.
* @param mask mask read from the data.
* @param diff_offset offset difference to previous/next corresponding record read.
* from the data.
* @param mask (output) mask read from the data.
* @param diff_offset (output) offset difference to previous/next corresponding record.
*
* @return offset after the record header.
* @return memory after the record header.
*/
__attribute__((always_inline)) INLINE static size_t logger_io_read_mask(
const struct header *h, void *data, size_t offset, size_t *mask,
__attribute__((always_inline)) INLINE static void* logger_io_read_mask(
const struct header *h, void *data, size_t *mask,
size_t *diff_offset) {
/* read mask */
if (mask) {
*mask = 0;
memcpy(mask, data + offset, LOGGER_MASK_SIZE);
memcpy(mask, data, LOGGER_MASK_SIZE);
}
offset += LOGGER_MASK_SIZE;
data += LOGGER_MASK_SIZE;
/* read offset */
if (diff_offset) {
*diff_offset = 0;
memcpy(diff_offset, data + offset, LOGGER_OFFSET_SIZE);
memcpy(diff_offset, data, LOGGER_OFFSET_SIZE);
}
offset += LOGGER_OFFSET_SIZE;
data += LOGGER_OFFSET_SIZE;
return offset;
return data;
}
/**
* @brief read a single value in a file.
* @brief read a single value from a file.
*
* @param data Pointer to the data to read.
* @param size size of the data to read.
* @param p pointer where to store the data.
* @param offset position to read.
* @return offset after the data written.
* @return memory after the data written.
*/
__attribute__((always_inline)) INLINE static size_t logger_io_read_data(
void *data, const size_t size, void *p, size_t offset) {
memcpy(p, data + offset, size);
return offset + size;
__attribute__((always_inline)) INLINE static void* logger_io_read_data(
void *data, const size_t size, void *p) {
memcpy(p, data, size);
return data + size;
};
/**
......@@ -87,15 +84,14 @@ __attribute__((always_inline)) INLINE static size_t logger_io_read_data(
* @param data Pointer to the data to read.
* @param size size of the data to write.
* @param p pointer to the data.
* @param offset position to write.
*
* @return offset after the data written.
* @return memory after the data written.
*/
__attribute__((always_inline)) INLINE static size_t logger_io_write_data(
void *data, const size_t size, const void *p, size_t offset) {
memcpy(data + offset, p, size);
__attribute__((always_inline)) INLINE static void* logger_io_write_data(
void *data, const size_t size, const void *p) {
memcpy(data, p, size);
return offset + size;
return data + size;
};
#endif // __LOGGER_LOGGER_IO_H__
logger/logger_particle.c
View file @ bc4eb419
......@@ -68,15 +68,13 @@ void logger_particle_init(struct logger_particle *part) {
*
* @param part The #logger_particle to update.
* @param map The mapped data.
* @param offset position to read.
* @param field field to read.
* @param size number of bits to read.
*
* @return position after the data read.
* @return mapped data after the block read.
*/
size_t logger_particle_read_field(struct logger_particle *part, void *map,
size_t offset, const char *field,
const size_t size) {
void* logger_particle_read_field(struct logger_particle *part, void *map,
const char *field, const size_t size) {
void *p = NULL;
/* Get the correct pointer */
......@@ -99,7 +97,7 @@ size_t logger_particle_read_field(struct logger_particle *part, void *map,
}
/* read the data */
offset = logger_io_read_data(map, size, p, offset);
map = logger_io_read_data(map, size, p);
/* Split the required fields */
if (strcmp("consts", field) == 0) {
......@@ -112,7 +110,7 @@ size_t logger_particle_read_field(struct logger_particle *part, void *map,
free(p);
}
return offset;
return map;
}
/**
......@@ -142,7 +140,7 @@ size_t logger_particle_read(struct logger_particle *part, const struct logger_re
logger_particle_init(part);
/* Read the record mask */
offset = logger_io_read_mask(h, map, offset, &mask, &h_offset);
map = logger_io_read_mask(h, map + offset, &mask, &h_offset);
/* Check if it is not a time record */
if (mask != 127) error("Unexpected mask: %lu", mask);
......@@ -150,19 +148,21 @@ size_t logger_particle_read(struct logger_particle *part, const struct logger_re
/* Read all the fields */
for (size_t i = 0; i < h->number_mask; i++) {
if (mask & h->masks[i].mask) {
offset = logger_particle_read_field(part, map, offset, h->masks[i].name,
h->masks[i].size);
map = logger_particle_read_field(part, map, h->masks[i].name,
h->masks[i].size);
}
}
/* Get the time of current record. */
if (times) {
/* move offset by 1 in order to be in the required record. */
part->time = time_array_get_time(times, offset - 1);
if (times->next) {
part->time = time_array_get_time(times, offset);
}
else
part->time = -1;
/* update the offset */
offset = (size_t) (map - reader->log.log.map);
/* Check if an interpolation is required. */
if (reader_type == logger_reader_const)
return offset;
......@@ -177,7 +177,7 @@ size_t logger_particle_read(struct logger_particle *part, const struct logger_re
/* No next particle. */
if (h_offset == 0)
return offset;
return (size_t) (map - reader->log.log.map);
/* get absolute offset of next particle. */
h_offset += offset - header_get_record_size_from_mask(h, mask) - LOGGER_MASK_SIZE -
......
logger/logger_particle.h
View file @ bc4eb419
......@@ -96,9 +96,8 @@ size_t logger_particle_read(struct logger_particle *part, const struct logger_re
void logger_particle_init(struct logger_particle *part);
size_t logger_particle_read_field(struct logger_particle *part, void *map,
size_t offset, const char *field,
const size_t size);
void* logger_particle_read_field(struct logger_particle *part, void *map,
const char *field, const size_t size);
void logger_particle_interpolate(struct logger_particle *part_curr,
const struct logger_particle *part_next,
......
logger/logger_time.c
View file @ bc4eb419
......@@ -44,7 +44,7 @@ size_t time_read(integertime_t *int_time, double *time, const struct logger_read
*time = 0;
/* read record header */
offset = logger_io_read_mask(h, map, offset, &mask, &prev_offset);
map = logger_io_read_mask(h, map + offset, &mask, &prev_offset);
#ifdef SWIFT_DEBUG_CHECKS
......@@ -57,10 +57,10 @@ size_t time_read(integertime_t *int_time, double *time, const struct logger_read
#endif
/* read the record */
offset = logger_io_read_data(map, sizeof(unsigned long long int), int_time, offset);
offset = logger_io_read_data(map, sizeof(double), time, offset);
map = logger_io_read_data(map, sizeof(unsigned long long int), int_time);
map = logger_io_read_data(map, sizeof(double), time);
return offset;
return (size_t) (map - h->log->log.map);
}
/**
......@@ -82,7 +82,7 @@ size_t time_offset_first_record(const struct header *h) {
if (i == -1) error("Time mask not present in the log file header");
size_t mask = 0;
logger_io_read_mask(h, map, offset, &mask, NULL);
logger_io_read_mask(h, map + offset, &mask, NULL);
if (mask != h->masks[i].mask) error("Log file should begin by timestep");
......
logger/logger_tools.c
View file @ bc4eb419
......@@ -50,7 +50,7 @@ int tools_get_next_record(const struct header *h, void *map, size_t *offset,
*
* @param h #header structure of the file
* @param map file mapping
* @param offset In: initial offset, Out: offset of the next record
* @param offset (Out) offset of the next record
*
* @return error code, -1 if no next record
*/
......@@ -59,7 +59,7 @@ int _tools_get_next_record_forward(const struct header *h, void *map,
size_t diff_offset = 0;
/* read offset */
*offset = logger_io_read_mask(h, map, *offset, NULL, &diff_offset);
map = logger_io_read_mask(h, map + *offset, NULL, &diff_offset);
if (diff_offset == 0) return -1;
......@@ -90,7 +90,7 @@ int _tools_get_next_record_backward(const struct header *h, void *map,
while (current_offset < file_size) {
size_t mask = 0;
size_t prev_offset;
current_offset = logger_io_read_mask(h, map, current_offset, &mask, &prev_offset);
logger_io_read_mask(h, map + current_offset, &mask, &prev_offset);
prev_offset = current_offset - prev_offset - record_header;
if (*offset == prev_offset) {
......@@ -109,26 +109,27 @@ int _tools_get_next_record_backward(const struct header *h, void *map,
*
* From current record, switch side of the offset of the previous one.
* @param h #header structure of the file.
* @param map file mapping.
* @param file_map file mapping.
* @param offset position of the record.
*
* @return position after the record.
*/
size_t tools_reverse_offset(const struct header *h, void *map, size_t offset) {
size_t tools_reverse_offset(const struct header *h, void *file_map, size_t offset) {
size_t mask = 0;
size_t prev_offset = 0;
const size_t cur_offset = offset;
void *map = file_map;
/* read mask + offset */
offset = logger_io_read_mask(h, map, offset, &mask, &prev_offset);
map = logger_io_read_mask(h, map + offset, &mask, &prev_offset);
/* write offset of zero (in case it is the last record) */
const size_t zero = 0;
offset -= LOGGER_OFFSET_SIZE;
offset = logger_io_write_data(map, LOGGER_OFFSET_SIZE, &zero, offset);
map -= LOGGER_OFFSET_SIZE;
map = logger_io_write_data(map, LOGGER_OFFSET_SIZE, &zero);
/* set offset after current record */
offset += header_get_record_size_from_mask(h, mask);
map += header_get_record_size_from_mask(h, mask);
/* first records do not have a previous partner */
if (prev_offset == cur_offset)
......@@ -139,17 +140,16 @@ size_t tools_reverse_offset(const struct header *h, void *map, size_t offset) {
cur_offset);
/* modify previous offset */
size_t abs_prev_offset = cur_offset - prev_offset + LOGGER_MASK_SIZE;
abs_prev_offset = logger_io_write_data(map, LOGGER_OFFSET_SIZE, &prev_offset, abs_prev_offset);
map = file_map + cur_offset - prev_offset + LOGGER_MASK_SIZE;
map = logger_io_write_data(map, LOGGER_OFFSET_SIZE, &prev_offset);
#ifdef SWIFT_DEBUG_CHECKS
size_t prev_mask = 0;
abs_prev_offset -= LOGGER_MASK_SIZE + LOGGER_OFFSET_SIZE;
abs_prev_offset = logger_io_read_mask(h, map, abs_prev_offset, &prev_mask, NULL);
map -= LOGGER_MASK_SIZE + LOGGER_OFFSET_SIZE;
map = logger_io_read_mask(h, map, &prev_mask, NULL);
if (prev_mask != mask)
error("Unexpected mask: %lu (at %lu), got %lu (at %lu)", mask, offset,
prev_mask, prev_offset);
error("Unexpected mask: %lu, got %lu", mask, prev_mask);
#endif // SWIFT_DEBUG_CHECKS
......@@ -173,52 +173,48 @@ size_t tools_check_record_consistency(const struct logger_reader *reader, size_t
#endif
const struct header *h = &reader->log.header;
void *map = reader->log.log.map;
size_t tmp = offset;
void *file_init = reader->log.log.map;
void *map = file_init + offset;
size_t mask;
size_t pointed_offset;
/* read mask + offset */
offset = logger_io_read_mask(h, map, offset, &mask, &pointed_offset);
map = logger_io_read_mask(h, map, &mask, &pointed_offset);
/* get absolute offset */
if (header_is_forward(h))
pointed_offset += tmp;
pointed_offset += offset;
else if (header_is_backward(h)) {
if (tmp < pointed_offset)
error("Offset too large (%lu) at %lu with mask %lu", pointed_offset, tmp,
if (offset < pointed_offset)
error("Offset too large (%lu) at %lu with mask %lu", pointed_offset, offset,
mask);
pointed_offset = tmp - pointed_offset;
pointed_offset = offset - pointed_offset;
}
else {
error("Offset are corrupted");
}
/* set offset after current record */
offset += header_get_record_size_from_mask(h, mask);
map += header_get_record_size_from_mask(h, mask);
if (pointed_offset == tmp || pointed_offset == 0)
return offset;
if (pointed_offset == offset || pointed_offset == 0)
return (size_t) (map - file_init);
/* read mask of the pointed record */
size_t pointed_mask = 0;
pointed_offset = logger_io_read_mask(h, map, pointed_offset, &pointed_mask, NULL);
map = logger_io_read_mask(h, map + pointed_offset, &pointed_mask, NULL);
/* check masks */
if (pointed_mask != mask)
error("Error in the offset (mask %lu != %lu) at %lu and %lu", mask,
pointed_mask,
offset - header_get_record_size_from_mask(h, mask) - LOGGER_MASK_SIZE -
LOGGER_OFFSET_SIZE,
pointed_offset - LOGGER_MASK_SIZE - LOGGER_OFFSET_SIZE);
error("Error in the offset (mask %lu != %lu)", mask,
pointed_mask);
if (pointed_mask == 128)
return offset;
return (size_t) (map - file_init);
struct logger_particle part;
tmp = logger_particle_read(&part, reader, tmp, 0, logger_reader_const);
size_t tmp = logger_particle_read(&part, reader, offset, 0, logger_reader_const);
size_t id = part.id;
tmp = pointed_offset - LOGGER_MASK_SIZE - LOGGER_OFFSET_SIZE;
......@@ -227,6 +223,6 @@ size_t tools_check_record_consistency(const struct logger_reader *reader, size_t
if (id != part.id)
error("Offset wrong, id incorrect (%lu != %lu) at %lu", id, part.id, tmp);
return offset;
return (size_t) (map - file_init);
}
logger/tests/Makefile.am
View file @ bc4eb419
......@@ -20,16 +20,17 @@ AM_CFLAGS = -I$(top_srcdir)/src -I$(top_srcdir)/logger $(HDF5_CPPFLAGS) $(GSL_IN
AM_LDFLAGS = ../../src/.libs/libswiftsim.a ../.libs/liblogger.a $(HDF5_LDFLAGS) $(HDF5_LIBS) $(FFTW_LIBS) $(TCMALLOC_LIBS) $(JEMALLOC_LIBS) $(TBBMALLOC_LIBS) $(GRACKLE_LIBS) $(GSL_LIBS) $(PROFILER_LIBS)
# List of programs and scripts to run in the test suite
TESTS = testHeader
TESTS = testHeader testReader
# List of test programs to compile
check_PROGRAMS = testHeader
check_PROGRAMS = testHeader testReader
# Rebuild tests when SWIFT is updated.
$(check_PROGRAMS): ../../src/.libs/libswiftsim.a
# Sources for the individual programs
testHeader_SOURCES = testHeader.c
testReader_SOURCES = testReader.c
# Files necessary for distribution
EXTRA_DIST = swift_params.yml
EXTRA_DIST = testHeader.yml testReader.yml
logger/tests/testHeader.c
View file @ bc4eb419
......@@ -32,7 +32,7 @@ int main(int argc, char *argv[]) {
/* Create required structures. */
struct logger log;
struct swift_params params;
char filename[200] = "swift_params.yml";
char filename[200] = "testHeader.yml";
/* Read parameters. */
parser_read_file(filename, &params);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment