Commit 62713885 authored by Loic Hausammann's avatar Loic Hausammann Committed by Matthieu Schaller
Browse files

Logger index file

parent 887351bd
......@@ -34,3 +34,12 @@ InitialConditions:
file_name: ./sedov.hdf5
periodic: 1
smoothing_length_scaling: 3.33
# Parameters governing the logger snapshot system
Logger:
delta_step: 10 # Update the particle log every this many updates
basename: index # Common part of the filenames
initial_buffer_size: 0.01 # (Optional) Buffer size in GB
buffer_scale: 10 # (Optional) When buffer size is too small, update it with required memory times buffer_scale
time_first_index: 0. # Time of the first output (in internal units)
delta_time_index: 1e-2 # Time difference between consecutive outputs (in internal units)
......@@ -56,4 +56,3 @@ StructureFinding:
basename: ./stf
scale_factor_first: 0.1 # z = 9
delta_time: 1.04
......@@ -1419,7 +1419,10 @@ int main(int argc, char *argv[]) {
}
#ifdef WITH_LOGGER
logger_log_all(e.logger, &e);
engine_dump_index(&e);
/* Write a sentinel timestamp */
logger_log_timestamp(e.logger, e.ti_current, e.time,
&e.logger->timestamp_offset);
#endif
/* Write final snapshot? */
......
......@@ -48,11 +48,11 @@ SUBDIRS = tests
# List required headers
include_HEADERS = logger_header.h logger_loader_io.h logger_particle.h logger_time.h logger_tools.h logger_reader.h \
logger_logfile.h
logger_logfile.h logger_index.h quick_sort.h
# Common source files
AM_SOURCES = logger_header.c logger_loader_io.c logger_particle.c logger_time.c logger_tools.c logger_reader.c \
logger_logfile.c
logger_logfile.c logger_index.c quick_sort.c
if HAVEPYTHON
AM_SOURCES += logger_python_wrapper.c
endif
......
#!/usr/bin/env python3
"""
Read a logger file by using an index file.
Example: ./reader_example.py ../../examples/SedovBlast_3D/index 0.1
"""
import sys
import numpy as np
import matplotlib.pyplot as plt
sys.path.append("../.libs/")
import liblogger as logger
def plot3D(pos):
from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure()
ax = fig.add_subplot(111, projection="3d")
ax.plot(pos[:, 0], pos[:, 1], pos[:, 2], ".", markersize=0.2)
def plot2D():
center = np.array([0.5]*3)
r2 = np.sum((pos - center)**2, axis=1)
# plot entropy vs distance
plt.plot(np.sqrt(r2), data["entropies"], '.',
markersize=0.2)
plt.xlim(0., 0.5)
plt.ylim(-1, 50)
plt.xlabel("Radius")
plt.ylabel("Entropy")
basename = "../../examples/HydroTests/SedovBlast_3D/index"
time = 0.05
if len(sys.argv) >= 2:
basename = sys.argv[1]
else:
print("No basename supplied (first argument), using default.")
if len(sys.argv) >= 3:
time = float(sys.argv[2])
else:
print("No time supplied (second argument), using default.")
if len(sys.argv) > 3:
print("Ignoring excess arguments '%s'." % sys.argv[3:])
print("basename: %s" % basename)
print("time: %g" % time)
# read dump
t = logger.getTimeLimits(basename)
data = logger.loadSnapshotAtTime(basename, time)
pos = data["positions"]
plot3D(pos)
plt.show()
......@@ -45,9 +45,9 @@ void header_print(const struct header *h) {
#endif
message("First Offset: %lu.", h->offset_first_record);
message("Offset direction: %s.", logger_offset_name[h->offset_direction]);
message("Number masks: %i.", h->number_mask);
message("Number masks: %i.", h->masks_count);
for (size_t i = 0; i < h->number_mask; i++) {
for (size_t i = 0; i < h->masks_count; i++) {
message(" Mask: %s.", h->masks[i].name);
message(" Value: %u.", h->masks[i].mask);
message(" Size: %i.", h->masks[i].size);
......@@ -70,7 +70,7 @@ void header_free(struct header *h) { free(h->masks); };
* @return Index of the field (-1 if not found).
*/
int header_get_field_index(const struct header *h, const char *field) {
for (size_t i = 0; i < h->number_mask; i++) {
for (size_t i = 0; i < h->masks_count; i++) {
if (strcmp(h->masks[i].name, field) == 0) {
return i;
}
......@@ -135,10 +135,12 @@ void header_read(struct header *h, struct logger_logfile *log) {
error("Wrong offset value in the header (%i).", h->offset_direction);
/* Read offset to first record. */
h->offset_first_record = 0;
map = logger_loader_io_read_data(map, LOGGER_OFFSET_SIZE,
&h->offset_first_record);
/* Read the size of the strings. */
h->string_length = 0;
map =
logger_loader_io_read_data(map, sizeof(unsigned int), &h->string_length);
......@@ -148,13 +150,14 @@ void header_read(struct header *h, struct logger_logfile *log) {
}
/* Read the number of masks. */
map = logger_loader_io_read_data(map, sizeof(unsigned int), &h->number_mask);
map = logger_loader_io_read_data(map, sizeof(unsigned int), &h->masks_count);
/* Allocate the masks memory. */
h->masks = malloc(sizeof(struct mask_data) * h->number_mask);
h->masks = malloc(sizeof(struct mask_data) * h->masks_count);
/* Loop over all masks. */
for (size_t i = 0; i < h->number_mask; i++) {
h->timestamp_mask = 0;
for (size_t i = 0; i < h->masks_count; i++) {
/* Read the mask name. */
map = logger_loader_io_read_data(map, h->string_length, h->masks[i].name);
......@@ -164,6 +167,16 @@ void header_read(struct header *h, struct logger_logfile *log) {
/* Read the mask data size. */
map = logger_loader_io_read_data(map, sizeof(unsigned int),
&h->masks[i].size);
/* Keep the timestamp mask in memory */
if (strcmp(h->masks[i].name, "timestamp") == 0) {
h->timestamp_mask = h->masks[i].mask;
}
}
/* Check that the timestamp mask exists */
if (h->timestamp_mask == 0) {
error("Unable to find the timestamp mask.");
}
/* Check the logfile header's size. */
......@@ -187,7 +200,7 @@ size_t header_get_record_size_from_mask(const struct header *h,
const size_t mask) {
size_t count = 0;
/* Loop over each masks. */
for (size_t i = 0; i < h->number_mask; i++) {
for (size_t i = 0; i < h->masks_count; i++) {
if (mask & h->masks[i].mask) {
count += h->masks[i].size;
}
......
......@@ -24,9 +24,13 @@
#include <stdio.h>
#include <stdlib.h>
/* Number of character for the version information */
#define LOGGER_VERSION_SIZE 20
#define LOGGER_OFFSET_SIZE 7
#define LOGGER_MASK_SIZE 1
/* Number of bytes for the mask information in the headers
(need to be large enough for the larges mask) */
#define LOGGER_MASK_SIZE 2
/* Number of bytes for the offset size in the headers */
#define LOGGER_OFFSET_SIZE (8 - LOGGER_MASK_SIZE)
enum logger_offset_direction {
logger_offset_backward = 0,
......@@ -68,7 +72,7 @@ struct header {
unsigned int string_length;
/* Number of masks. */
unsigned int number_mask;
unsigned int masks_count;
/* List of masks. */
struct mask_data *masks;
......@@ -78,6 +82,9 @@ struct header {
/* The corresponding log. */
struct logger_logfile *log;
/* Timestamp mask */
size_t timestamp_mask;
};
void header_print(const struct header *h);
......
/*******************************************************************************
* This file is part of SWIFT.
* Copyright (c) 2019 Loic Hausammann (loic.hausammann@epfl.ch)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
******************************************************************************/
/* Include the corresponding header */
#include "logger_index.h"
/* Include the standard headers */
#include <errno.h>
#include <fcntl.h>
/* Include local headers */
#include "logger_loader_io.h"
#include "logger_reader.h"
#include "quick_sort.h"
/* Define the place and size of each element in the header. */
/* The time in double precision. */
#define logger_index_time_offset 0
#define logger_index_time_size sizeof(double)
/* The time on the integer timeline. */
#define logger_index_integer_time_offset \
logger_index_time_offset + logger_index_time_size
#define logger_index_integer_time_size sizeof(integertime_t)
/* The number of particle (for each type). */
#define logger_index_npart_offset \
logger_index_integer_time_offset + logger_index_integer_time_size
#define logger_index_npart_size sizeof(uint64_t) * swift_type_count
/* The flag used for checking if the file is sorted or not. */
#define logger_index_is_sorted_offset \
logger_index_npart_offset + logger_index_npart_size
#define logger_index_is_sorted_size sizeof(char)
/* The array containing the offset and ids. Rounding to a multiplier of 8 in
* order to align the memory */
#define logger_index_data_offset \
((logger_index_is_sorted_offset + logger_index_is_sorted_size + 7) & ~7)
/**
* @brief Read the index file header.
*
* @param index The #logger_index.
* @param filename The filename of the index file.
*/
void logger_index_read_header(struct logger_index *index,
const char *filename) {
/* Open the file */
message("Reading %s", filename);
logger_index_map_file(index, filename, 0);
/* Read times */
memcpy(&index->time, index->index.map + logger_index_time_offset,
logger_index_time_size);
memcpy(&index->integer_time,
index->index.map + logger_index_integer_time_offset,
logger_index_integer_time_size);
/* Read the number of particles */
memcpy(index->nparts, index->index.map + logger_index_npart_offset,
logger_index_npart_size);
/* Read if the file is sorted */
memcpy(&index->is_sorted, index->index.map + logger_index_is_sorted_offset,
logger_index_is_sorted_size);
/* Close the file */
logger_index_free(index);
}
/**
* @brief Write that the file is sorted.
*
* WARNING The file must be mapped.
*
* @param index The #logger_index.
*/
void logger_index_write_sorted(struct logger_index *index) {
/* Set the value */
const char is_sorted = 1;
/* Write the value */
memcpy(index->index.map + logger_index_is_sorted_offset, &is_sorted,
logger_index_is_sorted_size);
}
/**
* @brief Get the #index_data of a particle type.
*
* @param index The #logger_index.
* @param type The particle type.
*/
struct index_data *logger_index_get_data(struct logger_index *index, int type) {
/* Get the offset of particles of this type by skipping entries of lower
* types. */
size_t count = 0;
for (int i = 0; i < type; i++) {
count += index->nparts[i];
}
const size_t type_offset = count * sizeof(struct index_data);
return index->index.map + logger_index_data_offset + type_offset;
}
/**
* @brief Map the file and if required sort it.
*
* @param index The #logger_index.
* @param filename The index filename.
* @param sorted Does the file needs to be sorted?
*/
void logger_index_map_file(struct logger_index *index, const char *filename,
int sorted) {
/* Un-map previous file */
if (index->index.map != NULL) {
error("Trying to remap.");
}
/* Check if need to sort the file */
if (sorted && !index->is_sorted) {
if (index->reader->verbose > 0) {
message("Sorting the index file.");
}
/* Map the index file */
logger_loader_io_mmap_file(&index->index, filename,
/* read_only */ 0);
/* Sort the file */
for (int i = 0; i < swift_type_count; i++) {
if (index->nparts[i] != 0) {
struct index_data *data = logger_index_get_data(index, i);
quick_sort(data, index->nparts[i]);
}
}
/* Write that the file is sorted */
logger_index_write_sorted(index);
/* Free the index file before opening it again in read only */
logger_index_free(index);
if (index->reader->verbose > 0) {
message("Sorting done.");
}
}
/* Map the index file */
logger_loader_io_mmap_file(&index->index, filename,
/* read_only */ 1);
}
/**
* @brief Cleanup the memory of a logger_index
*
* @param index The #logger_index.
*/
void logger_index_free(struct logger_index *index) {
if (index->index.map == NULL) {
error("Trying to unmap an unexisting map");
}
logger_loader_io_munmap_file(&index->index);
}
/**
* @brief Get the offset of a given particle
*
* @param index The #logger_index.
* @param id The ID of the particle.
* @param type The type of the particle.
*
* @return The offset of the particle or 0 if not found.
*/
size_t logger_index_get_particle_offset(struct logger_index *index,
long long id, int type) {
/* Define a few variables */
const struct index_data *data = logger_index_get_data(index, type);
size_t left = 0;
size_t right = index->nparts[type] - 1;
/* Search for the value (binary search) */
while (left <= right) {
size_t m = (left + right) / 2;
if (data[m].id < id) {
left = m + 1;
} else if (data[m].id > id) {
right = m - 1;
} else {
return data[m].offset;
}
}
return 0;
}
/**
* @brief Initialize the #logger_index.
*
* @param index The #logger_index.
* @param reader The #logger_reader.
*/
void logger_index_init(struct logger_index *index,
struct logger_reader *reader) {
/* Set the mapped file to NULL */
index->index.map = NULL;
/* Set the pointer to the reader */
index->reader = reader;
/* Set the time to its default value */
index->time = -1;
}
/*******************************************************************************
* This file is part of SWIFT.
* Copyright (c) 2019 Loic Hausammann (loic.hausammann@epfl.ch)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
******************************************************************************/
#ifndef LOGGER_LOGGER_INDEX_H
#define LOGGER_LOGGER_INDEX_H
#include "logger_loader_io.h"
#include "logger_tools.h"
/* predefine the structure */
struct logger_reader;
/**
* @brief Data structure contained in the logger files.
*/
struct index_data {
/* Id of the particle. */
int64_t id;
/* Offset of the particle in the file. */
uint64_t offset;
};
/**
* @brief Structure dealing with the index files.
*
* The structure is initialized with #logger_index_init and
* then a file can be read with #logger_index_read_header and
* #logger_index_map_file.
*
* The functions #logger_index_get_particle_offset and
* #logger_index_get_data should be used to access the element
* stored inside the index file.
* The first one access a particle through its ids and the second one
* gives a pointer to the first element that can be looped through.
*/
struct logger_index {
/* Pointer to the reader */
struct logger_reader *reader;
/* Time of the index file */
double time;
/* Integer time of the index file */
integertime_t integer_time;
/* Number of particles in the file */
uint64_t nparts[swift_type_count];
/* Is the file sorted ? */
char is_sorted;
/* The mapped file */
struct mapped_file index;
};
void logger_index_write_sorted(struct logger_index *index);
void logger_index_init(struct logger_index *index,
struct logger_reader *reader);
void logger_index_read_header(struct logger_index *index, const char *filename);
void logger_index_map_file(struct logger_index *index, const char *filename,
int sorted);
size_t logger_index_get_particle_offset(struct logger_index *index,
long long id, int type);
void logger_index_free(struct logger_index *index);
void logger_index_sort_file(struct logger_index *index);
struct index_data *logger_index_get_data(struct logger_index *index, int type);
#endif // LOGGER_LOGGER_INDEX_H
......@@ -44,13 +44,13 @@ size_t logger_loader_io_get_file_size(int fd) {
*
* #logger_loader_io_munmap_file should be called to unmap the file.
*
* @param map The #mapped_file.
* @param filename file to read.
* @param file_size (out) size of the file.
* @param read_only Open the file in read only mode?
*
*/
void *logger_loader_io_mmap_file(char *filename, size_t *file_size,
int read_only) {
void logger_loader_io_mmap_file(struct mapped_file *map, const char *filename,
int read_only) {
/* open the file. */
int fd;
......@@ -63,33 +63,34 @@ void *logger_loader_io_mmap_file(char *filename, size_t *file_size,
error("Unable to open file %s (%s).", filename, strerror(errno));
/* get the file size. */
*file_size = logger_loader_io_get_file_size(fd);
map->mmap_size = logger_loader_io_get_file_size(fd);
/* map the memory. */
int mode = PROT_READ;
if (!read_only) mode |= PROT_WRITE;
void *map = mmap(NULL, *file_size, mode, MAP_SHARED, fd, 0);
if (map == MAP_FAILED)
error("Failed to allocate map of size %zi bytes (%s).", *file_size,
map->map = mmap(NULL, map->mmap_size, mode, MAP_SHARED, fd, 0);
if (map->map == MAP_FAILED)
error("Failed to allocate map of size %zi bytes (%s).", map->mmap_size,
strerror(errno));
/* Close the file. */
close(fd);
return map;
}
/**
* @brief Unmap a file.
*
* @param map file mapping.
* @param file_size The file size.
* @param map The #mapped_file.
*
*/
void logger_loader_io_munmap_file(void *map, size_t file_size) {
void logger_loader_io_munmap_file(struct mapped_file *map) {
/* unmap the file. */
if (munmap(map, file_size) != 0) {
if (munmap(map->map, map->mmap_size) != 0) {
error("Unable to unmap the file (%s).", strerror(errno));
}
/* Reset values */
map->map = NULL;
map->mmap_size = 0;
}
......@@ -29,10 +29,19 @@
#include <stdio.h>
#include <stdlib.h>
/* Structure for mapping a file. */
struct mapped_file {
/* Mapped data. */
void *map;
/* File size. */
size_t mmap_size;
};
size_t logger_loader_io_get_file_size(int fd);
void *logger_loader_io_mmap_file(char *filename, size_t *file_size,
int read_only);
void logger_loader_io_munmap_file(void *map, size_t file_size);
void logger_loader_io_mmap_file(struct mapped_file *map, const char *filename,
int read_only);
void logger_loader_io_munmap_file(struct mapped_file *map);
/**
* @brief read a mask with its offset.
......
......@@ -42,8 +42,8 @@ void logger_logfile_init_from_file(struct logger_logfile *log, char *filename,
/* Open file, map it and get its size. */
if (reader->verbose > 1) message("Mapping the log file.");
log->log.map = logger_loader_io_mmap_file(filename, &log->log.file_size,
/* read_only */ 1);
logger_loader_io_mmap_file<