Commit 087254a4 authored by Loic Hausammann's avatar Loic Hausammann

Initial commit for logger loader

parent 2cf116fc
......@@ -23,6 +23,9 @@ SUBDIRS = src argparse examples doc tests tools
if HAVEEAGLECOOLING
SUBDIRS += examples/Cooling/CoolingRates
endif
if HAVELOGGER
SUBDIRS += logger
endif
# Non-standard files that should be part of the distribution.
EXTRA_DIST = INSTALL.swift .clang-format format.sh
......@@ -86,6 +86,7 @@ AC_ARG_ENABLE([logger],
if test "$with_logger" = "yes"; then
AC_DEFINE([WITH_LOGGER], 1, [logger enabled])
fi
AM_CONDITIONAL([HAVELOGGER],[test $with_logger = "yes"])
# Interprocedural optimization support. Needs special handling for linking and
# archiving as well as compilation with Intels, needs to be done before
......@@ -996,6 +997,50 @@ fi
AC_SUBST([TBBMALLOC_LIBS])
AM_CONDITIONAL([HAVETBBMALLOC],[test -n "$TBBMALLOC_LIBS"])
# Check for python.
have_python="no"
AC_ARG_WITH([python],
[AS_HELP_STRING([--with-python=PATH],
[root directory where python is installed @<:@yes/no@:>@]
)],
[with_python="$withval"],
[with_python="no"]
)
if test "x$with_python" != "xno"; then
AC_CACHE_CHECK(
"python version",
ac_cv_ver_python,
[ac_cv_ver_python=`python -c 'import sys;print(sys.version[[:3]])' 2> /dev/null`])
if test "x$with_python" != "xyes" -a "x$with_python" != "x"; then
PYTHON_LIBS=""
PYTHON_INCS="-I$with_python/include/python$ac_cv_ver_python"
else
PYTHON_LIBS=""
PYTHON_INCS=""
fi
have_python="yes"
AC_CHECK_PROGS(
[PYTHON_BIN],
[python$ac_cv_ver_python],
[AC_MSG_ERROR(Cannot find python binary!)],
[$with_python/bin]
)
AC_CHECK_LIB(
[python${ac_cv_ver_python}m],
[PyArg_ParseTuple],
[AC_DEFINE([HAVE_PYTHON],1,[The python library appears to be present.])
],
[AC_MSG_ERROR(Cannot find python library!)]
)
fi
AC_SUBST([PYTHON_LIBS])
AC_SUBST([PYTHON_INCS])
AM_CONDITIONAL([HAVEPYTHON],[test -n "$PYTHON_LIBS"])
# Check for HDF5. This is required.
AX_LIB_HDF5
if test "$with_hdf5" != "yes"; then
......@@ -1991,7 +2036,7 @@ AM_CONDITIONAL([HAVEEAGLEFEEDBACK], [test $with_feedback = "EAGLE"])
# Handle .in files.
AC_CONFIG_FILES([Makefile src/Makefile examples/Makefile examples/Cooling/CoolingRates/Makefile doc/Makefile doc/Doxyfile tests/Makefile])
AC_CONFIG_FILES([argparse/Makefile tools/Makefile])
AC_CONFIG_FILES([argparse/Makefile tools/Makefile logger/Makefile])
AC_CONFIG_FILES([tests/testReading.sh], [chmod +x tests/testReading.sh])
AC_CONFIG_FILES([tests/testActivePair.sh], [chmod +x tests/testActivePair.sh])
AC_CONFIG_FILES([tests/test27cells.sh], [chmod +x tests/test27cells.sh])
......@@ -2046,6 +2091,7 @@ AC_MSG_RESULT([
VELOCIraptor enabled : $have_velociraptor
Particle Logger : $with_logger
FoF activated: : $enable_fof
Python enabled : $have_python
Hydro scheme : $with_hydro
Dimensionality : $with_dimension
......
# This file is part of SWIFT.
# Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk),
# Matthieu Schaller (matthieu.schaller@durham.ac.uk).
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Add the non-standard paths to the included library headers
AM_CFLAGS = $(PYTHON_INCS)
# Assign a "safe" version number
AM_LDFLAGS = -version-info 0:0:0
# The git command, if available.
GIT_CMD = @GIT_CMD@
# Additional dependencies for shared libraries.
EXTRA_LIBS = $(PROFILER_LIBS) $(TCMALLOC_LIBS) $(JEMALLOC_LIBS) $(TBBMALLOC_LIBS) $(PYTHON_LIBS)
# MPI libraries.
# MPI_LIBS = $(MPI_THREAD_LIBS)
# MPI_FLAGS = -DWITH_MPI
# Build the libswiftsim library
lib_LTLIBRARIES = libswiftlogger.la
# Build a MPI-enabled version too?
# if HAVEMPI
# lib_LTLIBRARIES += libswiftlogger_mpi.la
# endif
# List required headers
include_HEADERS = header.h io.h particle.h timeline.h tools.h
# Common source files
AM_SOURCES = header.c io.c logger_loader.c particle.c \
timeline.c tools.c
# Include files for distribution, not installation.
nobase_noinst_HEADERS =
# Sources and flags for regular library
libswiftlogger_la_SOURCES = $(AM_SOURCES)
libswiftlogger_la_CFLAGS = $(AM_CFLAGS)
libswiftlogger_la_LDFLAGS = $(AM_LDFLAGS) $(EXTRA_LIBS)
# Sources and flags for MPI library
# libswiftlogger_mpi_la_SOURCES = $(AM_SOURCES)
# libswiftlogger_mpi_la_CFLAGS = $(AM_CFLAGS) $(MPI_FLAGS)
# libswiftlogger_mpi_la_LDFLAGS = $(AM_LDFLAGS) $(MPI_LIBS) $(EXTRA_LIBS)
# libswiftlogger_mpi_la_SHORTNAME = mpi
# libswiftlogger_mpi_la_LIBADD =
# Versioning. If any sources change then update the version_string.h file with
# the current git revision and package version.
# May have a checkout without a version_string.h file and no git command (tar/zip
# download), allow that, but make sure we know it.
version_string.h: ../src/version_string.h.in $(AM_SOURCES) $(include_HEADERS) $(noinst_HEADERS)
if test "X$(GIT_CMD)" != "X"; then \
GIT_REVISION=`$(GIT_CMD) describe --abbrev=8 --always --tags --dirty`; \
GIT_BRANCH=`$(GIT_CMD) branch | sed -n 's/^\* \(.*\)/\1/p'`; \
GIT_DATE=`$(GIT_CMD) log -1 --format=%ci`; \
sed -e "s,@PACKAGE_VERSION\@,$(PACKAGE_VERSION)," \
-e "s,@GIT_REVISION\@,$${GIT_REVISION}," \
-e "s|@GIT_BRANCH\@|$${GIT_BRANCH}|" \
-e "s|@GIT_DATE\@|$${GIT_DATE}|" \
-e "s|@SWIFT_CFLAGS\@|$(CFLAGS)|" $< > version_string.h; \
else \
if test ! -f version_string.h; then \
sed -e "s,@PACKAGE_VERSION\@,$(PACKAGE_VERSION)," \
-e "s,@GIT_REVISION\@,unknown," \
-e "s,@GIT_BRANCH\@,unknown," \
-e "s,@GIT_DATE\@,unknown," \
-e "s|@SWIFT_CFLAGS\@|$(CFLAGS)|" $< > version_string.h; \
fi; \
fi
# Make sure version_string.h is built first.
BUILT_SOURCES = version_string.h
# And distribute the built files.
EXTRA_DIST = version_string.h ../src/version_string.h.in
#include "header.h"
#include "io.h"
#include "tools.h"
#include <Python.h>
#include <stdio.h>
#include <stdlib.h>
void header_print(const struct header *h) {
#ifdef SWIFT_DEBUG_CHECKS
printf("Debug checks enabled\n");
#endif
printf("Version: %s\n", h->version);
printf("First Offset: %lu\n", h->offset_first);
char direction[20];
if (h->forward_offset)
strcpy(direction, "Forward");
else
strcpy(direction, "Backward");
printf("Offset direction: %s\n", direction);
printf("Number masks: %lu\n", h->nber_mask);
for (size_t i = 0; i < h->nber_mask; i++) {
printf("\tMask: %s\n", h->masks_name[i]);
printf("\tValue: %lu\n", h->masks[i]);
printf("\tSize: %lu\n", h->masks_size[i]);
printf("\n");
}
/* mask contains... TODO */
};
void header_free(struct header *h) {
for (size_t i = 0; i < h->nber_mask; i++) {
free(h->masks_name[i]);
}
free(h->masks_name);
free(h->masks);
free(h->masks_size);
};
int header_is_present_and_get_index(const struct header *h, const char *field,
size_t *ind) {
for (size_t i = 0; i < h->nber_mask; i++) {
if (strcmp(h->masks_name[i], field) == 0) {
if (ind != NULL) {
*ind = i;
}
return 1;
}
}
return 0;
};
int header_is_present(const struct header *h, const char *field) {
return header_is_present_and_get_index(h, field, NULL);
};
int header_change_offset_direction(struct header *h, void *map) {
h->forward_offset = !h->forward_offset;
size_t offset = LOGGER_VERSION_SIZE;
return io_write_data(map, LOGGER_NBER_SIZE, &h->forward_offset,
&offset);
}
int header_read(struct header *h, void *map) {
size_t offset = 0;
/* read version */
io_read_data(map, LOGGER_VERSION_SIZE, &h->version, &offset);
/* read offset direction */
h->forward_offset = 0;
io_read_data(map, LOGGER_NBER_SIZE, &h->forward_offset, &offset);
if (h->forward_offset != 0 && h->forward_offset != 1)
error(EIO, "Non boolean value for the offset direction (%i)", h->forward_offset);
/* read offset to first data */
h->offset_first = 0;
io_read_data(map, LOGGER_OFFSET_SIZE, &h->offset_first, &offset);
/* read name size */
h->name = 0;
io_read_data(map, LOGGER_NBER_SIZE, &h->name, &offset);
/* check if value defined in this file is large enough */
if (STRING_SIZE < h->name) {
error(EOVERFLOW, "Name too large in dump file");
}
/* read number of masks */
h->nber_mask = 0;
io_read_data(map, LOGGER_NBER_SIZE, &h->nber_mask, &offset);
/* allocate memory */
h->masks = malloc(sizeof(size_t) * h->nber_mask);
h->masks_name = malloc(sizeof(void *) * h->nber_mask);
h->masks_size = malloc(sizeof(size_t) * h->nber_mask);
/* loop over all masks */
for (size_t i = 0; i < h->nber_mask; i++) {
/* read mask name */
h->masks_name[i] = malloc(h->name);
io_read_data(map, h->name, h->masks_name[i], &offset);
/* get mask value */
h->masks[i] = 1 << i;
/* read mask data size */
h->masks_size[i] = 0;
io_read_data(map, LOGGER_NBER_SIZE, &h->masks_size[i], &offset);
}
if (offset != h->offset_first) {
#ifdef SWIFT_DEBUG_CHECKS
header_print(h);
#endif
error(EIO, "Wrong header size (in header %li, current %li)",
h->offset_first, offset);
}
return 0;
};
size_t header_get_mask_size(const struct header *h, const size_t mask) {
size_t count = 0;
for (size_t i = 0; i < h->nber_mask; i++) {
if (mask & h->masks[i]) {
count += h->masks_size[i];
}
}
return count;
}
#ifndef __LOGGER_HEADER_H__
#define __LOGGER_HEADER_H__
#include "tools.h"
#include <Python.h>
#include <stdio.h>
#include <stdlib.h>
#define LOGGER_VERSION_SIZE 20
#define LOGGER_NBER_SIZE 4
#define LOGGER_OFFSET_SIZE 7
#define LOGGER_MASK_SIZE 1
struct header {
/* Logger version */
char version[STRING_SIZE];
/* offset of the first header */
size_t offset_first;
/* Number of bytes for names */
size_t name;
/* number of masks */
size_t nber_mask;
/* list of masks */
size_t *masks;
/* list of mask name */
char **masks_name;
/* size of data in a mask */
size_t *masks_size;
/* offset direction */
int forward_offset;
};
/**
* @brief print a header struct
*/
void header_print(const struct header *h);
/**
* @brief free allocated memory
*/
void header_free(struct header *h);
/**
* @brief check if field is present in header
*
* @param field name of the requested field
* @param ind (return value) indice of the requested field
*/
int header_is_present_and_get_index(const struct header *h, const char *field,
size_t *ind);
/**
* @brief check if field is present in header
*
* @param field name of the requested field
*/
int header_is_present(const struct header *h, const char *field);
/**
* @brief read the logger header
*
* @param h out: header
* @param map file mapping
*/
int header_read(struct header *h, void *map);
/**
* @brief count number of bits in a given mask
*
* @param h #header file structure
* @param mask mask to compute
*
* @return number of bits in mask
*/
size_t header_get_mask_size(const struct header *h, const size_t mask);
/**
* @brief Inverse the offset direction
*
* @param h #header file structure
* @param map file mapping
*
* @return error code
*/
int header_change_offset_direction(struct header *h, void *map);
#endif // __LOGGER_HEADER_H__
#include "io.h"
#include "header.h"
#include "tools.h"
/* file size */
#include <sys/stat.h>
/* mapping */
#include <sys/mman.h>
/* open */
#include <fcntl.h>
int io_get_file_size(int fd, size_t *size) {
struct stat s;
int status = fstat(fd, &s);
if (status != 0) error(errno, "Unable to get file size");
*size = s.st_size;
return 0;
}
int io_open_file(char *filename, int *fd, void **map) {
/* open file */
*fd = open(filename, O_RDWR);
if (*fd == -1) error(errno, "Unable to open file %s", filename);
/* get file size */
size_t size;
int status = io_get_file_size(*fd, &size);
if (status != 0) return status;
/* map memory */
*map = mmap(NULL, size, PROT_WRITE | PROT_READ, MAP_SHARED, *fd, 0);
if (map == MAP_FAILED)
error(errno, "Failed to allocate map of size %zi bytes.", size);
return 0;
}
int io_close_file(int *fd, void **map) {
/* get file size */
size_t size;
int status = io_get_file_size(*fd, &size);
if (status != 0) return status;
/* unmap */
if (munmap(*map, size) != 0) error(errno, "Unable to unmap the file");
close(*fd);
return 0;
}
int io_read_mask(const struct header *h, void *map, size_t *offset,
size_t *mask, size_t *diff_offset) {
/* read mask */
if (mask) {
*mask = 0;
memcpy(mask, map + *offset, LOGGER_MASK_SIZE);
}
*offset += LOGGER_MASK_SIZE;
/* read offset */
if (diff_offset) {
*diff_offset = 0;
memcpy(diff_offset, map + *offset, LOGGER_OFFSET_SIZE);
}
*offset += LOGGER_OFFSET_SIZE;
return 0;
}
int io_read_data(void *map, const size_t size, void *p, size_t *offset) {
memcpy(p, map + *offset, size);
*offset += size;
return 0;
};
int io_write_data(void *map, const size_t size, const void *p, size_t *offset) {
memcpy(map + *offset, p, size);
*offset += size;
return 0;
};
#ifndef __SWIFT_LOGGER_IO_H__
#define __SWIFT_LOGGER_IO_H__
#include "header.h"
#include "tools.h"
#include <stdio.h>
#include <stdlib.h>
/**
* @brief get the size of a file
*
* @param fd file id
* @param size out: file size
*
* @return error code
*/
int io_get_file_size(int fd, size_t *size);
/**
* @brief Open a file and map it
*
* @param filename file to read
* @param fd out: file id
* @param map out: file mapping
*
* @return error code
*/
int io_open_file(char *filename, int *fd, void **map);
/**
* @brief Close a file and unmap it
*
* @param fd file id
* @param map file mapping
*
* @return error code
*/
int io_close_file(int *fd, void **map);
/**
* @brief read a single value in a file
*
* @param map file mapping
* @param size size of the chunk to read
* @param p pointer where to store the data
* @param offset In: position to read, Out: shifted by size
*
* @return error code
*/
int io_read_data(void *map, const size_t size, void *p, size_t *offset);
/**
* @brief write a single value in a file
*
* @param map file mapping
* @param size size of the chunk to write
* @param p pointer to the data
* @param offset In: position to write, Out: shifted by size
*
* @return error code
*/
int io_write_data(void *map, const size_t size, const void *p, size_t *offset);
/**
* @brief read a maks with its offset
*
* @param h #header file structure
* @param map file mapping
* @param offset In: position in the file, Out: shifted by the mask + offset
* size
* @param mask mask read
* @param diff_offset offset difference to previous/next corresponding chunk
*
* @return error code
*/
int io_read_mask(const struct header *h, void *map, size_t *offset,
size_t *mask, size_t *diff_offset);
#endif // __SWIFT_LOGGER_IO_H__
#include "header.h"
#include "io.h"
#include "particle.h"
#include "timeline.h"
#include <Python.h>
#include <errno.h>
#include <numpy/arrayobject.h>
#include <stdio.h>
#include <stdlib.h>
/**
* @brief Reverse offset in dump file
*
* @param filename string filename of the dump file
*/
int reverseOffset(char *filename) {
struct header h;
/* open file */
int fd;
void *map;
if (io_open_file(filename, &fd, &map) != 0) return 1;
/* read header */
if (header_read(&h, map) != 0) return 1;
header_print(&h);
/* check offset direction */
if (h.forward_offset) {
error_no_return(EIO, "Offset are already reversed");
return 1;
}
/* compute file size */
size_t sz;
int status = io_get_file_size(fd, &sz);
if (status != 0) return 1;
size_t offset;
int error_code;
#ifdef SWIFT_DEBUG_CHECKS
/* check offset */
printf("Check offsets...\n");
offset = h.offset_first;
while (offset < sz) {
error_code = tools_check_offset(&h, map, &offset);
if (error_code != 0) return 1;
}
printf("Check done\n");
#endif
/* reverse header offset */
header_change_offset_direction(&h, map);
offset = h.offset_first;
/* reverse chunks */
printf("Reversing offsets...\n");
while (offset < sz) {
error_code = tools_reverse_offset(&h, map, &offset);
if (error_code != 0) return 1;
}
printf("Reversing done\n");
#ifdef SWIFT_DEBUG_CHECKS
/* check offset */
printf("Check offsets...\n");
offset = h.offset_first;
while (offset < sz) {
error_code = tools_check_offset(&h, map, &offset);
if (error_code != 0) return 1;
}
printf("Check done\n");
#endif
/* free internal variables */
header_free(&h);
if (io_close_file(&fd, &map) != 0) return 1;
return 0;
}
/**
* @brief load data from the offset without any interpolation
*
* @param offset PyArrayObject list of offset for each particle
* @param filename string filename of the dump file
* @return dictionnary containing the data read
*/
static PyObject *loadFromIndex(__attribute__((unused)) PyObject *self,
PyObject *args) {
struct header h;
/* input */
PyArrayObject *offset = NULL;
char *filename = NULL;
/* output */
PyArrayObject *pos = NULL;
PyArrayObject *vel = NULL;
PyArrayObject *acc = NULL;
PyArrayObject *entropy = NULL;
PyArrayObject *h_sph = NULL;
PyArrayObject *rho = NULL;
PyArrayObject *mass = NULL;
PyArrayObject *id = NULL;
size_t time_offset;
/* parse arguments */
if (!PyArg_ParseTuple(args, "OsL", &offset, &filename, &time_offset))
return NULL;
if (!PyArray_Check(offset)) {
error_no_return(ENOTSUP, "Offset is not a numpy array");
return NULL;
}
if (PyArray_NDIM(offset) != 1) {
error_no_return(ENOTSUP, "Offset is not a 1 dimensional array");
return NULL;
}
if (PyArray_TYPE(offset) != NPY_UINT64) {
error_no_return(ENOTSUP, "Offset does not contain unsigned int");
return NULL;
}
/* open file */
int fd;
void *map;
if (io_open_file(filename, &fd, &map) != 0) return NULL;
/* read header */
if (header_read(&h, map) != 0) return NULL;
/* reverse offset if needed */
if (!h.forward_offset) {
if (io_close_file(&fd, &map) != 0) return NULL;
if (reverseOffset(filename) != 0) return NULL;
if (io_open_file(filename, &fd, &map) != 0) return NULL;
/* Reset header */
header_free(&h);
if (header_read(&h, map) != 0) return NULL;
}
/* read timestamps */
struct time_array times;
if (time_array_init(&times, &h, map, fd) != 0) return NULL;
time_array_print(&times);
/* get required time */