Initial commit for logger loader

Makefile.am

@@ -23,6 +23,9 @@ SUBDIRS = src argparse examples doc tests tools
 if HAVEEAGLECOOLING
 SUBDIRS += examples/Cooling/CoolingRates
 endif
+if HAVELOGGER
+SUBDIRS += logger
+endif
 
 # Non-standard files that should be part of the distribution.
 EXTRA_DIST = INSTALL.swift .clang-format format.sh

configure.ac

@@ -86,6 +86,7 @@ AC_ARG_ENABLE([logger],
 if test "$with_logger" = "yes"; then
    AC_DEFINE([WITH_LOGGER], 1, [logger enabled])
 fi
+AM_CONDITIONAL([HAVELOGGER],[test $with_logger = "yes"])
 
 # Interprocedural optimization support. Needs special handling for linking and
 # archiving as well as compilation with Intels, needs to be done before
@@ -996,6 +997,50 @@ fi
 AC_SUBST([TBBMALLOC_LIBS])
 AM_CONDITIONAL([HAVETBBMALLOC],[test -n "$TBBMALLOC_LIBS"])
 
+# Check for python.
+have_python="no"
+AC_ARG_WITH([python],
+    [AS_HELP_STRING([--with-python=PATH],
+       [root directory where python is installed @<:@yes/no@:>@]
+    )],
+    [with_python="$withval"],
+    [with_python="no"]
+)
+if test "x$with_python" != "xno"; then
+   AC_CACHE_CHECK(
+   "python version",
+   ac_cv_ver_python,
+   [ac_cv_ver_python=`python -c 'import sys;print(sys.version[[:3]])' 2> /dev/null`])
+   
+   if test "x$with_python" != "xyes" -a "x$with_python" != "x"; then
+      PYTHON_LIBS=""
+      PYTHON_INCS="-I$with_python/include/python$ac_cv_ver_python"
+   else
+      PYTHON_LIBS=""
+      PYTHON_INCS=""
+   fi
+   have_python="yes"
+
+   AC_CHECK_PROGS(
+	[PYTHON_BIN],
+	[python$ac_cv_ver_python],
+	[AC_MSG_ERROR(Cannot find python binary!)],
+	[$with_python/bin]
+	)
+
+   AC_CHECK_LIB(
+      [python${ac_cv_ver_python}m],
+      [PyArg_ParseTuple],
+      [AC_DEFINE([HAVE_PYTHON],1,[The python library appears to be present.])
+      ],
+      [AC_MSG_ERROR(Cannot find python library!)]
+      )
+fi
+AC_SUBST([PYTHON_LIBS])
+AC_SUBST([PYTHON_INCS])
+AM_CONDITIONAL([HAVEPYTHON],[test -n "$PYTHON_LIBS"])
+
+
 # Check for HDF5. This is required.
 AX_LIB_HDF5
 if test "$with_hdf5" != "yes"; then
@@ -1991,7 +2036,7 @@ AM_CONDITIONAL([HAVEEAGLEFEEDBACK], [test $with_feedback = "EAGLE"])
 
 # Handle .in files.
 AC_CONFIG_FILES([Makefile src/Makefile examples/Makefile examples/Cooling/CoolingRates/Makefile doc/Makefile doc/Doxyfile tests/Makefile])
-AC_CONFIG_FILES([argparse/Makefile tools/Makefile])
+AC_CONFIG_FILES([argparse/Makefile tools/Makefile logger/Makefile])
 AC_CONFIG_FILES([tests/testReading.sh], [chmod +x tests/testReading.sh])
 AC_CONFIG_FILES([tests/testActivePair.sh], [chmod +x tests/testActivePair.sh])
 AC_CONFIG_FILES([tests/test27cells.sh], [chmod +x tests/test27cells.sh])
@@ -2046,6 +2091,7 @@ AC_MSG_RESULT([
    VELOCIraptor enabled : $have_velociraptor
    Particle Logger      : $with_logger
    FoF activated:       : $enable_fof
+   Python enabled       : $have_python
 
    Hydro scheme       : $with_hydro
    Dimensionality     : $with_dimension

logger/Makefile.am

+# This file is part of SWIFT.
+# Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk),
+#                    Matthieu Schaller (matthieu.schaller@durham.ac.uk).
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Add the non-standard paths to the included library headers
+AM_CFLAGS = $(PYTHON_INCS)
+
+# Assign a "safe" version number
+AM_LDFLAGS = -version-info 0:0:0
+
+# The git command, if available.
+GIT_CMD = @GIT_CMD@
+
+# Additional dependencies for shared libraries.
+EXTRA_LIBS = $(PROFILER_LIBS) $(TCMALLOC_LIBS) $(JEMALLOC_LIBS) $(TBBMALLOC_LIBS) $(PYTHON_LIBS)
+
+# MPI libraries.
+# MPI_LIBS = $(MPI_THREAD_LIBS)
+# MPI_FLAGS = -DWITH_MPI
+
+# Build the libswiftsim library
+lib_LTLIBRARIES = libswiftlogger.la
+# Build a MPI-enabled version too?
+# if HAVEMPI
+# lib_LTLIBRARIES += libswiftlogger_mpi.la
+# endif
+
+# List required headers
+include_HEADERS = header.h io.h particle.h timeline.h tools.h
+
+# Common source files
+AM_SOURCES = header.c io.c logger_loader.c particle.c \
+	timeline.c tools.c
+
+# Include files for distribution, not installation.
+nobase_noinst_HEADERS = 
+
+# Sources and flags for regular library
+libswiftlogger_la_SOURCES = $(AM_SOURCES)
+libswiftlogger_la_CFLAGS = $(AM_CFLAGS)
+libswiftlogger_la_LDFLAGS = $(AM_LDFLAGS) $(EXTRA_LIBS)
+
+# Sources and flags for MPI library
+# libswiftlogger_mpi_la_SOURCES = $(AM_SOURCES)
+# libswiftlogger_mpi_la_CFLAGS = $(AM_CFLAGS) $(MPI_FLAGS)
+# libswiftlogger_mpi_la_LDFLAGS = $(AM_LDFLAGS) $(MPI_LIBS) $(EXTRA_LIBS)
+# libswiftlogger_mpi_la_SHORTNAME = mpi
+# libswiftlogger_mpi_la_LIBADD =
+
+
+# Versioning. If any sources change then update the version_string.h file with
+# the current git revision and package version.
+# May have a checkout without a version_string.h file and no git command (tar/zip
+# download), allow that, but make sure we know it.
+version_string.h: ../src/version_string.h.in $(AM_SOURCES) $(include_HEADERS) $(noinst_HEADERS)
+	if test "X$(GIT_CMD)" != "X"; then \
+	    GIT_REVISION=`$(GIT_CMD) describe --abbrev=8  --always --tags --dirty`; \
+	    GIT_BRANCH=`$(GIT_CMD) branch | sed -n 's/^\* \(.*\)/\1/p'`; \
+            GIT_DATE=`$(GIT_CMD) log -1 --format=%ci`; \
+	    sed -e "s,@PACKAGE_VERSION\@,$(PACKAGE_VERSION)," \
+	        -e "s,@GIT_REVISION\@,$${GIT_REVISION}," \
+	        -e "s|@GIT_BRANCH\@|$${GIT_BRANCH}|" \
+	        -e "s|@GIT_DATE\@|$${GIT_DATE}|" \
+	        -e "s|@SWIFT_CFLAGS\@|$(CFLAGS)|" $< > version_string.h; \
+	else \
+	    if test ! -f version_string.h; then \
+	        sed -e "s,@PACKAGE_VERSION\@,$(PACKAGE_VERSION)," \
+	            -e "s,@GIT_REVISION\@,unknown," \
+		    -e "s,@GIT_BRANCH\@,unknown," \
+		    -e "s,@GIT_DATE\@,unknown," \
+	            -e "s|@SWIFT_CFLAGS\@|$(CFLAGS)|" $< > version_string.h; \
+	    fi; \
+	fi
+
+#  Make sure version_string.h is built first.
+BUILT_SOURCES = version_string.h
+
+#  And distribute the built files.
+EXTRA_DIST = version_string.h ../src/version_string.h.in

logger/header.c

+#include "header.h"
+
+#include "io.h"
+#include "tools.h"
+
+#include <Python.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+void header_print(const struct header *h) {
+#ifdef SWIFT_DEBUG_CHECKS
+  printf("Debug checks enabled\n");
+#endif
+  printf("Version:          %s\n", h->version);
+  printf("First Offset:     %lu\n", h->offset_first);
+  char direction[20];
+  if (h->forward_offset)
+    strcpy(direction, "Forward");
+  else
+    strcpy(direction, "Backward");
+  printf("Offset direction: %s\n", direction);
+  printf("Number masks:     %lu\n", h->nber_mask);
+
+  for (size_t i = 0; i < h->nber_mask; i++) {
+    printf("\tMask:  %s\n", h->masks_name[i]);
+    printf("\tValue: %lu\n", h->masks[i]);
+    printf("\tSize:  %lu\n", h->masks_size[i]);
+    printf("\n");
+  }
+
+  /* mask contains... TODO */
+
+};
+
+void header_free(struct header *h) {
+  for (size_t i = 0; i < h->nber_mask; i++) {
+    free(h->masks_name[i]);
+  }
+  free(h->masks_name);
+  free(h->masks);
+  free(h->masks_size);
+};
+
+int header_is_present_and_get_index(const struct header *h, const char *field,
+                                    size_t *ind) {
+  for (size_t i = 0; i < h->nber_mask; i++) {
+    if (strcmp(h->masks_name[i], field) == 0) {
+      if (ind != NULL) {
+        *ind = i;
+      }
+      return 1;
+    }
+  }
+
+  return 0;
+};
+
+int header_is_present(const struct header *h, const char *field) {
+  return header_is_present_and_get_index(h, field, NULL);
+};
+
+int header_change_offset_direction(struct header *h, void *map) {
+  h->forward_offset = !h->forward_offset;
+  size_t offset = LOGGER_VERSION_SIZE;
+
+  return io_write_data(map, LOGGER_NBER_SIZE, &h->forward_offset,
+                       &offset);
+}
+
+int header_read(struct header *h, void *map) {
+  size_t offset = 0;
+
+  /* read version */
+  io_read_data(map, LOGGER_VERSION_SIZE, &h->version, &offset);
+
+  /* read offset direction */
+  h->forward_offset = 0;
+  io_read_data(map, LOGGER_NBER_SIZE, &h->forward_offset, &offset);
+
+  if (h->forward_offset != 0 && h->forward_offset != 1)
+    error(EIO, "Non boolean value for the offset direction (%i)", h->forward_offset);
+
+  /* read offset to first data */
+  h->offset_first = 0;
+  io_read_data(map, LOGGER_OFFSET_SIZE, &h->offset_first, &offset);
+
+  /* read name size */
+  h->name = 0;
+  io_read_data(map, LOGGER_NBER_SIZE, &h->name, &offset);
+
+  /* check if value defined in this file is large enough */
+  if (STRING_SIZE < h->name) {
+    error(EOVERFLOW, "Name too large in dump file");
+  }
+
+  /* read number of masks */
+  h->nber_mask = 0;
+  io_read_data(map, LOGGER_NBER_SIZE, &h->nber_mask, &offset);
+
+  /* allocate memory */
+  h->masks = malloc(sizeof(size_t) * h->nber_mask);
+  h->masks_name = malloc(sizeof(void *) * h->nber_mask);
+  h->masks_size = malloc(sizeof(size_t) * h->nber_mask);
+
+  /* loop over all masks */
+  for (size_t i = 0; i < h->nber_mask; i++) {
+    /* read mask name */
+    h->masks_name[i] = malloc(h->name);
+    io_read_data(map, h->name, h->masks_name[i], &offset);
+
+    /* get mask value */
+    h->masks[i] = 1 << i;
+
+    /* read mask data size */
+    h->masks_size[i] = 0;
+    io_read_data(map, LOGGER_NBER_SIZE, &h->masks_size[i], &offset);
+  }
+
+  if (offset != h->offset_first) {
+#ifdef SWIFT_DEBUG_CHECKS
+    header_print(h);
+#endif
+    error(EIO, "Wrong header size (in header %li, current %li)",
+	  h->offset_first, offset);
+  }
+
+  return 0;
+};
+
+size_t header_get_mask_size(const struct header *h, const size_t mask) {
+  size_t count = 0;
+  for (size_t i = 0; i < h->nber_mask; i++) {
+    if (mask & h->masks[i]) {
+      count += h->masks_size[i];
+    }
+  }
+  return count;
+}

logger/header.h

+#ifndef __LOGGER_HEADER_H__
+#define __LOGGER_HEADER_H__
+
+#include "tools.h"
+
+#include <Python.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#define LOGGER_VERSION_SIZE 20
+#define LOGGER_NBER_SIZE 4
+#define LOGGER_OFFSET_SIZE 7
+#define LOGGER_MASK_SIZE 1
+
+struct header {
+  /* Logger version */
+  char version[STRING_SIZE];
+
+  /* offset of the first header */
+  size_t offset_first;
+
+  /* Number of bytes for names */
+  size_t name;
+  
+  /* number of masks */
+  size_t nber_mask;
+
+  /* list of masks */
+  size_t *masks;
+
+  /* list of mask name */
+  char **masks_name;
+
+  /* size of data in a mask */
+  size_t *masks_size;
+
+  /* offset direction */
+  int forward_offset;
+};
+
+/**
+ * @brief print a header struct
+ */
+void header_print(const struct header *h);
+
+/**
+ * @brief free allocated memory
+ */
+void header_free(struct header *h);
+
+/**
+ * @brief check if field is present in header
+ *
+ * @param field name of the requested field
+ * @param ind (return value) indice of the requested field
+ */
+int header_is_present_and_get_index(const struct header *h, const char *field,
+                                    size_t *ind);
+
+/**
+ * @brief check if field is present in header
+ *
+ * @param field name of the requested field
+ */
+int header_is_present(const struct header *h, const char *field);
+
+/**
+ * @brief read the logger header
+ *
+ * @param h out: header
+ * @param map file mapping
+ */
+int header_read(struct header *h, void *map);
+
+/**
+ * @brief count number of bits in a given mask
+ *
+ * @param h #header file structure
+ * @param mask mask to compute
+ *
+ * @return number of bits in mask
+ */
+size_t header_get_mask_size(const struct header *h, const size_t mask);
+
+/**
+ * @brief Inverse the offset direction
+ *
+ * @param h #header file structure
+ * @param map file mapping
+ *
+ * @return error code
+ */
+int header_change_offset_direction(struct header *h, void *map);
+
+#endif  // __LOGGER_HEADER_H__

logger/io.c

+#include "io.h"
+#include "header.h"
+#include "tools.h"
+
+/* file size */
+#include <sys/stat.h>
+/* mapping */
+#include <sys/mman.h>
+/* open */
+#include <fcntl.h>
+
+int io_get_file_size(int fd, size_t *size) {
+  struct stat s;
+  int status = fstat(fd, &s);
+  if (status != 0) error(errno, "Unable to get file size");
+  *size = s.st_size;
+
+  return 0;
+}
+
+int io_open_file(char *filename, int *fd, void **map) {
+  /* open file */
+  *fd = open(filename, O_RDWR);
+  if (*fd == -1) error(errno, "Unable to open file %s", filename);
+
+  /* get file size */
+  size_t size;
+  int status = io_get_file_size(*fd, &size);
+  if (status != 0) return status;
+
+  /* map memory */
+  *map = mmap(NULL, size, PROT_WRITE | PROT_READ, MAP_SHARED, *fd, 0);
+  if (map == MAP_FAILED)
+    error(errno, "Failed to allocate map of size %zi bytes.", size);
+
+  return 0;
+}
+
+int io_close_file(int *fd, void **map) {
+  /* get file size */
+  size_t size;
+  int status = io_get_file_size(*fd, &size);
+  if (status != 0) return status;
+
+  /* unmap */
+  if (munmap(*map, size) != 0) error(errno, "Unable to unmap the file");
+
+  close(*fd);
+
+  return 0;
+}
+
+int io_read_mask(const struct header *h, void *map, size_t *offset,
+                 size_t *mask, size_t *diff_offset) {
+  /* read mask */
+  if (mask) {
+    *mask = 0;
+    memcpy(mask, map + *offset, LOGGER_MASK_SIZE);
+  }
+  *offset += LOGGER_MASK_SIZE;
+
+  /* read offset */
+  if (diff_offset) {
+    *diff_offset = 0;
+    memcpy(diff_offset, map + *offset, LOGGER_OFFSET_SIZE);
+  }
+  *offset += LOGGER_OFFSET_SIZE;
+
+  return 0;
+}
+
+int io_read_data(void *map, const size_t size, void *p, size_t *offset) {
+  memcpy(p, map + *offset, size);
+  *offset += size;
+
+  return 0;
+};
+
+int io_write_data(void *map, const size_t size, const void *p, size_t *offset) {
+  memcpy(map + *offset, p, size);
+  *offset += size;
+
+  return 0;
+};

logger/io.h

+#ifndef __SWIFT_LOGGER_IO_H__
+#define __SWIFT_LOGGER_IO_H__
+
+#include "header.h"
+#include "tools.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+/**
+ * @brief get the size of a file
+ *
+ * @param fd file id
+ * @param size out: file size
+ *
+ * @return error code
+ */
+int io_get_file_size(int fd, size_t *size);
+
+/**
+ * @brief Open a file and map it
+ *
+ * @param filename file to read
+ * @param fd out: file id
+ * @param map out: file mapping
+ *
+ * @return error code
+ */
+int io_open_file(char *filename, int *fd, void **map);
+
+/**
+ * @brief Close a file and unmap it
+ *
+ * @param fd file id
+ * @param map file mapping
+ *
+ * @return error code
+ */
+int io_close_file(int *fd, void **map);
+
+/**
+ * @brief read a single value in a file
+ *
+ * @param map file mapping
+ * @param size size of the chunk to read
+ * @param p pointer where to store the data
+ * @param offset In: position to read, Out: shifted by size
+ *
+ * @return error code
+ */
+int io_read_data(void *map, const size_t size, void *p, size_t *offset);
+
+/**
+ * @brief write a single value in a file
+ *
+ * @param map file mapping
+ * @param size size of the chunk to write
+ * @param p pointer to the data
+ * @param offset In: position to write, Out: shifted by size
+ *
+ * @return error code
+ */
+int io_write_data(void *map, const size_t size, const void *p, size_t *offset);
+
+/**
+ * @brief read a maks with its offset
+ *
+ * @param h #header file structure
+ * @param map file mapping
+ * @param offset In: position in the file, Out: shifted by the mask + offset
+ * size
+ * @param mask mask read
+ * @param diff_offset offset difference to previous/next corresponding chunk
+ *
+ * @return error code
+ */
+int io_read_mask(const struct header *h, void *map, size_t *offset,
+                 size_t *mask, size_t *diff_offset);
+
+#endif  // __SWIFT_LOGGER_IO_H__

logger/logger_loader.c

+#include "header.h"
+#include "io.h"
+#include "particle.h"
+#include "timeline.h"
+
+#include <Python.h>
+#include <errno.h>
+#include <numpy/arrayobject.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+/**
+ * @brief Reverse offset in dump file
+ *
+ * @param filename string filename of the dump file
+ */
+int reverseOffset(char *filename) {
+  struct header h;
+
+  /* open file */
+  int fd;
+  void *map;
+  if (io_open_file(filename, &fd, &map) != 0) return 1;
+
+  /* read header */
+  if (header_read(&h, map) != 0) return 1;
+
+  header_print(&h);
+
+  /* check offset direction */
+  if (h.forward_offset) {
+    error_no_return(EIO, "Offset are already reversed");
+    return 1;
+  }
+
+  /* compute file size */
+  size_t sz;
+  int status = io_get_file_size(fd, &sz);
+  if (status != 0) return 1;
+
+  size_t offset;
+  int error_code;
+
+#ifdef SWIFT_DEBUG_CHECKS
+  /* check offset */
+  printf("Check offsets...\n");
+  offset = h.offset_first;
+  while (offset < sz) {
+    error_code = tools_check_offset(&h, map, &offset);
+    if (error_code != 0) return 1;
+  }
+  printf("Check done\n");
+#endif
+
+  /* reverse header offset */
+  header_change_offset_direction(&h, map);
+
+  offset = h.offset_first;
+
+  /* reverse chunks */
+  printf("Reversing offsets...\n");
+  while (offset < sz) {
+    error_code = tools_reverse_offset(&h, map, &offset);
+    if (error_code != 0) return 1;
+  }
+  printf("Reversing done\n");
+
+#ifdef SWIFT_DEBUG_CHECKS
+  /* check offset */
+  printf("Check offsets...\n");
+  offset = h.offset_first;
+  while (offset < sz) {
+    error_code = tools_check_offset(&h, map, &offset);
+
+    if (error_code != 0) return 1;
+  }
+  printf("Check done\n");
+#endif
+
+  /* free internal variables */
+  header_free(&h);
+
+  if (io_close_file(&fd, &map) != 0) return 1;
+
+  return 0;
+}
+
+/**
+ * @brief load data from the offset without any interpolation
+ *
+ * @param offset PyArrayObject list of offset for each particle
+ * @param filename string filename of the dump file
+ * @return dictionnary containing the data read
+ */
+static PyObject *loadFromIndex(__attribute__((unused)) PyObject *self,
+                               PyObject *args) {
+
+  struct header h;
+
+  /* input */
+  PyArrayObject *offset = NULL;
+  char *filename = NULL;
+
+  /* output */
+  PyArrayObject *pos = NULL;
+  PyArrayObject *vel = NULL;
+  PyArrayObject *acc = NULL;
+  PyArrayObject *entropy = NULL;
+  PyArrayObject *h_sph = NULL;
+  PyArrayObject *rho = NULL;
+  PyArrayObject *mass = NULL;
+  PyArrayObject *id = NULL;
+
+  size_t time_offset;
+
+  /* parse arguments */
+
+  if (!PyArg_ParseTuple(args, "OsL", &offset, &filename, &time_offset))
+    return NULL;
+
+  if (!PyArray_Check(offset)) {
+    error_no_return(ENOTSUP, "Offset is not a numpy array");
+    return NULL;
+  }
+  if (PyArray_NDIM(offset) != 1) {
+    error_no_return(ENOTSUP, "Offset is not a 1 dimensional array");
+    return NULL;
+  }
+  if (PyArray_TYPE(offset) != NPY_UINT64) {
+    error_no_return(ENOTSUP, "Offset does not contain unsigned int");
+    return NULL;
+  }
+
+  /* open file */
+  int fd;
+  void *map;
+  if (io_open_file(filename, &fd, &map) != 0) return NULL;
+
+  /* read header */
+  if (header_read(&h, map) != 0) return NULL;
+
+  /* reverse offset if needed */
+  if (!h.forward_offset) {
+    if (io_close_file(&fd, &map) != 0) return NULL;
+
+    if (reverseOffset(filename) != 0) return NULL;
+
+    if (io_open_file(filename, &fd, &map) != 0) return NULL;
+
+    /* Reset header */
+    header_free(&h);
+    
+    if (header_read(&h, map) != 0) return NULL;
+  }
+
+  /* read timestamps */
+  struct time_array times;
+
+  if (time_array_init(&times, &h, map, fd) != 0) return NULL;
+
+  time_array_print(&times);
+  /* get required time */