Logger python

configure.ac

@@ -1130,6 +1130,7 @@ if test "x$with_python" != "xno"; then
       with_python="/usr/"
    fi
    AM_PATH_PYTHON([3], [], [AC_MSG_ERROR(python not found)])   
+   AC_ARG_VAR([PYTHON_LIBS], [Linking flags for python, bypassing python-config])
    AC_ARG_VAR([PYTHON_INCS], [Include flags for python, bypassing python-config])
    AC_ARG_VAR([PYTHON_CONFIG], [Path to python-config])
    AS_IF([test -z "$PYTHON_INCS"], [
@@ -1142,12 +1143,15 @@ if test "x$with_python" != "xno"; then
       ])
       AC_MSG_CHECKING([python include flags])
       PYTHON_INCS=`$PYTHON_CONFIG --includes`
+      PYTHON_LIBS=`$PYTHON_CONFIG --ldflags`
       AC_MSG_RESULT([$PYTHON_INCS])
+      AC_MSG_RESULT([$PYTHON_LIBS])
   ])
   have_python="yes"
   AC_DEFINE([HAVE_PYTHON],1,[Python appears to be present.])
 fi
 AC_SUBST([PYTHON_INCS])
+AC_SUBST([PYTHON_LIBS])
 AM_CONDITIONAL([HAVEPYTHON],[test -n "$PYTHON_INCS"])
 
 

logger/Makefile.am

@@ -61,6 +61,12 @@ if HAVEPYTHON
 AM_SOURCES += logger_python_wrapper.c
 endif
 
+if HAVEPYTHON
+# The main python structure is dependent on the version, thus we need to
+# remove this one in order to be safe.
+PYTHON_EXTRA_COMPILER_FLAG = -Wno-missing-field-initializers
+endif
+
 # Include files for distribution, not installation.
 nobase_noinst_HEADERS = logger_hydro.h hydro/Gadget2/logger_hydro.h \
 	logger_stars.h stars/Default/logger_stars.h \
@@ -68,12 +74,12 @@ nobase_noinst_HEADERS = logger_hydro.h hydro/Gadget2/logger_hydro.h \
 
 # Sources and flags for regular library
 liblogger_la_SOURCES = $(AM_SOURCES)
-liblogger_la_CFLAGS = $(AM_CFLAGS)
+liblogger_la_CFLAGS = $(AM_CFLAGS) $(PYTHON_EXTRA_COMPILER_FLAG)
 liblogger_la_LDFLAGS = $(AM_LDFLAGS) $(EXTRA_LIBS) $(BIN_LDFLAGS)
 
 # Sources and flags for MPI library
 # liblogger_mpi_la_SOURCES = $(AM_SOURCES)
-# liblogger_mpi_la_CFLAGS = $(AM_CFLAGS) $(MPI_FLAGS)
+# liblogger_mpi_la_CFLAGS = $(AM_CFLAGS) $(MPI_FLAGS) $(PYTHON_EXTRA_COMPILER_FLAG)
 # liblogger_mpi_la_LDFLAGS = $(AM_LDFLAGS) $(MPI_LIBS) $(EXTRA_LIBS)
 # liblogger_mpi_la_SHORTNAME = mpi
 # liblogger_mpi_la_LIBADD =

logger/examples/reader_example.py

@@ -47,9 +47,9 @@ print("basename: %s" % basename)
 print("time: %g" % time)
 
 # read the logger
-t = logger.getTimeLimits(basename)
-pos, ent = logger.get_particle_data(basename, ["Coordinates", "Entropies"],
-                                    time)
+with logger.Reader(basename, verbose=0) as reader:
+    t = reader.get_time_limits()
+    pos, ent = reader.get_particle_data(["Coordinates", "Entropies"], time)
 
 print("Min/Max of the position:", pos.min(), pos.max())
 print("Min/Max of the entropy:", ent.min(), ent.max())

logger/logger_header.c

@@ -113,7 +113,7 @@ void header_read(struct header *h, struct logger_logfile *log) {
   char file_format[STRING_SIZE];
   map = logger_loader_io_read_data(map, LOGGER_VERSION_SIZE, &file_format);
   if (strcmp(file_format, "SWIFT_LOGGER"))
-    error("Wrong file format (%s).", file_format);
+    error_python("Wrong file format (%s).", file_format);
 
   /* Read the major version number. */
   map = logger_loader_io_read_data(map, sizeof(int), &h->major_version);
@@ -122,7 +122,7 @@ void header_read(struct header *h, struct logger_logfile *log) {
   map = logger_loader_io_read_data(map, sizeof(int), &h->minor_version);
 
   struct logger_reader *reader = log->reader;
-  if (&reader->log != log) error("Wrong link to the reader.");
+  if (&reader->log != log) error_python("Wrong link to the reader.");
 
   if (reader->verbose > 0)
     message("File version %i.%i.", h->major_version, h->minor_version);
@@ -132,7 +132,7 @@ void header_read(struct header *h, struct logger_logfile *log) {
 
   if (!header_is_forward(h) && !header_is_backward(h) &&
       !header_is_corrupted(h))
-    error("Wrong offset value in the header (%i).", h->offset_direction);
+    error_python("Wrong offset value in the header (%i).", h->offset_direction);
 
   /* Read offset to first record. */
   h->offset_first_record = 0;
@@ -146,7 +146,7 @@ void header_read(struct header *h, struct logger_logfile *log) {
 
   /* Check if value defined in this file is large enough. */
   if (STRING_SIZE < h->string_length) {
-    error("Name too large in log file %i.", h->string_length);
+    error_python("Name too large in log file %i.", h->string_length);
   }
 
   /* Read the number of masks. */
@@ -155,7 +155,7 @@ void header_read(struct header *h, struct logger_logfile *log) {
   /* Allocate the masks memory. */
   h->masks = malloc(sizeof(struct mask_data) * h->masks_count);
   if (h->masks == NULL) {
-    error("Failed to allocate the memory for the masks.");
+    error_python("Failed to allocate the memory for the masks.");
   }
 
   /* Loop over all masks. */
@@ -179,14 +179,14 @@ void header_read(struct header *h, struct logger_logfile *log) {
 
   /* Check that the timestamp mask exists */
   if (h->timestamp_mask == 0) {
-    error("Unable to find the timestamp mask.");
+    error_python("Unable to find the timestamp mask.");
   }
 
   /* Check the logfile header's size. */
   if (map != (char *)log->log.map + h->offset_first_record) {
     size_t offset = (char *)map - (char *)log->log.map;
-    error("Wrong header size (in header %zi, current %zi).",
-          h->offset_first_record, offset);
+    error_python("Wrong header size (in header %zi, current %zi).",
+                 h->offset_first_record, offset);
   }
 
   /* Set the first and second derivatives as non existent. */
@@ -220,7 +220,6 @@ void header_read(struct header *h, struct logger_logfile *log) {
   hydro_logger_reader_link_derivatives(h);
 
   /* Gravity */
-
   /* Set the link between local and global */
   for (int j = 0; j < gravity_logger_field_count; j++) {
     gravity_logger_local_to_global[j] = -1;

logger/logger_index.c

@@ -132,7 +132,7 @@ void logger_index_map_file(struct logger_index *index, const char *filename,
                            int sorted) {
   /* Un-map previous file */
   if (index->index.map != NULL) {
-    error("Trying to remap.");
+    error_python("Trying to remap.");
   }
 
   /* Check if need to sort the file */
@@ -174,7 +174,7 @@ void logger_index_map_file(struct logger_index *index, const char *filename,
  */
 void logger_index_free(struct logger_index *index) {
   if (index->index.map == NULL) {
-    error("Trying to unmap an unexisting map");
+    error_python("Trying to unmap an unexisting map");
   }
   logger_loader_io_munmap_file(&index->index);
 }

logger/logger_loader_io.c

@@ -36,7 +36,8 @@
 size_t logger_loader_io_get_file_size(int fd) {
   struct stat s;
   int status = fstat(fd, &s);
-  if (status != 0) error("Unable to get file size (%s).", strerror(errno));
+  if (status != 0)
+    error_python("Unable to get file size (%s).", strerror(errno));
   return s.st_size;
 }
 
@@ -61,7 +62,7 @@ void logger_loader_io_mmap_file(struct mapped_file *map, const char *filename,
     fd = open(filename, O_RDWR);
 
   if (fd == -1)
-    error("Unable to open file %s (%s).", filename, strerror(errno));
+    error_python("Unable to open file %s (%s).", filename, strerror(errno));
 
   /* get the file size. */
   map->mmap_size = logger_loader_io_get_file_size(fd);
@@ -72,8 +73,8 @@ void logger_loader_io_mmap_file(struct mapped_file *map, const char *filename,
 
   map->map = mmap(NULL, map->mmap_size, mode, MAP_SHARED, fd, 0);
   if (map->map == MAP_FAILED)
-    error("Failed to allocate map of size %zi bytes (%s).", map->mmap_size,
-          strerror(errno));
+    error_python("Failed to allocate map of size %zi bytes (%s).",
+                 map->mmap_size, strerror(errno));
 
   /* Close the file. */
   close(fd);
@@ -88,7 +89,7 @@ void logger_loader_io_mmap_file(struct mapped_file *map, const char *filename,
 void logger_loader_io_munmap_file(struct mapped_file *map) {
   /* unmap the file. */
   if (munmap(map->map, map->mmap_size) != 0) {
-    error("Unable to unmap the file (%s).", strerror(errno));
+    error_python("Unable to unmap the file (%s).", strerror(errno));
   }
 
   /* Reset values */

logger/logger_logfile.c

@@ -36,7 +36,7 @@ void logger_logfile_init_from_file(struct logger_logfile *log, char *filename,
 
   /* Set the pointer to the reader. */
   log->reader = reader;
-  if (&reader->log != log) error("Wrong link to the reader.");
+  if (&reader->log != log) error_python("Wrong link to the reader.");
 
   /* Set pointers to zero. */
   time_array_init(&log->times);
@@ -61,7 +61,7 @@ void logger_logfile_init_from_file(struct logger_logfile *log, char *filename,
 
   /* Check if the offset are corrupted. */
   if (header_is_corrupted(&log->header)) {
-    error("The offsets have been corrupted.");
+    error_python("The offsets have been corrupted.");
   }
 
   /* Reverse the offsets direction. */
@@ -108,11 +108,11 @@ void logger_logfile_reverse_offset(struct logger_logfile *log, char *filename) {
   /* Get pointers */
   struct header *header = &log->header;
   const struct logger_reader *reader = log->reader;
-  if (&reader->log != log) error("Wrong link to the reader.");
+  if (&reader->log != log) error_python("Wrong link to the reader.");
 
   /* Check if the offsets need to be reversed. */
   if (!header_is_backward(header)) {
-    error("The offsets are already reversed.");
+    error_python("The offsets are already reversed.");
   }
 
 #ifdef SWIFT_DEBUG_CHECKS

logger/logger_particle.c

@@ -61,12 +61,12 @@ __attribute__((always_inline)) INLINE size_t logger_particle_read_field(
 
   /* Check that the mask is within the limits. */
   if (*mask > (unsigned int)(1 << h->masks_count)) {
-    error("Found an unexpected mask %zi", *mask);
+    error_python("Found an unexpected mask %zi", *mask);
   }
 
   /* Check if it is not a time record. */
   if (*mask == h->timestamp_mask) {
-    error("Cannot particle from timestep record.");
+    error_python("Cannot particle from timestep record.");
   }
 
   /* Read the record and copy it to a particle. */

logger/logger_reader.c

@@ -93,11 +93,11 @@ void logger_reader_init_index(struct logger_reader *reader) {
   /* Initialize the arrays */
   if ((reader->index.times = (double *)malloc(count * sizeof(double))) ==
       NULL) {
-    error("Failed to allocate the list of times");
+    error_python("Failed to allocate the list of times");
   }
   if ((reader->index.int_times =
            (integertime_t *)malloc(count * sizeof(integertime_t))) == NULL) {
-    error("Failed to allocate the list of times");
+    error_python("Failed to allocate the list of times");
   }
 
   /* Get the information contained in the headers */
@@ -255,7 +255,7 @@ void logger_reader_read_field(struct logger_reader *reader, double time,
       field_count = stars_logger_field_count;
       break;
     default:
-      error("Particle type not implemented");
+      error_python("Particle type not implemented");
   }
 
   /* Get the masks. */
@@ -344,7 +344,7 @@ void logger_reader_read_field(struct logger_reader *reader, double time,
 
       /* Check if we can still move forward */
       if (h_offset == 0) {
-        error("There is no record after the current one");
+        error_python("There is no record after the current one");
       }
 
       /* Go to the next record. */
@@ -412,7 +412,7 @@ void logger_reader_read_field(struct logger_reader *reader, double time,
                                        output, time, field);
         break;
       default:
-        error("Particle type not implemented");
+        error_python("Particle type not implemented");
     }
   }
 }
@@ -460,7 +460,7 @@ void logger_reader_global_to_local(
       local_names = stars_logger_field_names;
       break;
     default:
-      error("Particle type not implemented yet.");
+      error_python("Particle type not implemented yet.");
   }
 
   /* Initialize the arrays */
@@ -495,8 +495,8 @@ void logger_reader_global_to_local(
   /* Check that we found the fields */
   for (int local = 0; local < n_fields_wanted; local++) {
     if (local_fields_wanted[local] < 0) {
-      error("Field %s not found in particle type %s", local_names[local],
-            part_type_names[type]);
+      error_python("Field %s not found in particle type %s", local_names[local],
+                   part_type_names[type]);
     }
   }
 }
@@ -525,21 +525,21 @@ void logger_reader_read_all_particles(struct logger_reader *reader, double time,
   /* fields_wanted sorted according to the fields order (local index). */
   int *local_fields_wanted = (int *)malloc(sizeof(int) * n_fields_wanted);
   if (local_fields_wanted == NULL) {
-    error("Failed to allocate the array of sorted fields.");
+    error_python("Failed to allocate the array of sorted fields.");
   }
 
   /* Fields corresponding to the first derivative of fields_wanted (sorted and
    * local index). */
   int *local_first_deriv = malloc(sizeof(int) * n_fields_wanted);
   if (local_first_deriv == NULL) {
-    error("Failed to allocate the list of first derivative.");
+    error_python("Failed to allocate the list of first derivative.");
   }
 
   /* Fields corresponding to the second derivative of fields_wanted (sorted and
    * local index). */
   int *local_second_deriv = malloc(sizeof(int) * n_fields_wanted);
   if (local_second_deriv == NULL) {
-    error("Failed to allocate the list of second derivative.");
+    error_python("Failed to allocate the list of second derivative.");
   }
 
   /* Do the hydro. */

logger/logger_time.c

@@ -36,7 +36,7 @@ void time_array_ensure_size(struct time_array *t) {
 
   /* Allocate the new array */
   struct time_record *tmp = malloc(sizeof(struct time_record) * t->capacity);
-  if (tmp == NULL) error("Failed to allocate the time records.");
+  if (tmp == NULL) error_python("Failed to allocate the time records.");
 
   /* Copy the memory */
   memcpy(tmp, t->records, sizeof(struct time_record) * t->size);
@@ -100,10 +100,10 @@ size_t time_read(integertime_t *int_time, double *time,
 
   /* check if time mask is present in log file header. */
   int ind = header_get_field_index(h, "Timestamp");
-  if (ind == -1) error("File header does not contain a mask for time.");
+  if (ind == -1) error_python("File header does not contain a mask for time.");
 
   /* check if reading a time record. */
-  if (h->masks[ind].mask != mask) error("Not a time record.");
+  if (h->masks[ind].mask != mask) error_python("Not a time record.");
 #endif
 
   /* read the record. */
@@ -130,12 +130,13 @@ size_t time_offset_first_record(const struct header *h) {
   /* Check that the first record is really a time record. */
   int i = header_get_field_index(h, "Timestamp");
 
-  if (i == -1) error("Time mask not present in the log file header.");
+  if (i == -1) error_python("Time mask not present in the log file header.");
 
   size_t mask = 0;
   logger_loader_io_read_mask(h, (char *)map + offset, &mask, NULL);
 
-  if (mask != h->masks[i].mask) error("Log file should begin by timestep.");
+  if (mask != h->masks[i].mask)
+    error_python("Log file should begin by timestep.");
 
   return h->offset_first_record;
 }
@@ -148,7 +149,8 @@ size_t time_offset_first_record(const struct header *h) {
 void time_array_init(struct time_array *t) {
   /* Allocate the arrays */
   t->records = malloc(sizeof(struct time_record) * LOGGER_TIME_INIT_SIZE);
-  if (t->records == NULL) error("Failed to initialize the time records.");
+  if (t->records == NULL)
+    error_python("Failed to initialize the time records.");
 
   /* Initialize the sizes */
   t->size = 0;
@@ -223,11 +225,11 @@ double time_array_get_time(const struct time_array *t, const size_t offset) {
 size_t time_array_get_index(const struct time_array *t, const size_t offset) {
 
 #ifdef SWIFT_DEBUG_CHECKS
-  if (!t) error("NULL pointer.");
+  if (!t) error_python("NULL pointer.");
 
   if (offset < t->records[0].offset || offset > t->records[t->size - 1].offset)
-    error("Offset outside of range. %zi > %zi > %zi",
-          t->records[t->size - 1].offset, offset, t->records[0].offset);
+    error_python("Offset outside of range. %zi > %zi > %zi",
+                 t->records[t->size - 1].offset, offset, t->records[0].offset);
 #endif
 
   /* right will contain the index at the end of the loop */
@@ -257,7 +259,7 @@ size_t time_array_get_index(const struct time_array *t, const size_t offset) {
 #ifdef SWIFT_DEBUG_CHECKS
   if (t->records[right].offset > offset ||
       t->records[right + 1].offset <= offset) {
-    error("Found the wrong element");
+    error_python("Found the wrong element");
   }
 
 #endif
@@ -277,11 +279,11 @@ size_t time_array_get_index_from_time(const struct time_array *t,
                                       const double time) {
 
 #ifdef SWIFT_DEBUG_CHECKS
-  if (!t) error("NULL pointer.");
+  if (!t) error_python("NULL pointer.");
 
   if (time < t->records[0].time || time > t->records[t->size - 1].time)
-    error("Time outside of range (%g > %g).", time,
-          t->records[t->size - 1].time);
+    error_python("Time outside of range (%g > %g).", time,
+                 t->records[t->size - 1].time);
 #endif
 
   /* right will contain the index at the end of the loop */
@@ -309,7 +311,7 @@ size_t time_array_get_index_from_time(const struct time_array *t,
 
 #ifdef SWIFT_DEBUG_CHECKS
   if (t->records[right].time > time || t->records[right + 1].time <= time) {
-    error("Found the wrong element");
+    error_python("Found the wrong element");
   }
 
 #endif

logger/logger_tools.c

@@ -42,7 +42,7 @@ int tools_get_next_record(const struct header *h, void *map, size_t *offset,
   if (header_is_backward(h))
     return _tools_get_next_record_backward(h, map, offset, file_size);
   else
-    error("Offsets are corrupted.");
+    error_python("Offsets are corrupted.");
 }
 
 /**
@@ -84,7 +84,7 @@ int _tools_get_next_record_forward(const struct header *h, void *map,
 int _tools_get_next_record_backward(const struct header *h, void *map,
                                     size_t *offset, size_t file_size) {
 #ifndef SWIFT_DEBUG_CHECKS
-  error("Should not be used, method too slow");
+  error_python("Should not be used, method too slow");
 #endif
   size_t current_offset = *offset;
   size_t record_header = LOGGER_MASK_SIZE + LOGGER_OFFSET_SIZE;
@@ -141,8 +141,8 @@ size_t tools_reverse_offset(const struct header *h, void *file_map,
   if (prev_offset == cur_offset) return after_current_record;
 
   if (prev_offset > cur_offset)
-    error("Unexpected offset: header %lu, current %lu.", prev_offset,
-          cur_offset);
+    error_python("Unexpected offset: header %lu, current %lu.", prev_offset,
+                 cur_offset);
 
   /* modify previous offset. */
   map = (char *)file_map + cur_offset - prev_offset + LOGGER_MASK_SIZE;
@@ -156,7 +156,7 @@ size_t tools_reverse_offset(const struct header *h, void *file_map,
   /* Check if we are not mixing timestamp and particles */
   if ((prev_mask != h->timestamp_mask && mask == h->timestamp_mask) ||
       (prev_mask == h->timestamp_mask && mask != h->timestamp_mask))
-    error("Unexpected mask: %lu, got %lu.", mask, prev_mask);
+    error_python("Unexpected mask: %lu, got %lu.", mask, prev_mask);
 
 #endif  // SWIFT_DEBUG_CHECKS
 
@@ -177,7 +177,7 @@ size_t tools_reverse_offset(const struct header *h, void *file_map,
 size_t tools_check_record_consistency(const struct logger_reader *reader,
                                       size_t offset) {
 #ifndef SWIFT_DEBUG_CHECKS
-  error("Should not check in non debug mode.");
+  error_python("Should not check in non debug mode.");
 #endif
 
   const struct header *h = &reader->log.header;
@@ -195,11 +195,11 @@ size_t tools_check_record_consistency(const struct logger_reader *reader,
     pointed_offset += offset;
   else if (header_is_backward(h)) {
     if (offset < pointed_offset)
-      error("Offset too large (%lu) at %lu with mask %lu.", pointed_offset,
-            offset, mask);
+      error_python("Offset too large (%lu) at %lu with mask %lu.",
+                   pointed_offset, offset, mask);
     pointed_offset = offset - pointed_offset;
   } else {
-    error("Offset are corrupted.");
+    error_python("Offset are corrupted.");
   }
 
   /* set offset after current record. */
@@ -216,8 +216,8 @@ size_t tools_check_record_consistency(const struct logger_reader *reader,
   /* check if not mixing timestamp and particles. */
   if ((pointed_mask != h->timestamp_mask && mask == h->timestamp_mask) ||
       (pointed_mask == h->timestamp_mask && mask != h->timestamp_mask))
-    error("Error in the offset (mask %lu at %lu != %lu at %lu).", mask, offset,
-          pointed_mask, pointed_offset);
+    error_python("Error in the offset (mask %lu at %lu != %lu at %lu).", mask,
+                 offset, pointed_mask, pointed_offset);
 
   return (size_t)((char *)map - (char *)file_init);
 }

logger/logger_tools.h

@@ -43,7 +43,6 @@
 #include <string.h>
 
 #define STRING_SIZE 200
-#define LOGGER_DOUBLE_NOT_ASSIGNED nan("214");
 
 struct header;
 struct logger_reader;
@@ -79,4 +78,27 @@ float logger_tools_cubic_hermite_spline(double t0, float v0, float a0,
                                         double t1, float v1, float a1,
                                         double t);
 
+#ifndef HAVE_PYTHON
+#define error_python(...) error(##__VA_ARGS__);
+#else
+/**
+ * @brief Print the python trace back
+ */
+__attribute__((always_inline)) INLINE static void logger_loader_print_traceback(
+    void) {
+
+  /* Import the traceback module */
+  PyObject *pyth_module = PyImport_ImportModule("traceback");
+  PyObject_CallMethod(pyth_module, "print_stack", "");
+
+  Py_DECREF(pyth_module);
+}
+
+#define error_python(s, ...)         \
+  ({                                 \
+    logger_loader_print_traceback(); \
+    error(s, ##__VA_ARGS__);         \
+  })
+#endif
+
 #endif  // LOGGER_LOGGER_TOOLS_H

logger/tests/Makefile.am

@@ -17,7 +17,7 @@
 # Add the source directory and the non-standard paths to the included library headers to CFLAGS
 AM_CFLAGS = $(PYTHON_INCS) -I$(top_srcdir)/src -I$(top_srcdir)/logger $(HDF5_CPPFLAGS) $(GSL_INCS) $(FFTW_INCS)
 
-AM_LDFLAGS = ../../src/.libs/libswiftsim.a ../.libs/liblogger.a $(HDF5_LDFLAGS) $(HDF5_LIBS) $(FFTW_LIBS) $(TCMALLOC_LIBS) $(JEMALLOC_LIBS) $(TBBMALLOC_LIBS) $(GRACKLE_LIBS) $(GSL_LIBS) $(PROFILER_LIBS) -lswiftsim
+AM_LDFLAGS = ../../src/.libs/libswiftsim.a ../.libs/liblogger.a $(HDF5_LDFLAGS) $(HDF5_LIBS) $(FFTW_LIBS) $(TCMALLOC_LIBS) $(JEMALLOC_LIBS) $(TBBMALLOC_LIBS) $(GRACKLE_LIBS) $(GSL_LIBS) $(PROFILER_LIBS) $(PYTHON_LIBS) -lswiftsim
 
 # List of programs and scripts to run in the test suite
 TESTS = testLogfileHeader testLogfileReader testTimeArray testQuickSort testVR