diff --git a/Makefile.am b/Makefile.am
index b5ede6fd9772fd93b97645c6ef39995a6659f947..8746f602d6c46d82d042eaa98a8d7b97d92f3a39 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -38,14 +38,14 @@ MYFLAGS =
# Add the source directory and the non-standard paths to the included library headers to CFLAGS
AM_CFLAGS = -I$(top_srcdir)/src -I$(top_srcdir)/argparse $(HDF5_CPPFLAGS) \
$(GSL_INCS) $(FFTW_INCS) $(NUMA_INCS) $(GRACKLE_INCS) \
- $(CHEALPIX_CFLAGS)
+ $(CHEALPIX_CFLAGS) $(LUSTREAPI_CFLAGS)
AM_LDFLAGS = $(HDF5_LDFLAGS)
# Extra libraries.
EXTRA_LIBS = $(GSL_LIBS) $(HDF5_LIBS) $(FFTW_LIBS) $(NUMA_LIBS) $(PROFILER_LIBS) \
$(TCMALLOC_LIBS) $(JEMALLOC_LIBS) $(TBBMALLOC_LIBS) $(GRACKLE_LIBS) \
- $(CHEALPIX_LIBS)
+ $(CHEALPIX_LIBS) $(LUSTREAPI_LIBS)
# MPI libraries.
MPI_LIBS = $(PARMETIS_LIBS) $(METIS_LIBS) $(MPI_THREAD_LIBS) $(FFTW_MPI_LIBS)
diff --git a/configure.ac b/configure.ac
index 347133e15136dfa0d0301beb97397fdad19df09b..25551dc4170fa7d096a5aa01bafdb9db1e7307f3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1431,6 +1431,38 @@ if test "$with_hdf5" = "yes"; then
fi
AM_CONDITIONAL([HAVEPARALLELHDF5],[test "$have_parallel_hdf5" = "yes"])
+# Check for lustre support. Attempted by default.
+have_lustreapi="no"
+AC_ARG_WITH([lustreapi],
+ [AS_HELP_STRING([--with-lustreapi=PATH],
+ [use the lustre api library for some striping control @<:@yes/no@:>@]
+ )],
+ [with_lustreapi="$withval"],
+ [with_lustreapi="yes"]
+)
+
+if test "x$with_lustreapi" != "xno"; then
+ if test "x$with_lustreapi" != "xyes" -a "x$with_lustreapi" != "x"; then
+ LUSTREAPI_LIBS="-L$with_lustreapi -llustreapi"
+ LUSTREAPI_INCS="-I$with_lustreapi/include"
+ else
+ LUSTREAPI_LIBS="-llustreapi"
+ LUSTREAPI_INCS=""
+ fi
+ AC_CHECK_LIB([lustreapi], [llapi_obd_statfs], [have_lustreapi="yes"],
+ [have_lustreapi="no"],[$LUSTREAPI_LIBS])
+
+ if test "$have_lustreapi" = "yes"; then
+ AC_DEFINE([HAVE_LUSTREAPI],1,[The lustre API library appears to be present.])
+ else
+ LUSTREAPI_LIBS=""
+ fi
+fi
+AC_SUBST([LUSTREAPI_LIBS])
+AC_SUBST([LUSTREAPI_INCS])
+AM_CONDITIONAL([HAVELUSTREAPI],[test -n "$LUSTREAPI_LIBS"])
+
+
# Check for grackle.
have_grackle="no"
AC_ARG_WITH([grackle],
@@ -3318,6 +3350,7 @@ AC_MSG_RESULT([
MPI enabled : $enable_mpi
HDF5 enabled : $with_hdf5
- parallel : $have_parallel_hdf5
+ LUSTRE API enabled : $have_lustreapi
METIS/ParMETIS : $have_metis / $have_parmetis
FFTW3 enabled : $have_fftw
- threaded/openmp : $have_threaded_fftw / $have_openmp_fftw
diff --git a/doc/RTD/source/ParameterFiles/parameter_description.rst b/doc/RTD/source/ParameterFiles/parameter_description.rst
index 522f000816b024f2d8e3b3bf365a5d3bf86db7d0..5827427469aa2766ae3d87a5d98d3fb36b1aa97f 100644
--- a/doc/RTD/source/ParameterFiles/parameter_description.rst
+++ b/doc/RTD/source/ParameterFiles/parameter_description.rst
@@ -506,7 +506,7 @@ The full section to start a typical cosmological run would be:
minimal_temperature: 100 # U_T
H_mass_fraction: 0.755
H_ionization_temperature: 1e4 # U_T
- particle_splitting: 1
+ particle_splitting: 1
particle_splitting_mass_threshold: 5e-3 # U_M
.. _Parameters_Stars:
@@ -517,7 +517,7 @@ Stars
The ``Stars`` section is used to set parameters that describe the Stars
calculations when doing feedback or enrichment. Note that if stars only act
gravitationally (i.e. SWIFT is run *without* ``--feedback``) no parameters
-in this section are used.
+in this section are used.
The first four parameters are related to the neighbour search:
@@ -589,7 +589,7 @@ Below you will find the description of the ``none`` which is the default model.
By default, the code is configured with ``--with-sink=none``. Then, the ``DefaultSink`` section is used to set parameters that describe the sinks in this model. The unique parameter is the sink accretion radius (also called cut-off radius): ``cut_off_radius``.
-Note that this model does not create sink particles or accrete gas.
+Note that this model does not create sink particles or accrete gas.
The full section is:
@@ -646,7 +646,7 @@ the start and end times or scale factors from the parameter file.
``max_dt_RMS_factor`` (default: ``0.25``)
* Whether or not only the gas particle masses should be considered for
the baryon component of the calculation: ``dt_RMS_use_gas_only`` (default: ``0``)
-
+
These values rarely need altering. The second parameter is only
meaningful if a subgrid model produces star (or other) particles with
masses substantially smaller than the gas masses. See the theory
@@ -671,7 +671,7 @@ Whilst for a cosmological run, one would need:
dt_min: 1e-10
max_dt_RMS_factor: 0.25 # Default optional value
dt_RMS_use_gas_only: 0 # Default optional value
-
+
.. _Parameters_ICs:
Initial Conditions
@@ -716,7 +716,7 @@ Finally, SWIFT also offers these options:
* Whether to replicate the box along each axis: ``replicate`` (default: ``1``).
* Whether to re-map the IDs to the range ``[0, N]`` and hence discard
the original IDs from the IC file: ``remap_ids`` (default: ``0``).
-
+
The shift is expressed in internal units and will be written to the header of
the snapshots. The option to replicate the box is especially useful for
weak-scaling tests. When set to an integer >1, the box size is multiplied by
@@ -881,24 +881,42 @@ that can be used as if it was a non-distributed snapshot. In this case, the
HDF5 library itself can figure out which file is needed when manipulating the
snapshot.
-On Lustre filesystems [#f4]_ it is important to properly stripe files to achieve
-a good writing speed. If the parameter ``lustre_OST_count`` is set to the number
-of OSTs present on the system, then SWIFT will set the `stripe count` of each
-distributed file to `1` and set each file's `stripe index` to the MPI rank
-generating it modulo the OST count [#f5]_. If the parameter is not set then the
-files will be created with the default system policy (or whatever was set for
-the directory where the files are written). This parameter has no effect on
-non-Lustre file systems and no effect if distributed snapshots are not used.
-
-* The number of Lustre OSTs to distribute the single-striped distributed
- snapshot files over: ``lustre_OST_count`` (default: ``0``)
-
+On Lustre filesystems [#f4]_ it is important to properly stripe files to
+achieve a good writing and reading speed. If the parameter
+``lustre_OST_checks`` is set and the lustre API is available SWIFT will
+determine the number of OSTs available and rank these by free space, it will
+then set the `stripe count` of each file to `1` and choose an OST
+`offset` so each rank writes to a different OST, unless there are more ranks
+than OSTs in which case the assignment wraps. In this way OSTs should be
+filled evenly and written to using an optimal access pattern.
+
+If the parameter is not set then the files will be created with the default
+system policy (or whatever was set for the directory where the files are
+written). This parameter has no effect on non-Lustre file systems.
+
+Other parameters are also provided to handle the cases when individual OSTs do
+not have sufficient free space to write a file: ``lustre_OST_free`` and
+when OSTs are closed for administrative reasons: ``lustre_OST_test``, in which
+case they cannot be written. This is important as the `offset` assignment in
+this case is not used by lustre which picks the next writable OST, so in our
+scheme such OSTs will be used more times than we intended.
+
+* Use the lustre API to assign a stripe and offset to the distributed snapshot
+ files:
+ ``lustre_OST_checks`` (default: ``0``)
+
+* Do not use OSTs that do not have a certain amount of free space in MiB.
+ Zero disables and -1 activates a guess based on the size of the process:
+ ``lustre_OST_free`` (default: ``0``)
+
+* Check OSTs can be written to and remove those from consideration:
+ ``lustre_OST_test`` (default: ``0``)
Users can optionally ask to randomly sub-sample the particles in the snapshots.
This is specified for each particle type individually:
-* Whether to switch on sub-sampling: ``subsample``
-* Whether to switch on sub-sampling: ``subsample_fraction``
+* Whether to switch on sub-sampling: ``subsample``
+* Whether to switch on sub-sampling: ``subsample_fraction``
These are arrays of 7 elements defaulting to seven 0s if left unspecified. Each
entry corresponds to the particle type used in the initial conditions and
@@ -908,7 +926,7 @@ indicating the fraction of particles to keep in the outputs. Note that the
selection of particles is selected randomly for each individual
snapshot. Particles can hence not be traced back from output to output when this
is switched on.
-
+
Users can optionally specify the level of compression used by the HDF5 library
using the parameter:
@@ -1126,7 +1144,7 @@ output field will be enabled. If this is 0 lossy compression is not applied.
* Whether to use lossless compression in the particle outputs: ``particles_gzip_level``
If this is non-zero the HDF5 deflate filter will be applied to lightcone particle output with
-the compression level set to the specified value.
+the compression level set to the specified value.
* HEALPix map resolution: ``nside``
@@ -1167,7 +1185,7 @@ filter names. Set the filter name to ``on`` to disable compression.
* Whether to use lossless compression in the HEALPix map outputs: ``maps_gzip_level``
If this is non-zero the HDF5 deflate filter will be applied to the lightcone map output with
-the compression level set to the specified value.
+the compression level set to the specified value.
The following shows a full set of light cone parameters for the case where we're making two
light cones which only differ in the location of the observer:
@@ -1181,7 +1199,7 @@ light cones which only differ in the location of the observer:
buffer_chunk_size: 100000
max_particles_buffered: 1000000
hdf5_chunk_size: 10000
-
+
# Redshift ranges for particle types
z_range_for_Gas: [0.0, 0.05]
z_range_for_DM: [0.0, 0.05]
@@ -1189,7 +1207,7 @@ light cones which only differ in the location of the observer:
z_range_for_Stars: [0.0, 0.05]
z_range_for_BH: [0.0, 0.05]
z_range_for_Neutrino: [0.0, 0.05]
-
+
# Healpix map parameters
nside: 512
radius_file: ./shell_radii.txt
@@ -1214,7 +1232,7 @@ light cones which only differ in the location of the observer:
enabled: 1
basename: lightcone1
observer_position: [74.2, 10.80, 53.59]
-
+
An example of the radius file::
@@ -1240,11 +1258,11 @@ Equation of State (EoS)
The ``EoS`` section contains options for the equations of state.
Multiple EoS can be used for :ref:`planetary`,
-see :ref:`planetary_eos` for more information.
+see :ref:`planetary_eos` for more information.
To enable one or multiple EoS, the corresponding ``planetary_use_*:``
flag(s) must be set to ``1`` in the parameter file for a simulation,
-along with the path to any table files, which are set by the
+along with the path to any table files, which are set by the
``planetary_*_table_file:`` parameters.
For the (non-planetary) isothermal EoS, the ``isothermal_internal_energy:``
@@ -1430,17 +1448,35 @@ the MPI-rank. SWIFT writes one file per MPI rank. If the ``save`` option has
been activated, the previous set of restart files will be named
``basename_000000.rst.prev``.
-On Lustre filesystems [#f4]_ it is important to properly stripe files to achieve
-a good writing and reading speed. If the parameter ``lustre_OST_count`` is set
-to the number of OSTs present on the system, then SWIFT will set the `stripe
-count` of each restart file to `1` and set each file's `stripe index` to the MPI
-rank generating it modulo the OST count [#f5]_. If the parameter is not set then
-the files will be created with the default system policy (or whatever was set
-for the directory where the files are written). This parameter has no effect on
-non-Lustre file systems.
+On Lustre filesystems [#f4]_ it is important to properly stripe files to
+achieve a good writing and reading speed. If the parameter
+``lustre_OST_checks`` is set and the lustre API is available SWIFT will
+determine the number of OSTs available and rank these by free space, it will
+then set the `stripe count` of each restart file to `1` and choose an OST
+`offset` so each rank writes to a different OST, unless there are more ranks
+than OSTs in which case the assignment wraps. In this way OSTs should be
+filled evenly and written to using an optimal access pattern.
+
+If the parameter is not set then the files will be created with the default
+system policy (or whatever was set for the directory where the files are
+written). This parameter has no effect on non-Lustre file systems.
+
+Other parameters are also provided to handle the cases when individual OSTs do
+not have sufficient free space to write a restart file: ``lustre_OST_free`` and
+when OSTs are closed for administrative reasons: ``lustre_OST_test``, in which
+case they cannot be written. This is important as the `offset` assignment in
+this case is not used by lustre which picks the next writable OST, so in our
+scheme such OSTs will be used more times than we intended.
+
+* Use the lustre API to assign a stripe and offset to restart files:
+ ``lustre_OST_checks`` (default: ``0``)
-* The number of Lustre OSTs to distribute the single-striped restart files over:
- ``lustre_OST_count`` (default: ``0``)
+* Do not use OSTs that do not have a certain amount of free space in MiB.
+ Zero disables and -1 activates a guess based on the size of the process:
+ ``lustre_OST_free`` (default: ``0``)
+
+* Check OSTs can be written to and remove those from consideration:
+ ``lustre_OST_test`` (default: ``0``)
SWIFT can also be stopped by creating an empty file called ``stop`` in the
directory where the restart files are written (i.e. the directory speicified by
@@ -1848,11 +1884,11 @@ necessary and one would use:
invoke_stf: 1 # We want VELOCIraptor to be called when snapshots are dumped.
# ...
# Rest of the snapshots properties
-
+
StructureFinding:
config_file_name: my_stf_configuration_file.cfg # See the VELOCIraptor manual for the content of this file.
basename: ./haloes/ # Write the catalogs in this sub-directory
-
+
If one additionally want to call VELOCIraptor at times not linked with
snapshots, the additional parameters need to be supplied.
@@ -1932,7 +1968,7 @@ Fermi-Dirac momenta will be generated if ``generate_ics`` is used. The
``use_delta_f``. Finally, a random seed for the Fermi-Dirac momenta can
be set with ``neutrino_seed``.
-For mode details on the neutrino implementation, refer to :ref:`Neutrinos`.
+For mode details on the neutrino implementation, refer to :ref:`Neutrinos`.
A complete specification of the model looks like
.. code:: YAML
@@ -1944,7 +1980,7 @@ A complete specification of the model looks like
------------------------
-
+
.. [#f1] The thorough reader (or overly keen SWIFT tester) would find that the speed of light is :math:`c=1.8026\times10^{12}\,\rm{fur}\,\rm{ftn}^{-1}`, Newton's constant becomes :math:`G_N=4.896735\times10^{-4}~\rm{fur}^3\,\rm{fir}^{-1}\,\rm{ftn}^{-2}` and Planck's constant turns into :math:`h=4.851453\times 10^{-34}~\rm{fur}^2\,\rm{fir}\,\rm{ftn}^{-1}`.
diff --git a/examples/parameter_example.yml b/examples/parameter_example.yml
index 7d9dd6e18c3c94a6b9d7ba197c79346628015d7e..ca3258a13db07cd11957cddcdb85d8004e56f705 100644
--- a/examples/parameter_example.yml
+++ b/examples/parameter_example.yml
@@ -181,7 +181,9 @@ Snapshots:
invoke_ps: 0 # (Optional) Call a power-spectrum calculation every time a snapshot is written
compression: 0 # (Optional) Set the level of GZIP compression of the HDF5 datasets [0-9]. 0 does no compression. The lossless compression is applied to *all* the fields.
distributed: 0 # (Optional) When running over MPI, should each rank write a partial snapshot or do we want a single file? 1 implies one file per MPI rank.
- lustre_OST_count: 0 # (Optional) If > 0, the number of lustre OSTs to distribure the single-striped files over. Has no effect on non-Lustre filesystems. Has an effect only on distributed snapshots.
+ lustre_OST_checks: 0 # (Optional) Perform OST selection checks
+ lustre_OST_free: 0 # (Optional) OST free space requirement, -1 for guess.
+ lustre_OST_test: 0 # (Optional) Check that OSTs are writable.
use_delta_from_edge: 0 # (Optional) Should particles close to the box edge be moved back towards 0 by a vector perpendicular to the box edge? This is useful in cases where lossy compression moves particle beyond the edge.
delta_from_edge: 0. # (Optional) Norm of the vector to use when moving particles away from the edge
UnitMass_in_cgs: 1 # (Optional) Unit system for the outputs (Grams)
@@ -245,7 +247,9 @@ Restarts:
max_run_time: 24.0 # (optional) Maximal wall-clock time in hours. The application will exit when this limit is reached.
resubmit_on_exit: 0 # (Optional) whether to run a command when exiting after the time limit has been reached.
resubmit_command: ./resub.sh # (Optional) Command to run when time limit is reached. Compulsory if resubmit_on_exit is switched on. Note potentially unsafe.
- lustre_OST_count: 0 # (Optional) If > 0, the number of lustre OSTs to distribure the single-striped restart files over. Has no effect on non-Lustre filesystems.
+ lustre_OST_checks: 0 # (Optional) Perform OST selection checks
+ lustre_OST_free: 0 # (Optional) OST free space requirement, -1 for guess.
+ lustre_OST_test: 0 # (Optional) Check that OSTs are writable.
# Parameters governing domain decomposition
DomainDecomposition:
diff --git a/src/Makefile.am b/src/Makefile.am
index cbf6d4f78579a89bada366b7efe81ba6efd7d825..f41b280f1057d5db577bca2e342c5bcae3905c6a 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -16,7 +16,8 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Add the non-standard paths to the included library headers
-AM_CFLAGS = $(HDF5_CPPFLAGS) $(GSL_INCS) $(FFTW_INCS) $(NUMA_INCS) $(GRACKLE_INCS) $(SUNDIALS_INCS) $(CHEALPIX_CFLAGS)
+AM_CFLAGS = $(HDF5_CPPFLAGS) $(GSL_INCS) $(FFTW_INCS) $(NUMA_INCS) \
+ $(GRACKLE_INCS) $(SUNDIALS_INCS) $(CHEALPIX_CFLAGS) $(LUSTREAPI_INCS)
# Assign a "safe" version number
AM_LDFLAGS = $(HDF5_LDFLAGS) $(FFTW_LIBS)
@@ -25,7 +26,9 @@ AM_LDFLAGS = $(HDF5_LDFLAGS) $(FFTW_LIBS)
GIT_CMD = @GIT_CMD@
# Additional dependencies for shared libraries.
-EXTRA_LIBS = $(GSL_LIBS) $(GMP_LIBS) $(HDF5_LIBS) $(FFTW_LIBS) $(NUMA_LIBS) $(PROFILER_LIBS) $(TCMALLOC_LIBS) $(JEMALLOC_LIBS) $(TBBMALLOC_LIBS) $(GRACKLE_LIBS) $(SUNDIALS_LIBS) $(CHEALPIX_LIBS)
+EXTRA_LIBS = $(GSL_LIBS) $(GMP_LIBS) $(HDF5_LIBS) $(FFTW_LIBS) $(NUMA_LIBS) \
+ $(PROFILER_LIBS) $(TCMALLOC_LIBS) $(JEMALLOC_LIBS) $(TBBMALLOC_LIBS) \
+ $(GRACKLE_LIBS) $(SUNDIALS_LIBS) $(CHEALPIX_LIBS) $(LUSTREAPI_LIBS)
# MPI libraries.
MPI_LIBS = $(PARMETIS_LIBS) $(METIS_LIBS) $(MPI_THREAD_LIBS)
@@ -33,7 +36,7 @@ MPI_FLAGS = -DWITH_MPI $(PARMETIS_INCS) $(METIS_INCS)
# Build the libswiftsim library and a convenience library just for the gravity tasks
lib_LTLIBRARIES = libswiftsim.la
-noinst_LTLIBRARIES = libgrav.la
+noinst_LTLIBRARIES = libgrav.la
# Build a MPI-enabled version too?
if HAVEMPI
lib_LTLIBRARIES += libswiftsim_mpi.la
@@ -41,27 +44,27 @@ noinst_LTLIBRARIES += libgrav_mpi.la
endif
# List required headers
-include_HEADERS = space.h runner.h queue.h task.h lock.h cell.h part.h const.h
+include_HEADERS = space.h runner.h queue.h task.h lock.h cell.h part.h const.h
include_HEADERS += cell_hydro.h cell_stars.h cell_grav.h cell_sinks.h cell_black_holes.h cell_rt.h cell_grid.h
-include_HEADERS += engine.h swift.h serial_io.h timers.h debug.h scheduler.h proxy.h parallel_io.h
-include_HEADERS += common_io.h single_io.h distributed_io.h map.h tools.h partition_fixed_costs.h
-include_HEADERS += partition.h clocks.h parser.h physical_constants.h physical_constants_cgs.h potential.h version.h
+include_HEADERS += engine.h swift.h serial_io.h timers.h debug.h scheduler.h proxy.h parallel_io.h
+include_HEADERS += common_io.h single_io.h distributed_io.h map.h tools.h partition_fixed_costs.h
+include_HEADERS += partition.h clocks.h parser.h physical_constants.h physical_constants_cgs.h potential.h version.h
include_HEADERS += hydro_properties.h riemann.h threadpool.h cooling_io.h cooling.h cooling_struct.h cooling_properties.h cooling_debug.h
include_HEADERS += statistics.h memswap.h cache.h runner_doiact_hydro_vec.h runner_doiact_undef.h profiler.h entropy_floor.h
-include_HEADERS += csds.h active.h timeline.h xmf.h gravity_properties.h gravity_derivatives.h
-include_HEADERS += gravity_softened_derivatives.h vector_power.h collectgroup.h hydro_space.h sort_part.h
+include_HEADERS += csds.h active.h timeline.h xmf.h gravity_properties.h gravity_derivatives.h
+include_HEADERS += gravity_softened_derivatives.h vector_power.h collectgroup.h hydro_space.h sort_part.h
include_HEADERS += chemistry.h chemistry_additions.h chemistry_io.h chemistry_struct.h chemistry_debug.h
include_HEADERS += cosmology.h restart.h space_getsid.h utilities.h
-include_HEADERS += cbrt.h exp10.h velociraptor_interface.h swift_velociraptor_part.h output_list.h
+include_HEADERS += cbrt.h exp10.h velociraptor_interface.h swift_velociraptor_part.h output_list.h
include_HEADERS += csds_io.h
include_HEADERS += tracers_io.h tracers.h tracers_triggers.h tracers_struct.h tracers_debug.h
include_HEADERS += star_formation_io.h star_formation_debug.h extra_io.h
include_HEADERS += fof.h fof_struct.h fof_io.h fof_catalogue_io.h
-include_HEADERS += multipole.h multipole_accept.h multipole_struct.h binomial.h integer_power.h sincos.h
-include_HEADERS += star_formation_struct.h star_formation.h star_formation_iact.h
-include_HEADERS += star_formation_logger.h star_formation_logger_struct.h
+include_HEADERS += multipole.h multipole_accept.h multipole_struct.h binomial.h integer_power.h sincos.h
+include_HEADERS += star_formation_struct.h star_formation.h star_formation_iact.h
+include_HEADERS += star_formation_logger.h star_formation_logger_struct.h
include_HEADERS += pressure_floor.h pressure_floor_struct.h pressure_floor_iact.h pressure_floor_debug.h
-include_HEADERS += velociraptor_struct.h velociraptor_io.h random.h memuse.h mpiuse.h memuse_rnodes.h
+include_HEADERS += velociraptor_struct.h velociraptor_io.h random.h memuse.h mpiuse.h memuse_rnodes.h
include_HEADERS += black_holes.h black_holes_iact.h black_holes_io.h black_holes_properties.h black_holes_struct.h black_holes_debug.h
include_HEADERS += feedback.h feedback_new_stars.h feedback_struct.h feedback_properties.h feedback_debug.h feedback_iact.h
include_HEADERS += space_unique_id.h line_of_sight.h io_compression.h
@@ -79,6 +82,7 @@ include_HEADERS += adaptive_softening.h adaptive_softening_iact.h adaptive_softe
include_HEADERS += forcing.h
include_HEADERS += power_spectrum.h
include_HEADERS += ghost_stats.h
+include_HEADERS += swift_lustre_api.h
# source files for EAGLE extra I/O
EAGLE_EXTRA_IO_SOURCES=
@@ -158,32 +162,32 @@ SPHM1RT_RT_SOURCES += rt/SPHM1RT/rt_cooling.c
endif
# Common source files
-AM_SOURCES = space.c space_rebuild.c space_regrid.c space_unique_id.c
-AM_SOURCES += space_sort.c space_split.c space_extras.c space_first_init.c space_init.c
-AM_SOURCES += space_cell_index.c space_recycle.c
-AM_SOURCES += runner_main.c runner_doiact_hydro.c runner_doiact_limiter.c
+AM_SOURCES = space.c space_rebuild.c space_regrid.c space_unique_id.c
+AM_SOURCES += space_sort.c space_split.c space_extras.c space_first_init.c space_init.c
+AM_SOURCES += space_cell_index.c space_recycle.c
+AM_SOURCES += runner_main.c runner_doiact_hydro.c runner_doiact_limiter.c
AM_SOURCES += runner_doiact_stars.c runner_doiact_black_holes.c runner_doiact_sinks.c runner_ghost.c
AM_SOURCES += runner_recv.c runner_pack.c
-AM_SOURCES += runner_sort.c runner_drift.c runner_black_holes.c runner_time_integration.c
+AM_SOURCES += runner_sort.c runner_drift.c runner_black_holes.c runner_time_integration.c
AM_SOURCES += runner_doiact_hydro_vec.c runner_others.c
AM_SOURCES += runner_sinks.c
AM_SOURCES += cell.c cell_convert_part.c cell_drift.c cell_lock.c cell_pack.c cell_split.c
AM_SOURCES += cell_unskip.c cell_grid.c
-AM_SOURCES += engine.c engine_maketasks.c engine_split_particles.c engine_strays.c
+AM_SOURCES += engine.c engine_maketasks.c engine_split_particles.c engine_strays.c
AM_SOURCES += engine_drift.c engine_unskip.c engine_collect_end_of_step.c
-AM_SOURCES += engine_redistribute.c engine_fof.c engine_proxy.c engine_io.c engine_config.c
-AM_SOURCES += queue.c task.c timers.c debug.c scheduler.c proxy.c version.c
-AM_SOURCES += common_io.c common_io_copy.c common_io_cells.c common_io_fields.c
-AM_SOURCES += single_io.c serial_io.c distributed_io.c parallel_io.c
-AM_SOURCES += output_options.c line_of_sight.c restart.c parser.c xmf.c
-AM_SOURCES += kernel_hydro.c tools.c map.c part.c partition.c clocks.c
-AM_SOURCES += physical_constants.c units.c potential.c hydro_properties.c
-AM_SOURCES += threadpool.c cooling.c star_formation.c
+AM_SOURCES += engine_redistribute.c engine_fof.c engine_proxy.c engine_io.c engine_config.c
+AM_SOURCES += queue.c task.c timers.c debug.c scheduler.c proxy.c version.c
+AM_SOURCES += common_io.c common_io_copy.c common_io_cells.c common_io_fields.c
+AM_SOURCES += single_io.c serial_io.c distributed_io.c parallel_io.c
+AM_SOURCES += output_options.c line_of_sight.c restart.c parser.c xmf.c
+AM_SOURCES += kernel_hydro.c tools.c map.c part.c partition.c clocks.c
+AM_SOURCES += physical_constants.c units.c potential.c hydro_properties.c
+AM_SOURCES += threadpool.c cooling.c star_formation.c
AM_SOURCES += hydro.c stars.c sink.c
-AM_SOURCES += statistics.c profiler.c csds.c part_type.c
-AM_SOURCES += gravity_properties.c gravity.c multipole.c
-AM_SOURCES += collectgroup.c hydro_space.c equation_of_state.c io_compression.c
-AM_SOURCES += chemistry.c cosmology.c velociraptor_interface.c
+AM_SOURCES += statistics.c profiler.c csds.c part_type.c
+AM_SOURCES += gravity_properties.c gravity.c multipole.c
+AM_SOURCES += collectgroup.c hydro_space.c equation_of_state.c io_compression.c
+AM_SOURCES += chemistry.c cosmology.c velociraptor_interface.c
AM_SOURCES += output_list.c csds_io.c memuse.c mpiuse.c memuse_rnodes.c
AM_SOURCES += fof.c fof_catalogue_io.c
AM_SOURCES += hashmap.c
@@ -198,157 +202,158 @@ AM_SOURCES += power_spectrum.c
AM_SOURCES += forcing.c
AM_SOURCES += ghost_stats.c
AM_SOURCES += $(EAGLE_EXTRA_IO_SOURCES)
-AM_SOURCES += $(QLA_COOLING_SOURCES) $(QLA_EAGLE_COOLING_SOURCES)
-AM_SOURCES += $(EAGLE_COOLING_SOURCES) $(EAGLE_FEEDBACK_SOURCES)
+AM_SOURCES += $(QLA_COOLING_SOURCES) $(QLA_EAGLE_COOLING_SOURCES)
+AM_SOURCES += $(EAGLE_COOLING_SOURCES) $(EAGLE_FEEDBACK_SOURCES)
AM_SOURCES += $(GRACKLE_COOLING_SOURCES) $(GEAR_FEEDBACK_SOURCES)
AM_SOURCES += $(EAGLE_FLOOR_SOURCES)
AM_SOURCES += $(AGORA_FEEDBACK_SOURCES)
AM_SOURCES += $(PS2020_COOLING_SOURCES)
AM_SOURCES += $(SPHM1RT_RT_SOURCES)
AM_SOURCES += $(GEAR_RT_SOURCES)
+AM_SOURCES += swift_lustre_api.c
# Include files for distribution, not installation.
-nobase_noinst_HEADERS = align.h approx_math.h atomic.h barrier.h cycle.h error.h inline.h kernel_hydro.h kernel_gravity.h
+nobase_noinst_HEADERS = align.h approx_math.h atomic.h barrier.h cycle.h error.h inline.h kernel_hydro.h kernel_gravity.h
nobase_noinst_HEADERS += gravity_iact.h kernel_long_gravity.h vector.h accumulate.h cache.h exp.h log.h
nobase_noinst_HEADERS += runner_doiact_nosort.h runner_doiact_hydro.h runner_doiact_stars.h runner_doiact_black_holes.h runner_doiact_grav.h
-nobase_noinst_HEADERS += runner_doiact_functions_hydro.h runner_doiact_functions_stars.h runner_doiact_functions_black_holes.h
-nobase_noinst_HEADERS += runner_doiact_functions_limiter.h runner_doiact_functions_sinks.h runner_doiact_limiter.h units.h intrinsics.h minmax.h
+nobase_noinst_HEADERS += runner_doiact_functions_hydro.h runner_doiact_functions_stars.h runner_doiact_functions_black_holes.h
+nobase_noinst_HEADERS += runner_doiact_functions_limiter.h runner_doiact_functions_sinks.h runner_doiact_limiter.h units.h intrinsics.h minmax.h
nobase_noinst_HEADERS += runner_doiact_sinks.h
nobase_noinst_HEADERS += kick.h timestep.h drift.h adiabatic_index.h io_properties.h dimension.h part_type.h periodic.h memswap.h
-nobase_noinst_HEADERS += timestep_limiter.h timestep_limiter_iact.h timestep_sync.h timestep_sync_part.h timestep_limiter_struct.h
+nobase_noinst_HEADERS += timestep_limiter.h timestep_limiter_iact.h timestep_sync.h timestep_sync_part.h timestep_limiter_struct.h
nobase_noinst_HEADERS += csds.h sign.h csds_io.h hashmap.h gravity.h gravity_io.h gravity_csds.h gravity_cache.h output_options.h
-nobase_noinst_HEADERS += gravity/Default/gravity.h gravity/Default/gravity_iact.h gravity/Default/gravity_io.h
-nobase_noinst_HEADERS += gravity/Default/gravity_debug.h gravity/Default/gravity_part.h
-nobase_noinst_HEADERS += gravity/MultiSoftening/gravity.h gravity/MultiSoftening/gravity_iact.h gravity/MultiSoftening/gravity_io.h
-nobase_noinst_HEADERS += gravity/MultiSoftening/gravity_debug.h gravity/MultiSoftening/gravity_part.h
-nobase_noinst_HEADERS += gravity/MultiSoftening/gravity_csds.h
-nobase_noinst_HEADERS += equation_of_state.h
+nobase_noinst_HEADERS += gravity/Default/gravity.h gravity/Default/gravity_iact.h gravity/Default/gravity_io.h
+nobase_noinst_HEADERS += gravity/Default/gravity_debug.h gravity/Default/gravity_part.h
+nobase_noinst_HEADERS += gravity/MultiSoftening/gravity.h gravity/MultiSoftening/gravity_iact.h gravity/MultiSoftening/gravity_io.h
+nobase_noinst_HEADERS += gravity/MultiSoftening/gravity_debug.h gravity/MultiSoftening/gravity_part.h
+nobase_noinst_HEADERS += gravity/MultiSoftening/gravity_csds.h
+nobase_noinst_HEADERS += equation_of_state.h
nobase_noinst_HEADERS += equation_of_state/ideal_gas/equation_of_state.h equation_of_state/isothermal/equation_of_state.h equation_of_state/barotropic/equation_of_state.h
nobase_noinst_HEADERS += signal_velocity.h
-nobase_noinst_HEADERS += hydro.h hydro_io.h hydro_csds.h hydro_parameters.h
-nobase_noinst_HEADERS += hydro/None/hydro.h hydro/None/hydro_iact.h hydro/None/hydro_io.h
-nobase_noinst_HEADERS += hydro/None/hydro_debug.h hydro/None/hydro_part.h
-nobase_noinst_HEADERS += hydro/None/hydro_parameters.h
-nobase_noinst_HEADERS += hydro/Minimal/hydro.h hydro/Minimal/hydro_iact.h hydro/Minimal/hydro_io.h
-nobase_noinst_HEADERS += hydro/Minimal/hydro_debug.h hydro/Minimal/hydro_part.h
-nobase_noinst_HEADERS += hydro/Minimal/hydro_parameters.h
-nobase_noinst_HEADERS += hydro/Phantom/hydro.h hydro/Phantom/hydro_iact.h hydro/Phantom/hydro_io.h
-nobase_noinst_HEADERS += hydro/Phantom/hydro_debug.h hydro/Phantom/hydro_part.h
-nobase_noinst_HEADERS += hydro/Phantom/hydro_parameters.h
-nobase_noinst_HEADERS += hydro/Gadget2/hydro.h hydro/Gadget2/hydro_iact.h hydro/Gadget2/hydro_io.h
-nobase_noinst_HEADERS += hydro/Gadget2/hydro_debug.h hydro/Gadget2/hydro_part.h
-nobase_noinst_HEADERS += hydro/Gadget2/hydro_parameters.h hydro/Gadget2/hydro_csds.h
-nobase_noinst_HEADERS += hydro/PressureEntropy/hydro.h hydro/PressureEntropy/hydro_iact.h hydro/PressureEntropy/hydro_io.h
-nobase_noinst_HEADERS += hydro/PressureEntropy/hydro_debug.h hydro/PressureEntropy/hydro_part.h
-nobase_noinst_HEADERS += hydro/PressureEntropy/hydro_parameters.h
-nobase_noinst_HEADERS += hydro/PressureEnergy/hydro.h hydro/PressureEnergy/hydro_iact.h hydro/PressureEnergy/hydro_io.h
-nobase_noinst_HEADERS += hydro/PressureEnergy/hydro_debug.h hydro/PressureEnergy/hydro_part.h
-nobase_noinst_HEADERS += hydro/PressureEnergy/hydro_parameters.h
-nobase_noinst_HEADERS += hydro/PressureEnergyMorrisMonaghanAV/hydro.h hydro/PressureEnergyMorrisMonaghanAV/hydro_iact.h hydro/PressureEnergyMorrisMonaghanAV/hydro_io.h
-nobase_noinst_HEADERS += hydro/PressureEnergyMorrisMonaghanAV/hydro_debug.h hydro/PressureEnergyMorrisMonaghanAV/hydro_part.h
-nobase_noinst_HEADERS += hydro/PressureEnergyMorrisMonaghanAV/hydro_parameters.h
-nobase_noinst_HEADERS += hydro/AnarchyPU/hydro.h hydro/AnarchyPU/hydro_iact.h hydro/AnarchyPU/hydro_io.h
-nobase_noinst_HEADERS += hydro/AnarchyPU/hydro_debug.h hydro/AnarchyPU/hydro_part.h
-nobase_noinst_HEADERS += hydro/AnarchyPU/hydro_parameters.h
-nobase_noinst_HEADERS += hydro/SPHENIX/hydro.h hydro/SPHENIX/hydro_iact.h hydro/SPHENIX/hydro_io.h
+nobase_noinst_HEADERS += hydro.h hydro_io.h hydro_csds.h hydro_parameters.h
+nobase_noinst_HEADERS += hydro/None/hydro.h hydro/None/hydro_iact.h hydro/None/hydro_io.h
+nobase_noinst_HEADERS += hydro/None/hydro_debug.h hydro/None/hydro_part.h
+nobase_noinst_HEADERS += hydro/None/hydro_parameters.h
+nobase_noinst_HEADERS += hydro/Minimal/hydro.h hydro/Minimal/hydro_iact.h hydro/Minimal/hydro_io.h
+nobase_noinst_HEADERS += hydro/Minimal/hydro_debug.h hydro/Minimal/hydro_part.h
+nobase_noinst_HEADERS += hydro/Minimal/hydro_parameters.h
+nobase_noinst_HEADERS += hydro/Phantom/hydro.h hydro/Phantom/hydro_iact.h hydro/Phantom/hydro_io.h
+nobase_noinst_HEADERS += hydro/Phantom/hydro_debug.h hydro/Phantom/hydro_part.h
+nobase_noinst_HEADERS += hydro/Phantom/hydro_parameters.h
+nobase_noinst_HEADERS += hydro/Gadget2/hydro.h hydro/Gadget2/hydro_iact.h hydro/Gadget2/hydro_io.h
+nobase_noinst_HEADERS += hydro/Gadget2/hydro_debug.h hydro/Gadget2/hydro_part.h
+nobase_noinst_HEADERS += hydro/Gadget2/hydro_parameters.h hydro/Gadget2/hydro_csds.h
+nobase_noinst_HEADERS += hydro/PressureEntropy/hydro.h hydro/PressureEntropy/hydro_iact.h hydro/PressureEntropy/hydro_io.h
+nobase_noinst_HEADERS += hydro/PressureEntropy/hydro_debug.h hydro/PressureEntropy/hydro_part.h
+nobase_noinst_HEADERS += hydro/PressureEntropy/hydro_parameters.h
+nobase_noinst_HEADERS += hydro/PressureEnergy/hydro.h hydro/PressureEnergy/hydro_iact.h hydro/PressureEnergy/hydro_io.h
+nobase_noinst_HEADERS += hydro/PressureEnergy/hydro_debug.h hydro/PressureEnergy/hydro_part.h
+nobase_noinst_HEADERS += hydro/PressureEnergy/hydro_parameters.h
+nobase_noinst_HEADERS += hydro/PressureEnergyMorrisMonaghanAV/hydro.h hydro/PressureEnergyMorrisMonaghanAV/hydro_iact.h hydro/PressureEnergyMorrisMonaghanAV/hydro_io.h
+nobase_noinst_HEADERS += hydro/PressureEnergyMorrisMonaghanAV/hydro_debug.h hydro/PressureEnergyMorrisMonaghanAV/hydro_part.h
+nobase_noinst_HEADERS += hydro/PressureEnergyMorrisMonaghanAV/hydro_parameters.h
+nobase_noinst_HEADERS += hydro/AnarchyPU/hydro.h hydro/AnarchyPU/hydro_iact.h hydro/AnarchyPU/hydro_io.h
+nobase_noinst_HEADERS += hydro/AnarchyPU/hydro_debug.h hydro/AnarchyPU/hydro_part.h
+nobase_noinst_HEADERS += hydro/AnarchyPU/hydro_parameters.h
+nobase_noinst_HEADERS += hydro/SPHENIX/hydro.h hydro/SPHENIX/hydro_iact.h hydro/SPHENIX/hydro_io.h
nobase_noinst_HEADERS += hydro/SPHENIX/hydro_debug.h hydro/SPHENIX/hydro_part.h hydro/SPHENIX/hydro_csds.h
nobase_noinst_HEADERS += hydro/SPHENIX/hydro_parameters.h
-nobase_noinst_HEADERS += hydro/Gasoline/hydro.h hydro/Gasoline/hydro_iact.h hydro/Gasoline/hydro_io.h
+nobase_noinst_HEADERS += hydro/Gasoline/hydro.h hydro/Gasoline/hydro_iact.h hydro/Gasoline/hydro_io.h
nobase_noinst_HEADERS += hydro/Gasoline/hydro_debug.h hydro/Gasoline/hydro_part.h
-nobase_noinst_HEADERS += hydro/Gasoline/hydro_parameters.h
-nobase_noinst_HEADERS += hydro/Gizmo/hydro_parameters.h
-nobase_noinst_HEADERS += hydro/Gizmo/hydro_io.h hydro/Gizmo/hydro_debug.h
-nobase_noinst_HEADERS += hydro/Gizmo/hydro.h hydro/Gizmo/hydro_iact.h
-nobase_noinst_HEADERS += hydro/Gizmo/hydro_part.h
-nobase_noinst_HEADERS += hydro/Gizmo/hydro_gradients.h
-nobase_noinst_HEADERS += hydro/Gizmo/hydro_getters.h
-nobase_noinst_HEADERS += hydro/Gizmo/hydro_setters.h
-nobase_noinst_HEADERS += hydro/Gizmo/hydro_flux.h
-nobase_noinst_HEADERS += hydro/Gizmo/hydro_slope_limiters.h
-nobase_noinst_HEADERS += hydro/Gizmo/hydro_slope_limiters_face.h
-nobase_noinst_HEADERS += hydro/Gizmo/hydro_slope_limiters_cell.h
-nobase_noinst_HEADERS += hydro/Gizmo/hydro_unphysical.h
-nobase_noinst_HEADERS += hydro/Gizmo/hydro_gradients_sph.h
-nobase_noinst_HEADERS += hydro/Gizmo/hydro_gradients_gizmo.h
-nobase_noinst_HEADERS += hydro/Gizmo/hydro_velocities.h
-nobase_noinst_HEADERS += hydro/Gizmo/hydro_lloyd.h
-nobase_noinst_HEADERS += hydro/Gizmo/MFV/hydro_debug.h
-nobase_noinst_HEADERS += hydro/Gizmo/MFV/hydro_part.h
-nobase_noinst_HEADERS += hydro/Gizmo/MFV/hydro_velocities.h
-nobase_noinst_HEADERS += hydro/Gizmo/MFV/hydro_flux.h
-nobase_noinst_HEADERS += hydro/Gizmo/MFM/hydro_debug.h
-nobase_noinst_HEADERS += hydro/Gizmo/MFM/hydro_part.h
-nobase_noinst_HEADERS += hydro/Gizmo/MFM/hydro_flux.h
-nobase_noinst_HEADERS += hydro/Gizmo/MFM/hydro_velocities.h
-nobase_noinst_HEADERS += hydro/Shadowswift/hydro_debug.h
+nobase_noinst_HEADERS += hydro/Gasoline/hydro_parameters.h
+nobase_noinst_HEADERS += hydro/Gizmo/hydro_parameters.h
+nobase_noinst_HEADERS += hydro/Gizmo/hydro_io.h hydro/Gizmo/hydro_debug.h
+nobase_noinst_HEADERS += hydro/Gizmo/hydro.h hydro/Gizmo/hydro_iact.h
+nobase_noinst_HEADERS += hydro/Gizmo/hydro_part.h
+nobase_noinst_HEADERS += hydro/Gizmo/hydro_gradients.h
+nobase_noinst_HEADERS += hydro/Gizmo/hydro_getters.h
+nobase_noinst_HEADERS += hydro/Gizmo/hydro_setters.h
+nobase_noinst_HEADERS += hydro/Gizmo/hydro_flux.h
+nobase_noinst_HEADERS += hydro/Gizmo/hydro_slope_limiters.h
+nobase_noinst_HEADERS += hydro/Gizmo/hydro_slope_limiters_face.h
+nobase_noinst_HEADERS += hydro/Gizmo/hydro_slope_limiters_cell.h
+nobase_noinst_HEADERS += hydro/Gizmo/hydro_unphysical.h
+nobase_noinst_HEADERS += hydro/Gizmo/hydro_gradients_sph.h
+nobase_noinst_HEADERS += hydro/Gizmo/hydro_gradients_gizmo.h
+nobase_noinst_HEADERS += hydro/Gizmo/hydro_velocities.h
+nobase_noinst_HEADERS += hydro/Gizmo/hydro_lloyd.h
+nobase_noinst_HEADERS += hydro/Gizmo/MFV/hydro_debug.h
+nobase_noinst_HEADERS += hydro/Gizmo/MFV/hydro_part.h
+nobase_noinst_HEADERS += hydro/Gizmo/MFV/hydro_velocities.h
+nobase_noinst_HEADERS += hydro/Gizmo/MFV/hydro_flux.h
+nobase_noinst_HEADERS += hydro/Gizmo/MFM/hydro_debug.h
+nobase_noinst_HEADERS += hydro/Gizmo/MFM/hydro_part.h
+nobase_noinst_HEADERS += hydro/Gizmo/MFM/hydro_flux.h
+nobase_noinst_HEADERS += hydro/Gizmo/MFM/hydro_velocities.h
+nobase_noinst_HEADERS += hydro/Shadowswift/hydro_debug.h
nobase_noinst_HEADERS += hydro/Shadowswift/hydro.h
-nobase_noinst_HEADERS += hydro/Shadowswift/hydro_iact.h
-nobase_noinst_HEADERS += hydro/Shadowswift/hydro_io.h
+nobase_noinst_HEADERS += hydro/Shadowswift/hydro_iact.h
+nobase_noinst_HEADERS += hydro/Shadowswift/hydro_io.h
nobase_noinst_HEADERS += hydro/Shadowswift/hydro_part.h
-nobase_noinst_HEADERS += hydro/Shadowswift/hydro_parameters.h
+nobase_noinst_HEADERS += hydro/Shadowswift/hydro_parameters.h
nobase_noinst_HEADERS += mhd.h mhd_struct.h mhd_io.h
nobase_noinst_HEADERS += mhd/None/mhd.h mhd/None/mhd_iact.h mhd/None/mhd_struct.h mhd/None/mhd_io.h mhd/None/mhd_debug.h mhd/None/mhd_parameters.h
-nobase_noinst_HEADERS += riemann/riemann_hllc.h riemann/riemann_trrs.h
-nobase_noinst_HEADERS += riemann/riemann_exact.h riemann/riemann_vacuum.h
-nobase_noinst_HEADERS += riemann/riemann_checks.h
-nobase_noinst_HEADERS += rt.h
-nobase_noinst_HEADERS += rt_additions.h
-nobase_noinst_HEADERS += rt_io.h
-nobase_noinst_HEADERS += rt_parameters.h
-nobase_noinst_HEADERS += rt_properties.h
-nobase_noinst_HEADERS += rt_struct.h
-nobase_noinst_HEADERS += rt/none/rt.h
-nobase_noinst_HEADERS += rt/none/rt_additions.h
-nobase_noinst_HEADERS += rt/none/rt_iact.h
-nobase_noinst_HEADERS += rt/none/rt_io.h
+nobase_noinst_HEADERS += riemann/riemann_hllc.h riemann/riemann_trrs.h
+nobase_noinst_HEADERS += riemann/riemann_exact.h riemann/riemann_vacuum.h
+nobase_noinst_HEADERS += riemann/riemann_checks.h
+nobase_noinst_HEADERS += rt.h
+nobase_noinst_HEADERS += rt_additions.h
+nobase_noinst_HEADERS += rt_io.h
+nobase_noinst_HEADERS += rt_parameters.h
+nobase_noinst_HEADERS += rt_properties.h
+nobase_noinst_HEADERS += rt_struct.h
+nobase_noinst_HEADERS += rt/none/rt.h
+nobase_noinst_HEADERS += rt/none/rt_additions.h
+nobase_noinst_HEADERS += rt/none/rt_iact.h
+nobase_noinst_HEADERS += rt/none/rt_io.h
nobase_noinst_HEADERS += rt/none/rt_parameters.h
-nobase_noinst_HEADERS += rt/none/rt_properties.h
+nobase_noinst_HEADERS += rt/none/rt_properties.h
nobase_noinst_HEADERS += rt/none/rt_struct.h
-nobase_noinst_HEADERS += rt/debug/rt.h
-nobase_noinst_HEADERS += rt/debug/rt_additions.h
-nobase_noinst_HEADERS += rt/debug/rt_debugging.h
-nobase_noinst_HEADERS += rt/debug/rt_gradients.h
-nobase_noinst_HEADERS += rt/debug/rt_iact.h
-nobase_noinst_HEADERS += rt/debug/rt_io.h
+nobase_noinst_HEADERS += rt/debug/rt.h
+nobase_noinst_HEADERS += rt/debug/rt_additions.h
+nobase_noinst_HEADERS += rt/debug/rt_debugging.h
+nobase_noinst_HEADERS += rt/debug/rt_gradients.h
+nobase_noinst_HEADERS += rt/debug/rt_iact.h
+nobase_noinst_HEADERS += rt/debug/rt_io.h
nobase_noinst_HEADERS += rt/debug/rt_parameters.h
-nobase_noinst_HEADERS += rt/debug/rt_properties.h
+nobase_noinst_HEADERS += rt/debug/rt_properties.h
nobase_noinst_HEADERS += rt/debug/rt_struct.h
-nobase_noinst_HEADERS += rt/GEAR/rt_additions.h
-nobase_noinst_HEADERS += rt/GEAR/rt_blackbody.h
-nobase_noinst_HEADERS += rt/GEAR/rt_debugging.h
-nobase_noinst_HEADERS += rt/GEAR/rt_flux.h
-nobase_noinst_HEADERS += rt/GEAR/rt_getters.h
+nobase_noinst_HEADERS += rt/GEAR/rt_additions.h
+nobase_noinst_HEADERS += rt/GEAR/rt_blackbody.h
+nobase_noinst_HEADERS += rt/GEAR/rt_debugging.h
+nobase_noinst_HEADERS += rt/GEAR/rt_flux.h
+nobase_noinst_HEADERS += rt/GEAR/rt_getters.h
nobase_noinst_HEADERS += rt/GEAR/rt_grackle_utils.h
nobase_noinst_HEADERS += rt/GEAR/rt_gradients.h
nobase_noinst_HEADERS += rt/GEAR/rt.h
nobase_noinst_HEADERS += rt/GEAR/rt_iact.h
nobase_noinst_HEADERS += rt/GEAR/rt_interaction_cross_sections.h
nobase_noinst_HEADERS += rt/GEAR/rt_interaction_rates.h
-nobase_noinst_HEADERS += rt/GEAR/rt_io.h
-nobase_noinst_HEADERS += rt/GEAR/rt_ionization_equilibrium.h
+nobase_noinst_HEADERS += rt/GEAR/rt_io.h
+nobase_noinst_HEADERS += rt/GEAR/rt_ionization_equilibrium.h
nobase_noinst_HEADERS += rt/GEAR/rt_parameters.h
-nobase_noinst_HEADERS += rt/GEAR/rt_properties.h
-nobase_noinst_HEADERS += rt/GEAR/rt_riemann_GLF.h
+nobase_noinst_HEADERS += rt/GEAR/rt_properties.h
+nobase_noinst_HEADERS += rt/GEAR/rt_riemann_GLF.h
nobase_noinst_HEADERS += rt/GEAR/rt_riemann_HLL_eigenvalues.h
nobase_noinst_HEADERS += rt/GEAR/rt_riemann_HLL.h
nobase_noinst_HEADERS += rt/GEAR/rt_slope_limiters_cell.h
-nobase_noinst_HEADERS += rt/GEAR/rt_slope_limiters_face.h
-nobase_noinst_HEADERS += rt/GEAR/rt_species.h
+nobase_noinst_HEADERS += rt/GEAR/rt_slope_limiters_face.h
+nobase_noinst_HEADERS += rt/GEAR/rt_species.h
nobase_noinst_HEADERS += rt/GEAR/rt_stellar_emission_model.h
nobase_noinst_HEADERS += rt/GEAR/rt_struct.h
nobase_noinst_HEADERS += rt/GEAR/rt_thermochemistry.h
nobase_noinst_HEADERS += rt/GEAR/rt_thermochemistry_utils.h
nobase_noinst_HEADERS += rt/GEAR/rt_unphysical.h
-nobase_noinst_HEADERS += rt/SPHM1RT/rt.h
+nobase_noinst_HEADERS += rt/SPHM1RT/rt.h
nobase_noinst_HEADERS += rt/SPHM1RT/rt_getters.h
nobase_noinst_HEADERS += rt/SPHM1RT/rt_setters.h
nobase_noinst_HEADERS += rt/SPHM1RT/rt_iact.h
-nobase_noinst_HEADERS += rt/SPHM1RT/rt_io.h
+nobase_noinst_HEADERS += rt/SPHM1RT/rt_io.h
nobase_noinst_HEADERS += rt/SPHM1RT/rt_parameters.h
-nobase_noinst_HEADERS += rt/SPHM1RT/rt_properties.h
+nobase_noinst_HEADERS += rt/SPHM1RT/rt_properties.h
nobase_noinst_HEADERS += rt/SPHM1RT/rt_struct.h
-nobase_noinst_HEADERS += rt/SPHM1RT/rt_gradients.h
-nobase_noinst_HEADERS += rt/SPHM1RT/rt_additions.h
+nobase_noinst_HEADERS += rt/SPHM1RT/rt_gradients.h
+nobase_noinst_HEADERS += rt/SPHM1RT/rt_additions.h
nobase_noinst_HEADERS += rt/SPHM1RT/rt_cooling.h
nobase_noinst_HEADERS += rt/SPHM1RT/rt_cooling_rates.h
nobase_noinst_HEADERS += rt/SPHM1RT/rt_unphysical.h
@@ -356,130 +361,130 @@ nobase_noinst_HEADERS += rt/SPHM1RT/rt_species_and_elements.h
nobase_noinst_HEADERS += rt/SPHM1RT/rt_stellar_emission_rate.h
nobase_noinst_HEADERS += shadowswift/voronoi.h
nobase_noinst_HEADERS += stars.h stars_io.h stars_csds.h
-nobase_noinst_HEADERS += stars/None/stars.h stars/None/stars_iact.h stars/None/stars_io.h
+nobase_noinst_HEADERS += stars/None/stars.h stars/None/stars_iact.h stars/None/stars_io.h
nobase_noinst_HEADERS += stars/None/stars_debug.h stars/None/stars_part.h
-nobase_noinst_HEADERS += stars/Basic/stars.h stars/Basic/stars_iact.h stars/Basic/stars_io.h
-nobase_noinst_HEADERS += stars/Basic/stars_debug.h stars/Basic/stars_part.h stars/Basic/stars_csds.h
-nobase_noinst_HEADERS += stars/EAGLE/stars.h stars/EAGLE/stars_iact.h stars/EAGLE/stars_io.h
-nobase_noinst_HEADERS += stars/EAGLE/stars_debug.h stars/EAGLE/stars_part.h
-nobase_noinst_HEADERS += stars/GEAR/stars.h stars/GEAR/stars_iact.h stars/GEAR/stars_io.h
+nobase_noinst_HEADERS += stars/Basic/stars.h stars/Basic/stars_iact.h stars/Basic/stars_io.h
+nobase_noinst_HEADERS += stars/Basic/stars_debug.h stars/Basic/stars_part.h stars/Basic/stars_csds.h
+nobase_noinst_HEADERS += stars/EAGLE/stars.h stars/EAGLE/stars_iact.h stars/EAGLE/stars_io.h
+nobase_noinst_HEADERS += stars/EAGLE/stars_debug.h stars/EAGLE/stars_part.h
+nobase_noinst_HEADERS += stars/GEAR/stars.h stars/GEAR/stars_iact.h stars/GEAR/stars_io.h
nobase_noinst_HEADERS += stars/GEAR/stars_debug.h stars/GEAR/stars_csds.h stars/GEAR/stars_part.h
nobase_noinst_HEADERS += stars/GEAR/stars_stellar_type.h
nobase_noinst_HEADERS += forcing/none/forcing.h forcing/roberts_flow/forcing.h forcing/roberts_flow_acceleration/forcing.h
nobase_noinst_HEADERS += forcing/ABC_flow/forcing.h
-nobase_noinst_HEADERS += potential/none/potential.h potential/point_mass/potential.h
-nobase_noinst_HEADERS += potential/isothermal/potential.h potential/disc_patch/potential.h
-nobase_noinst_HEADERS += potential/sine_wave/potential.h potential/constant/potential.h
-nobase_noinst_HEADERS += potential/hernquist/potential.h potential/nfw/potential.h
-nobase_noinst_HEADERS += potential/nfw_mn/potential.h potential/point_mass_softened/potential.h
-nobase_noinst_HEADERS += star_formation/none/star_formation.h star_formation/none/star_formation_struct.h
+nobase_noinst_HEADERS += potential/none/potential.h potential/point_mass/potential.h
+nobase_noinst_HEADERS += potential/isothermal/potential.h potential/disc_patch/potential.h
+nobase_noinst_HEADERS += potential/sine_wave/potential.h potential/constant/potential.h
+nobase_noinst_HEADERS += potential/hernquist/potential.h potential/nfw/potential.h
+nobase_noinst_HEADERS += potential/nfw_mn/potential.h potential/point_mass_softened/potential.h
+nobase_noinst_HEADERS += star_formation/none/star_formation.h star_formation/none/star_formation_struct.h
nobase_noinst_HEADERS += star_formation/none/star_formation_io.h star_formation/none/star_formation_iact.h
nobase_noinst_HEADERS += star_formation/none/star_formation_csds.h star_formation/none/star_formation_debug.h
-nobase_noinst_HEADERS += star_formation/QLA/star_formation.h star_formation/QLA/star_formation_struct.h
-nobase_noinst_HEADERS += star_formation/QLA/star_formation_io.h star_formation/QLA/star_formation_iact.h
+nobase_noinst_HEADERS += star_formation/QLA/star_formation.h star_formation/QLA/star_formation_struct.h
+nobase_noinst_HEADERS += star_formation/QLA/star_formation_io.h star_formation/QLA/star_formation_iact.h
nobase_noinst_HEADERS += star_formation/QLA/star_formation_debug.h
nobase_noinst_HEADERS += star_formation/EAGLE/star_formation.h star_formation/EAGLE/star_formation_struct.h
-nobase_noinst_HEADERS += star_formation/EAGLE/star_formation_io.h star_formation/EAGLE/star_formation_iact.h
+nobase_noinst_HEADERS += star_formation/EAGLE/star_formation_io.h star_formation/EAGLE/star_formation_iact.h
nobase_noinst_HEADERS += star_formation/EAGLE/star_formation_debug.h
nobase_noinst_HEADERS += star_formation/GEAR/star_formation.h star_formation/GEAR/star_formation_struct.h
-nobase_noinst_HEADERS += star_formation/GEAR/star_formation_io.h star_formation/GEAR/star_formation_iact.h
+nobase_noinst_HEADERS += star_formation/GEAR/star_formation_io.h star_formation/GEAR/star_formation_iact.h
nobase_noinst_HEADERS += star_formation/GEAR/star_formation_csds.h star_formation/GEAR/star_formation_debug.h
nobase_noinst_HEADERS += star_formation/GEAR/star_formation_setters.h
-nobase_noinst_HEADERS += star_formation/EAGLE/star_formation_logger.h star_formation/EAGLE/star_formation_logger_struct.h
-nobase_noinst_HEADERS += star_formation/GEAR/star_formation_logger.h star_formation/GEAR/star_formation_logger_struct.h
-nobase_noinst_HEADERS += star_formation/none/star_formation_logger.h star_formation/none/star_formation_logger_struct.h
-nobase_noinst_HEADERS += cooling/none/cooling.h cooling/none/cooling_struct.h
+nobase_noinst_HEADERS += star_formation/EAGLE/star_formation_logger.h star_formation/EAGLE/star_formation_logger_struct.h
+nobase_noinst_HEADERS += star_formation/GEAR/star_formation_logger.h star_formation/GEAR/star_formation_logger_struct.h
+nobase_noinst_HEADERS += star_formation/none/star_formation_logger.h star_formation/none/star_formation_logger_struct.h
+nobase_noinst_HEADERS += cooling/none/cooling.h cooling/none/cooling_struct.h
nobase_noinst_HEADERS += cooling/none/cooling_io.h cooling/none/cooling_properties.h cooling/none/cooling_debug.h
-nobase_noinst_HEADERS += cooling/const_du/cooling.h cooling/const_du/cooling_struct.h
+nobase_noinst_HEADERS += cooling/const_du/cooling.h cooling/const_du/cooling_struct.h
nobase_noinst_HEADERS += cooling/const_du/cooling_io.h cooling/const_du/cooling_properties.h cooling/const_du/cooling_debug.h
-nobase_noinst_HEADERS += cooling/const_lambda/cooling.h cooling/const_lambda/cooling_struct.h
+nobase_noinst_HEADERS += cooling/const_lambda/cooling.h cooling/const_lambda/cooling_struct.h
nobase_noinst_HEADERS += cooling/const_lambda/cooling_io.h cooling/const_lambda/cooling_properties.h cooling/const_lambda/cooling_debug.h
-nobase_noinst_HEADERS += cooling/grackle/cooling.h cooling/grackle/cooling_struct.h
+nobase_noinst_HEADERS += cooling/grackle/cooling.h cooling/grackle/cooling_struct.h
nobase_noinst_HEADERS += cooling/grackle/cooling_io.h cooling/grackle/cooling_properties.h cooling/grackle/cooling_debug.h
-nobase_noinst_HEADERS += cooling/EAGLE/cooling.h cooling/EAGLE/cooling_struct.h cooling/EAGLE/cooling_tables.h
-nobase_noinst_HEADERS += cooling/EAGLE/cooling_io.h cooling/EAGLE/interpolate.h cooling/EAGLE/cooling_rates.h
+nobase_noinst_HEADERS += cooling/EAGLE/cooling.h cooling/EAGLE/cooling_struct.h cooling/EAGLE/cooling_tables.h
+nobase_noinst_HEADERS += cooling/EAGLE/cooling_io.h cooling/EAGLE/interpolate.h cooling/EAGLE/cooling_rates.h
nobase_noinst_HEADERS += cooling/EAGLE/cooling_properties.h cooling/EAGLE/cooling_debug.h
-nobase_noinst_HEADERS += cooling/QLA_EAGLE/cooling.h cooling/QLA_EAGLE/cooling_struct.h cooling/QLA_EAGLE/cooling_tables.h
-nobase_noinst_HEADERS += cooling/QLA_EAGLE/cooling_io.h cooling/QLA_EAGLE/interpolate.h cooling/QLA_EAGLE/cooling_rates.h
+nobase_noinst_HEADERS += cooling/QLA_EAGLE/cooling.h cooling/QLA_EAGLE/cooling_struct.h cooling/QLA_EAGLE/cooling_tables.h
+nobase_noinst_HEADERS += cooling/QLA_EAGLE/cooling_io.h cooling/QLA_EAGLE/interpolate.h cooling/QLA_EAGLE/cooling_rates.h
nobase_noinst_HEADERS += cooling/QLA_EAGLE/cooling_properties.h cooling/QLA_EAGLE/cooling_debug.h
-nobase_noinst_HEADERS += cooling/QLA/cooling.h cooling/QLA/cooling_struct.h cooling/QLA/cooling_tables.h
-nobase_noinst_HEADERS += cooling/QLA/cooling_io.h cooling/QLA/interpolate.h cooling/QLA/cooling_rates.h
+nobase_noinst_HEADERS += cooling/QLA/cooling.h cooling/QLA/cooling_struct.h cooling/QLA/cooling_tables.h
+nobase_noinst_HEADERS += cooling/QLA/cooling_io.h cooling/QLA/interpolate.h cooling/QLA/cooling_rates.h
nobase_noinst_HEADERS += cooling/QLA/cooling_properties.h cooling/QLA/cooling_debug.h
nobase_noinst_HEADERS += cooling/PS2020/cooling.h cooling/PS2020/cooling_struct.h cooling/PS2020/cooling_subgrid.h
nobase_noinst_HEADERS += cooling/PS2020/cooling_io.h cooling/PS2020/interpolate.h cooling/PS2020/cooling_rates.h
nobase_noinst_HEADERS += cooling/PS2020/cooling_tables.h cooling/PS2020/cooling_subgrid.h
nobase_noinst_HEADERS += cooling/PS2020/cooling_properties.h cooling/PS2020/cooling_debug.h
-nobase_noinst_HEADERS += chemistry/none/chemistry.h
+nobase_noinst_HEADERS += chemistry/none/chemistry.h
nobase_noinst_HEADERS += chemistry/none/chemistry_additions.h
nobase_noinst_HEADERS += chemistry/none/chemistry_io.h
nobase_noinst_HEADERS += chemistry/none/chemistry_csds.h
-nobase_noinst_HEADERS += chemistry/none/chemistry_struct.h
-nobase_noinst_HEADERS += chemistry/none/chemistry_iact.h
-nobase_noinst_HEADERS += chemistry/none/chemistry_debug.h
-nobase_noinst_HEADERS += chemistry/GEAR/chemistry.h
+nobase_noinst_HEADERS += chemistry/none/chemistry_struct.h
+nobase_noinst_HEADERS += chemistry/none/chemistry_iact.h
+nobase_noinst_HEADERS += chemistry/none/chemistry_debug.h
+nobase_noinst_HEADERS += chemistry/GEAR/chemistry.h
nobase_noinst_HEADERS += chemistry/GEAR/chemistry_io.h
nobase_noinst_HEADERS += chemistry/GEAR/chemistry_csds.h
-nobase_noinst_HEADERS += chemistry/GEAR/chemistry_struct.h
-nobase_noinst_HEADERS += chemistry/GEAR/chemistry_iact.h
+nobase_noinst_HEADERS += chemistry/GEAR/chemistry_struct.h
+nobase_noinst_HEADERS += chemistry/GEAR/chemistry_iact.h
nobase_noinst_HEADERS += chemistry/GEAR/chemistry_debug.h
nobase_noinst_HEADERS += chemistry/GEAR_DIFFUSION/chemistry.h
nobase_noinst_HEADERS += chemistry/GEAR_DIFFUSION/chemistry_io.h
nobase_noinst_HEADERS += chemistry/GEAR_DIFFUSION/chemistry_struct.h
nobase_noinst_HEADERS += chemistry/GEAR_DIFFUSION/chemistry_iact.h
nobase_noinst_HEADERS += chemistry/GEAR_DIFFUSION/chemistry_debug.h
-nobase_noinst_HEADERS += chemistry/AGORA/chemistry.h
+nobase_noinst_HEADERS += chemistry/AGORA/chemistry.h
nobase_noinst_HEADERS += chemistry/AGORA/chemistry_io.h
nobase_noinst_HEADERS += chemistry/AGORA/chemistry_csds.h
-nobase_noinst_HEADERS += chemistry/AGORA/chemistry_struct.h
-nobase_noinst_HEADERS += chemistry/AGORA/chemistry_iact.h
-nobase_noinst_HEADERS += chemistry/AGORA/chemistry_debug.h
-nobase_noinst_HEADERS += chemistry/EAGLE/chemistry.h
+nobase_noinst_HEADERS += chemistry/AGORA/chemistry_struct.h
+nobase_noinst_HEADERS += chemistry/AGORA/chemistry_iact.h
+nobase_noinst_HEADERS += chemistry/AGORA/chemistry_debug.h
+nobase_noinst_HEADERS += chemistry/EAGLE/chemistry.h
nobase_noinst_HEADERS += chemistry/EAGLE/chemistry_additions.h
nobase_noinst_HEADERS += chemistry/EAGLE/chemistry_io.h
nobase_noinst_HEADERS += chemistry/EAGLE/chemistry_struct.h
-nobase_noinst_HEADERS += chemistry/EAGLE/chemistry_iact.h
-nobase_noinst_HEADERS += chemistry/EAGLE/chemistry_debug.h
-nobase_noinst_HEADERS += chemistry/QLA/chemistry.h
+nobase_noinst_HEADERS += chemistry/EAGLE/chemistry_iact.h
+nobase_noinst_HEADERS += chemistry/EAGLE/chemistry_debug.h
+nobase_noinst_HEADERS += chemistry/QLA/chemistry.h
nobase_noinst_HEADERS += chemistry/QLA/chemistry_io.h
nobase_noinst_HEADERS += chemistry/QLA/chemistry_struct.h
-nobase_noinst_HEADERS += chemistry/QLA/chemistry_iact.h
+nobase_noinst_HEADERS += chemistry/QLA/chemistry_iact.h
nobase_noinst_HEADERS += chemistry/QLA/chemistry_debug.h
nobase_noinst_HEADERS += entropy_floor/none/entropy_floor.h
-nobase_noinst_HEADERS += entropy_floor/EAGLE/entropy_floor.h
-nobase_noinst_HEADERS += entropy_floor/QLA/entropy_floor.h
-nobase_noinst_HEADERS += tracers/none/tracers.h tracers/none/tracers_struct.h
+nobase_noinst_HEADERS += entropy_floor/EAGLE/entropy_floor.h
+nobase_noinst_HEADERS += entropy_floor/QLA/entropy_floor.h
+nobase_noinst_HEADERS += tracers/none/tracers.h tracers/none/tracers_struct.h
nobase_noinst_HEADERS += tracers/none/tracers_io.h tracers/none/tracers_debug.h
-nobase_noinst_HEADERS += tracers/EAGLE/tracers.h tracers/EAGLE/tracers_struct.h
+nobase_noinst_HEADERS += tracers/EAGLE/tracers.h tracers/EAGLE/tracers_struct.h
nobase_noinst_HEADERS += tracers/EAGLE/tracers_io.h tracers/EAGLE/tracers_debug.h
nobase_noinst_HEADERS += extra_io/EAGLE/extra_io.h extra_io/EAGLE/extra.h
-nobase_noinst_HEADERS += feedback/none/feedback.h feedback/none/feedback_struct.h feedback/none/feedback_iact.h
+nobase_noinst_HEADERS += feedback/none/feedback.h feedback/none/feedback_struct.h feedback/none/feedback_iact.h
nobase_noinst_HEADERS += feedback/none/feedback_properties.h feedback/none/feedback.h
nobase_noinst_HEADERS += feedback/none/feedback_debug.h
nobase_noinst_HEADERS += feedback/AGORA/feedback.h feedback/AGORA/feedback_struct.h feedback/AGORA/feedback_iact.h
nobase_noinst_HEADERS += feedback/AGORA/feedback_properties.h feedback/AGORA/feedback.h
nobase_noinst_HEADERS += feedback/AGORA/feedback_debug.h
nobase_noinst_HEADERS += feedback/EAGLE_kinetic/feedback.h feedback/EAGLE_kinetic/feedback_struct.h
-nobase_noinst_HEADERS += feedback/EAGLE_kinetic/feedback_properties.h feedback/EAGLE_kinetic/feedback_iact.h
+nobase_noinst_HEADERS += feedback/EAGLE_kinetic/feedback_properties.h feedback/EAGLE_kinetic/feedback_iact.h
nobase_noinst_HEADERS += feedback/EAGLE_kinetic/feedback_debug.h
nobase_noinst_HEADERS += feedback/EAGLE_thermal/feedback.h feedback/EAGLE_thermal/feedback_struct.h
-nobase_noinst_HEADERS += feedback/EAGLE_thermal/feedback_properties.h feedback/EAGLE_thermal/feedback_iact.h
+nobase_noinst_HEADERS += feedback/EAGLE_thermal/feedback_properties.h feedback/EAGLE_thermal/feedback_iact.h
nobase_noinst_HEADERS += feedback/EAGLE_thermal/feedback_debug.h
nobase_noinst_HEADERS += feedback/EAGLE/yield_tables.h feedback/EAGLE/imf.h feedback/EAGLE/interpolate.h
nobase_noinst_HEADERS += feedback/EAGLE/enrichment.h
-nobase_noinst_HEADERS += feedback/GEAR/stellar_evolution_struct.h feedback/GEAR/stellar_evolution.h
-nobase_noinst_HEADERS += feedback/GEAR/feedback.h feedback/GEAR/feedback_iact.h
-nobase_noinst_HEADERS += feedback/GEAR/feedback_properties.h feedback/GEAR/feedback_struct.h
-nobase_noinst_HEADERS += feedback/GEAR/initial_mass_function.h feedback/GEAR/supernovae_ia.h feedback/GEAR/supernovae_ii.h
-nobase_noinst_HEADERS += feedback/GEAR/lifetime.h feedback/GEAR/hdf5_functions.h feedback/GEAR/interpolation.h
+nobase_noinst_HEADERS += feedback/GEAR/stellar_evolution_struct.h feedback/GEAR/stellar_evolution.h
+nobase_noinst_HEADERS += feedback/GEAR/feedback.h feedback/GEAR/feedback_iact.h
+nobase_noinst_HEADERS += feedback/GEAR/feedback_properties.h feedback/GEAR/feedback_struct.h
+nobase_noinst_HEADERS += feedback/GEAR/initial_mass_function.h feedback/GEAR/supernovae_ia.h feedback/GEAR/supernovae_ii.h
+nobase_noinst_HEADERS += feedback/GEAR/lifetime.h feedback/GEAR/hdf5_functions.h feedback/GEAR/interpolation.h
nobase_noinst_HEADERS += feedback/GEAR/feedback_debug.h
nobase_noinst_HEADERS += black_holes/Default/black_holes.h black_holes/Default/black_holes_io.h
-nobase_noinst_HEADERS += black_holes/Default/black_holes_part.h black_holes/Default/black_holes_iact.h
-nobase_noinst_HEADERS += black_holes/Default/black_holes_properties.h
+nobase_noinst_HEADERS += black_holes/Default/black_holes_part.h black_holes/Default/black_holes_iact.h
+nobase_noinst_HEADERS += black_holes/Default/black_holes_properties.h
nobase_noinst_HEADERS += black_holes/Default/black_holes_struct.h
nobase_noinst_HEADERS += black_holes/Default/black_holes_debug.h
-nobase_noinst_HEADERS += black_holes/EAGLE/black_holes.h black_holes/EAGLE/black_holes_io.h
-nobase_noinst_HEADERS += black_holes/EAGLE/black_holes_part.h black_holes/EAGLE/black_holes_iact.h
-nobase_noinst_HEADERS += black_holes/EAGLE/black_holes_properties.h black_holes/EAGLE/black_holes_parameters.h
+nobase_noinst_HEADERS += black_holes/EAGLE/black_holes.h black_holes/EAGLE/black_holes_io.h
+nobase_noinst_HEADERS += black_holes/EAGLE/black_holes_part.h black_holes/EAGLE/black_holes_iact.h
+nobase_noinst_HEADERS += black_holes/EAGLE/black_holes_properties.h black_holes/EAGLE/black_holes_parameters.h
nobase_noinst_HEADERS += black_holes/EAGLE/black_holes_struct.h black_holes/EAGLE/black_holes_debug.h
nobase_noinst_HEADERS += black_holes/SPIN_JET/black_holes.h black_holes/SPIN_JET/black_holes_io.h
nobase_noinst_HEADERS += black_holes/SPIN_JET/black_holes_part.h black_holes/SPIN_JET/black_holes_iact.h
@@ -487,8 +492,8 @@ nobase_noinst_HEADERS += black_holes/SPIN_JET/black_holes_properties.h black_hol
nobase_noinst_HEADERS += black_holes/SPIN_JET/black_holes_spin.h black_holes/SPIN_JET/black_holes_struct.h
nobase_noinst_HEADERS += black_holes/SPIN_JET/black_holes_debug.h
nobase_noinst_HEADERS += pressure_floor/GEAR/pressure_floor.h pressure_floor/none/pressure_floor.h
-nobase_noinst_HEADERS += pressure_floor/GEAR/pressure_floor_iact.h pressure_floor/none/pressure_floor_iact.h
-nobase_noinst_HEADERS += pressure_floor/GEAR/pressure_floor_struct.h pressure_floor/none/pressure_floor_struct.h
+nobase_noinst_HEADERS += pressure_floor/GEAR/pressure_floor_iact.h pressure_floor/none/pressure_floor_iact.h
+nobase_noinst_HEADERS += pressure_floor/GEAR/pressure_floor_struct.h pressure_floor/none/pressure_floor_struct.h
nobase_noinst_HEADERS += pressure_floor/GEAR/pressure_floor_debug.h pressure_floor/none/pressure_floor_debug.h
nobase_noinst_HEADERS += sink/Default/sink.h sink/Default/sink_io.h sink/Default/sink_part.h sink/Default/sink_properties.h
nobase_noinst_HEADERS += sink/Default/sink_iact.h sink/Default/sink_struct.h sink/Default/sink_debug.h
@@ -502,9 +507,9 @@ nobase_noinst_HEADERS += neutrino/Default/neutrino.h neutrino/Default/relativity
nobase_noinst_HEADERS += neutrino/Default/neutrino_properties.h neutrino/Default/neutrino_io.h
nobase_noinst_HEADERS += neutrino/Default/neutrino_response.h
nobase_noinst_HEADERS += fvpm_geometry.h fvpm_geometry_struct.h
-nobase_noinst_HEADERS += fvpm_geometry/None/fvpm_geometry.h fvpm_geometry/None/fvpm_geometry_struct.h
-nobase_noinst_HEADERS += fvpm_geometry/Gizmo/fvpm_geometry.h fvpm_geometry/Gizmo/fvpm_geometry_struct.h
-nobase_noinst_HEADERS += fvpm_geometry/Gizmo/MFM/fvpm_geometry.h fvpm_geometry/Gizmo/MFV/fvpm_geometry.h
+nobase_noinst_HEADERS += fvpm_geometry/None/fvpm_geometry.h fvpm_geometry/None/fvpm_geometry_struct.h
+nobase_noinst_HEADERS += fvpm_geometry/Gizmo/fvpm_geometry.h fvpm_geometry/Gizmo/fvpm_geometry_struct.h
+nobase_noinst_HEADERS += fvpm_geometry/Gizmo/MFM/fvpm_geometry.h fvpm_geometry/Gizmo/MFV/fvpm_geometry.h
# Sources and special flags for the gravity library
libgrav_la_SOURCES = runner_doiact_grav.c
diff --git a/src/distributed_io.c b/src/distributed_io.c
index 2634111d29792953867317a742e13e06633d1dfc..45500fdf0f801ab4e64198b7e6356cdead41c75d 100644
--- a/src/distributed_io.c
+++ b/src/distributed_io.c
@@ -56,6 +56,7 @@
#include "sink_io.h"
#include "star_formation_io.h"
#include "stars_io.h"
+#include "swift_lustre_api.h"
#include "tools.h"
#include "units.h"
#include "version.h"
@@ -992,20 +993,47 @@ void write_output_distributed(struct engine* e,
};
/* Use a single Lustre stripe with a rank-based OST offset? */
- if (e->snapshot_lustre_OST_count != 0) {
-
- /* Use a random offset to avoid placing things in the same OSTs. We do
- * this to keep the use of OSTs balanced, much like using -1 for the
- * stripe. */
- int offset = rand() % e->snapshot_lustre_OST_count;
- MPI_Bcast(&offset, 1, MPI_INT, 0, MPI_COMM_WORLD);
-
- char string[1200];
- sprintf(string, "lfs setstripe -c 1 -i %d %s",
- ((e->nodeID + offset) % e->snapshot_lustre_OST_count), fileName);
- const int result = system(string);
- if (result != 0) {
- message("lfs setstripe command returned error code %d", result);
+ if (e->snapshot_lustre_OST_checks != 0) {
+
+ /* Gather information about the current state of the OSTs. */
+ struct swift_ost_store ost_infos;
+
+ /* Select good OSTs sorted by free space. */
+ if (e->nodeID == 0) {
+ swift_ost_select(&ost_infos, fileName, e->snapshot_lustre_OST_free,
+ e->snapshot_lustre_OST_test, e->verbose);
+ }
+
+ /* Distribute the OST information. */
+ MPI_Bcast(&ost_infos, sizeof(struct swift_ost_store), MPI_BYTE, 0,
+ MPI_COMM_WORLD);
+
+ /* Need to make space for the OSTs and copy those locally. If the count is
+ * zero this is probably not a lustre mount. */
+ if (ost_infos.count > 0) {
+ if (e->nodeID != 0) swift_ost_store_alloc(&ost_infos, ost_infos.size);
+ MPI_Bcast(ost_infos.infos, sizeof(struct swift_ost_info) * ost_infos.size,
+ MPI_BYTE, 0, MPI_COMM_WORLD);
+
+ /* We now know how many OSTs are available, each rank should attempt to
+ * use a different one, but overtime we should try not to use the same
+ * ones. Culling will order things by free space so we should get some
+ * reordering of those if we do this process each time. */
+ int dummy = e->nodeID;
+ int offset = swift_ost_next(&ost_infos, &dummy, 1);
+
+ /* And create the file with a stripe of 1 on the OST. */
+ const int result = swift_create_striped_file(fileName, offset, 1, &dummy);
+ if (result != 0) message("failed to set stripe of snapshot");
+
+ /* Finished with this. */
+ swift_ost_store_free(&ost_infos);
+ } else if (e->nodeID == 0) {
+ swift_ost_store_free(&ost_infos);
+
+ /* Don't try this again until next launch. */
+ e->snapshot_lustre_OST_checks = 0;
+ message("Disabling further lustre OST checks");
}
}
diff --git a/src/engine.c b/src/engine.c
index daf491b0b9b56e73d53cf6965c8f4c71d4bcb3cb..effbe577ac2cb94bde8e9676f14b6f5cb38c2e1c 100644
--- a/src/engine.c
+++ b/src/engine.c
@@ -3477,8 +3477,12 @@ void engine_init(
parser_get_opt_param_int(params, "Snapshots:compression", 0);
e->snapshot_distributed =
parser_get_opt_param_int(params, "Snapshots:distributed", 0);
- e->snapshot_lustre_OST_count =
- parser_get_opt_param_int(params, "Snapshots:lustre_OST_count", 0);
+ e->snapshot_lustre_OST_checks =
+ parser_get_opt_param_int(params, "Snapshots:lustre_OST_checks", 0);
+ e->snapshot_lustre_OST_free =
+ parser_get_opt_param_int(params, "Snapshots:lustre_OST_free", 0);
+ e->snapshot_lustre_OST_test =
+ parser_get_opt_param_int(params, "Snapshots:lustre_OST_test", 0);
e->snapshot_invoke_stf =
parser_get_opt_param_int(params, "Snapshots:invoke_stf", 0);
e->snapshot_invoke_fof =
diff --git a/src/engine.h b/src/engine.h
index df4e8b534c619cf98c9eb0e3b98b9a4e229fcdfc..b0143a7d1ab541a1583d4b01e6ab16d7073466bf 100644
--- a/src/engine.h
+++ b/src/engine.h
@@ -350,7 +350,9 @@ struct engine {
float snapshot_subsample_fraction[swift_type_count];
int snapshot_run_on_dump;
int snapshot_distributed;
- int snapshot_lustre_OST_count;
+ int snapshot_lustre_OST_checks;
+ int snapshot_lustre_OST_free;
+ int snapshot_lustre_OST_test;
int snapshot_compression;
int snapshot_invoke_stf;
int snapshot_invoke_fof;
@@ -589,8 +591,16 @@ struct engine {
/* Whether to dump restart files after the last step. */
int restart_onexit;
- /* Number of Lustre OSTs on the system to use as rank-based striping offset */
- int restart_lustre_OST_count;
+ /* Perform OST checks and assign each restart file a stripe on the basis of
+ * most free space first. */
+ int restart_lustre_OST_checks;
+
+ /* Free space that an OST should have to be used, -1 makes this
+ * the rss size. In MiB so we can use an int and human sized. */
+ int restart_lustre_OST_free;
+
+ /* Whether to check is OSTs are writable, if not then they are not used. */
+ int restart_lustre_OST_test;
/* Do we free the foreign data before writing restart files? */
int free_foreign_when_dumping_restart;
diff --git a/src/engine_config.c b/src/engine_config.c
index c3aa430f9c4103f9614187a8e4c23cc5e56da59d..bf5bef510ebe1ddaab3be36e0b4c18d4d6c4dd37 100644
--- a/src/engine_config.c
+++ b/src/engine_config.c
@@ -831,9 +831,13 @@ void engine_config(int restart, int fof, struct engine *e,
* on restart. */
e->restart_onexit = parser_get_opt_param_int(params, "Restarts:onexit", 0);
- /* Read the number of Lustre OSTs to distribute the restart files over */
- e->restart_lustre_OST_count =
- parser_get_opt_param_int(params, "Restarts:lustre_OST_count", 0);
+ /* Lustre OST options. Disabled by default. */
+ e->restart_lustre_OST_checks =
+ parser_get_opt_param_int(params, "Restarts:lustre_OST_checks", 0);
+ e->restart_lustre_OST_free =
+ parser_get_opt_param_int(params, "Restarts:lustre_OST_free", 0);
+ e->restart_lustre_OST_test =
+ parser_get_opt_param_int(params, "Restarts:lustre_OST_test", 0);
/* Hours between restart dumps. Can be changed on restart. */
float dhours =
diff --git a/src/restart.c b/src/restart.c
index e3511bc48828b7fa952441213d169031628d578a..be23734f9b66d97fd0857d46729085289bd8b2bf 100644
--- a/src/restart.c
+++ b/src/restart.c
@@ -29,6 +29,7 @@
#include "engine.h"
#include "error.h"
#include "restart.h"
+#include "swift_lustre_api.h"
#include "version.h"
#include <errno.h>
@@ -133,24 +134,54 @@ void restart_write(struct engine *e, const char *filename) {
/* Save a backup the existing restart file, if requested. */
if (e->restart_save) restart_save_previous(filename);
- /* Use a single Lustre stripe with a rank-based OST offset? */
- if (e->restart_lustre_OST_count != 0) {
-
- /* Use a random offset to avoid placing things in the same OSTs. We do
- * this to keep the use of OSTs balanced, much like using -1 for the
- * stripe. */
- int offset = rand() % e->restart_lustre_OST_count;
#ifdef WITH_MPI
- MPI_Bcast(&offset, 1, MPI_INT, 0, MPI_COMM_WORLD);
-#endif
- char string[1200];
- sprintf(string, "lfs setstripe -c 1 -i %d %s",
- ((e->nodeID + offset) % e->restart_lustre_OST_count), filename);
- const int result = system(string);
- if (result != 0) {
- message("lfs setstripe command returned error code %d", result);
+ /* Attempt to use lustre OSTs intelligently so we avoid issues with full
+ * OSTs, OSTs that are not writable and making sure we only write restart
+ * files using one stripe. Only relevant for MPI. */
+ if (e->restart_lustre_OST_checks != 0) {
+
+ /* Gather information about the current state of the OSTs. */
+ struct swift_ost_store ost_infos;
+
+ /* Select good OSTs sorted by free space. */
+ if (e->nodeID == 0) {
+ swift_ost_select(&ost_infos, filename, e->restart_lustre_OST_free,
+ e->restart_lustre_OST_test, e->verbose);
+ }
+
+ /* Distribute the OST information. */
+ MPI_Bcast(&ost_infos, sizeof(struct swift_ost_store), MPI_BYTE, 0,
+ MPI_COMM_WORLD);
+
+ /* Need to make space for the OSTs and copy those locally. If the count is
+ * zero this is probably not a lustre mount. */
+ if (ost_infos.count > 0) {
+ if (e->nodeID != 0) swift_ost_store_alloc(&ost_infos, ost_infos.size);
+ MPI_Bcast(ost_infos.infos, sizeof(struct swift_ost_info) * ost_infos.size,
+ MPI_BYTE, 0, MPI_COMM_WORLD);
+
+ /* We now know how many OSTs are available, each rank should attempt to
+ * use a different one, but overtime we should try not to use the same
+ * ones. Culling will order things by free space so we should get some
+ * reordering of those if we do this process each time. */
+ int dummy = e->nodeID;
+ int offset = swift_ost_next(&ost_infos, &dummy, 1);
+
+ /* And create the file with a stripe of 1 on the OST. */
+ const int result = swift_create_striped_file(filename, offset, 1, &dummy);
+ if (result != 0) message("failed to set stripe of restart file");
+
+ /* Finished with this. */
+ swift_ost_store_free(&ost_infos);
+ } else if (e->nodeID == 0) {
+ swift_ost_store_free(&ost_infos);
+
+ /* Don't try this again until next launch. */
+ e->snapshot_lustre_OST_checks = 0;
+ message("Disabling further lustre OST checks");
}
}
+#endif /* Lustre OST checks. */
FILE *stream = fopen(filename, "w");
if (stream == NULL)
diff --git a/src/swift_lustre_api.c b/src/swift_lustre_api.c
new file mode 100644
index 0000000000000000000000000000000000000000..93dc8b5d0b1e77c9a27f6f0b2336d9f82f442848
--- /dev/null
+++ b/src/swift_lustre_api.c
@@ -0,0 +1,555 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2025 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+/* Config parameters. */
+#include <config.h>
+
+/* Standard includes. */
+#include <errno.h>
+#include <libgen.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+/* Local includes. */
+#include "error.h"
+#include "swift_lustre_api.h"
+
+/* Lustre API */
+#ifdef HAVE_LUSTREAPI
+#include <lustre/lustre_user.h>
+#include <lustre/lustreapi.h>
+#endif
+
+/* Number of OSTs to pre-allocate space for. */
+#define PREALLOC (100)
+
+/* Bytes in a TiB */
+#define TiB (1024.0 * 1024.0 * 1024.0)
+
+/* Bytes in a MiB */
+#define MiB (1024.0 * 1024.0)
+
+/**
+ * @brief Allocate storage for a number of OSTs in a OST scan storage struct.
+ *
+ * Note does not reset count or fullcount. Zero these if you want an
+ * empty struct.
+ *
+ * @param ost_infos pointer to the storage structure.
+ * @param size number of OSTs to make space for.
+ */
+void swift_ost_store_alloc(struct swift_ost_store *ost_infos, int size) {
+#ifdef HAVE_LUSTREAPI
+ ost_infos->size = size;
+ ost_infos->infos =
+ (struct swift_ost_info *)malloc(sizeof(struct swift_ost_info) * size);
+ if (ost_infos->infos == NULL)
+ error("Failed to allocate space for an OST scan");
+ memset(ost_infos->infos, 0, sizeof(struct swift_ost_info) * size);
+#endif
+}
+
+/**
+ * @brief Create a copy of an OST scan storage struct.
+ *
+ * @param ost_infos_src pointer to the storage structure to copy
+ * @param ost_infos_dst pointer to a storage structure to populate with
+ * the copy. Assumed to have no OST space so not
+ * used or initialized.
+ */
+void swift_ost_store_copy(struct swift_ost_store *ost_infos_src,
+ struct swift_ost_store *ost_infos_dst) {
+#ifdef HAVE_LUSTREAPI
+ ost_infos_dst->size = ost_infos_src->fullcount; /* Used size. */
+ ost_infos_dst->count = ost_infos_src->count;
+ ost_infos_dst->fullcount = ost_infos_src->fullcount;
+ ost_infos_dst->infos = (struct swift_ost_info *)malloc(
+ sizeof(struct swift_ost_info) * ost_infos_dst->size);
+ if (ost_infos_dst->infos == NULL)
+ error("Failed to allocate space for an OST scan copy");
+ memcpy(ost_infos_dst->infos, ost_infos_src->infos,
+ sizeof(struct swift_ost_info) * ost_infos_dst->size);
+#endif
+}
+
+/**
+ * @brief Initialize an OST scan storage structure.
+ *
+ * @param ost_infos pointer to the storage structure.
+ */
+void swift_ost_store_init(struct swift_ost_store *ost_infos) {
+#ifdef HAVE_LUSTREAPI
+ swift_ost_store_alloc(ost_infos, PREALLOC);
+ ost_infos->count = 0;
+ ost_infos->fullcount = 0;
+#endif
+}
+
+/**
+ * @brief Release any storage associated with an OST scan storage structure.
+ *
+ * @param ost_infos pointer to the storage structure.
+ */
+void swift_ost_store_free(struct swift_ost_store *ost_infos) {
+#ifdef HAVE_LUSTREAPI
+ free(ost_infos->infos);
+ ost_infos->infos = NULL;
+ ost_infos->count = 0;
+ ost_infos->fullcount = 0;
+ ost_infos->size = 0;
+#endif
+}
+
+/**
+ * @brief Write about an OST storage structure to a given FILE.
+ *
+ * @param file FILE stream to write output to.
+ * @param ost_infos pointer to the storage structure.
+ */
+void swift_ost_store_write(FILE *file, struct swift_ost_store *ost_infos) {
+#ifdef HAVE_LUSTREAPI
+ fprintf(file, "# %5s %21s %21s %21s\n", "Index", "Size (MiB)", "Used (MiB)",
+ "Free (MiB)");
+ size_t ssum = 0;
+ size_t usum = 0;
+ size_t smin = ost_infos->infos[0].size;
+ size_t smax = 0;
+ size_t umin = ost_infos->infos[0].used;
+ size_t umax = 0;
+
+ for (int i = 0; i < ost_infos->count; i++) {
+ int msize = (int)(ost_infos->infos[i].size / MiB);
+ int mused = (int)(ost_infos->infos[i].used / MiB);
+ fprintf(file, "# %5d %21d %21d %21d\n", ost_infos->infos[i].index, msize,
+ mused, msize - mused);
+
+ ssum += ost_infos->infos[i].size;
+ usum += ost_infos->infos[i].used;
+
+ if (ost_infos->infos[i].size > smax) smax = ost_infos->infos[i].size;
+ if (ost_infos->infos[i].size < smin) smin = ost_infos->infos[i].size;
+
+ if (ost_infos->infos[i].used > umax) umax = ost_infos->infos[i].used;
+ if (ost_infos->infos[i].used < umin) umin = ost_infos->infos[i].used;
+ }
+ if (ost_infos->count == ost_infos->fullcount) {
+ /* Size is for used OSTs not all, so don't report as misleading. */
+ fprintf(file,
+ "# Filesystem size:%.2f TiB used:%.2f TiB free:%.2f TiB %.2f%%\n",
+ ssum / TiB, usum / TiB, (ssum - usum) / TiB,
+ 100.0 * (double)(ssum - usum) / (double)ssum);
+ fprintf(file, "# Min/max size: %.2f/%.2f TiB Min/max used: %.2f/%.2f TiB\n",
+ smin / TiB, smax / TiB, umin / TiB, umax / TiB);
+ } else {
+ fprintf(file, "#\n");
+ }
+#endif
+}
+
+/**
+ * @brief Print information about OST storage structure
+ *
+ * @param ost_infos pointer to the storage structure.
+ * @param verbose if non zero additional information will be written
+ * to stdout.
+ */
+void swift_ost_store_print(struct swift_ost_store *ost_infos, int verbose) {
+#ifdef HAVE_LUSTREAPI
+ message("# OSTs, using %d of %d", ost_infos->count, ost_infos->fullcount);
+ if (verbose) swift_ost_store_write(stdout, ost_infos);
+#endif
+}
+
+#ifdef HAVE_LUSTREAPI
+/**
+ * @brief Store information about an OST.
+ *
+ * @param ost_infos pointer to the storage structure.
+ * @param index the index, zero based.
+ * @param size the total size in bytes.
+ * @param used the number of bytes used.
+ */
+static void swift_ost_store(struct swift_ost_store *ost_infos, int index,
+ size_t size, size_t used) {
+ /* Add extra space if needed. Note not thread safe. */
+ if (ost_infos->fullcount == ost_infos->size - 1) {
+ size_t newsize = ost_infos->size + PREALLOC;
+ struct swift_ost_info *newinfos = (struct swift_ost_info *)malloc(
+ sizeof(struct swift_ost_info) * newsize);
+ if (newinfos == NULL) error("Failed to allocate space for OST information");
+ memset(newinfos, 0, sizeof(struct swift_ost_info) * newsize);
+ memcpy(newinfos, ost_infos->infos,
+ sizeof(struct swift_ost_info) * ost_infos->size);
+ free(ost_infos->infos);
+ ost_infos->infos = newinfos;
+ ost_infos->size = newsize;
+ }
+ int count = ost_infos->count++;
+ ost_infos->infos[count].index = index;
+ ost_infos->infos[count].size = size;
+ ost_infos->infos[count].used = used;
+ ost_infos->fullcount = ost_infos->count;
+}
+#endif
+
+/**
+ * @brief Scan the OSTs associated with a lustre file system given a path.
+ *
+ * On exit the ost_infos struct will be populated with the
+ * the number of OSTs found and details of the size and used bytes in each
+ * OST.
+ *
+ * @param path a directory on the lustre file system, ideally the mount point.
+ * @param ost_infos pointer to the storage structure.
+ *
+ * @return 0 on success, otherwise an error will have been reported to stdout.
+ * If an error occurs the store will never be changed.
+ */
+int swift_ost_scan(const char *path, struct swift_ost_store *ost_infos) {
+
+ int rc = 0;
+#ifdef HAVE_LUSTREAPI
+ char mntdir[PATH_MAX] = {0};
+ char fsname[PATH_MAX] = {0};
+ char cpath[PATH_MAX] = {0};
+
+ /* Check this path exists. */
+ if (!realpath(path, cpath)) {
+ rc = errno;
+ message("Not a filesystem path '%s': %s", path, strerror(rc));
+ } else {
+
+ /* Parse the path into the mount point and file system name. */
+ if (llapi_search_mounts(cpath, 0, mntdir, fsname) == 0) {
+ if (mntdir[0] != '\0') {
+ struct obd_statfs stat_buf;
+ struct obd_uuid uuid_buf;
+
+ /* Loop while OSTs are located. */
+ for (int index = 0;; index++) {
+ memset(&stat_buf, 0, sizeof(struct obd_statfs));
+ memset(&uuid_buf, 0, sizeof(struct obd_uuid));
+
+ rc = llapi_obd_statfs(mntdir, LL_STATFS_LOV, index, &stat_buf,
+ &uuid_buf);
+ rc = -rc;
+ if (rc == ENODEV || rc == EAGAIN || rc == EINVAL || rc == EFAULT) {
+ /* Nothing we can query here, so time to stop search. */
+ break;
+ }
+
+ /* Inactive devices are empty. */
+ if (rc == ENODATA) {
+ swift_ost_store(ost_infos, index, 0, 0);
+ } else {
+ size_t used =
+ (stat_buf.os_blocks - stat_buf.os_bfree) * stat_buf.os_bsize;
+ size_t total = stat_buf.os_blocks * stat_buf.os_bsize;
+ swift_ost_store(ost_infos, index, total, used);
+ }
+ }
+ rc = 0;
+
+ } else {
+ message("No lustre mount point found for path: %s", path);
+ rc = 1;
+ }
+ } else {
+ message("Failed to locate a lustre mount point using path: %s", path);
+ rc = 1;
+ }
+ }
+#endif
+ return rc;
+}
+
+#ifdef HAVE_LUSTREAPI
+/** Comparison function for OST free space. */
+static int ostcmp(const void *p1, const void *p2) {
+ const struct swift_ost_info *i1 = (const struct swift_ost_info *)p1;
+ const struct swift_ost_info *i2 = (const struct swift_ost_info *)p2;
+
+ /* size_t ints so some care is needed to return an int. */
+ size_t f1 = i1->size - i1->used;
+ size_t f2 = i2->size - i2->used;
+ if (f1 < f2) return 1;
+ if (f1 > f2) return -1;
+ return 0;
+}
+#endif
+
+/**
+ * @brief Sort the OSTs into decreasing free space culling those that do not
+ * meet a free space threshold.
+ *
+ * @param ost_infos pointer to populated storage structure.
+ * @param minfree the number of MiB that the OST should be capable of
+ * storing. Zero for no effect.
+ */
+void swift_ost_cull(struct swift_ost_store *ost_infos, int minfree) {
+#ifdef HAVE_LUSTREAPI
+ /* Sort by free space. */
+ qsort(ost_infos->infos, ost_infos->count, sizeof(struct swift_ost_info),
+ ostcmp);
+
+ /* And cull if needed. */
+ if (minfree > 0) {
+ size_t bytesfree = minfree * (size_t)MiB;
+
+ /* Always keep at least one! */
+ for (int i = 1; i < ost_infos->count; i++) {
+ struct swift_ost_info *curr = &ost_infos->infos[i];
+ if ((curr->size - curr->used) < bytesfree) {
+
+ /* Throw the rest away. Note fullcount now decoupled. */
+ ost_infos->count = i;
+ }
+ }
+ }
+#endif
+}
+
+/**
+ * @brief Get the next OST in an incrementing sequence.
+ *
+ * @param ost_infos pointer to populated storage structure.
+ * @param arrayindex the last used array index, start with 0.
+ * This will be wrapped as needed use as input for next
+ * call.
+ * @param count number of OSTs that will be used to stripe, that is the
+ * increment, usually 1. Only makes sense if the OST list is not
+ * culled as this implicitly assumes OSTs are in index order.
+ * @return the selected OST index.
+ */
+int swift_ost_next(struct swift_ost_store *ost_infos, int *arrayindex,
+ int count) {
+#ifdef HAVE_LUSTREAPI
+ int index = (*arrayindex % ost_infos->count);
+ *arrayindex = index + count;
+ return ost_infos->infos[index].index;
+#else
+ return 0;
+#endif
+}
+
+/**
+ * @brief Remove an OST by index from the store.
+ *
+ * @param ost_infos pointer to populated storage structure.
+ * @param index index of the OST to remove.
+ */
+void swift_ost_remove(struct swift_ost_store *ost_infos, int index) {
+
+#ifdef HAVE_LUSTREAPI
+ /* Find the array index. */
+ int arrayindex = -1;
+ for (int i = 0; i < ost_infos->fullcount; i++) {
+ if (ost_infos->infos[i].index == index) {
+ arrayindex = i;
+ break;
+ }
+ }
+
+ /* Do nothing if not found or we have the end array index. */
+ if ((arrayindex != -1) && arrayindex != (ost_infos->fullcount - 1)) {
+
+ /* Copy remaining infos down one place. Overlapping.. */
+ memmove(&ost_infos->infos[arrayindex], &ost_infos->infos[arrayindex + 1],
+ (ost_infos->fullcount - arrayindex - 1) *
+ sizeof(struct swift_ost_info));
+ if (arrayindex < ost_infos->count) ost_infos->count = ost_infos->count - 1;
+ ost_infos->fullcount = ost_infos->fullcount - 1;
+
+ } else if (arrayindex == ost_infos->fullcount - 1) {
+
+ /* End array index, just adjust counts. */
+ if (arrayindex < ost_infos->count) ost_infos->count = ost_infos->count - 1;
+ ost_infos->fullcount = ost_infos->fullcount - 1;
+ }
+#endif
+}
+
+/**
+ * @brief Create a file with a given OST index and number of OSTs to stripe.
+ *
+ * @param filename name of the file to create.
+ * @param offset index of the first OST used with this file.
+ * @param count number of OSTs to stripe this file over.
+ * @param usedoffset the offset actually used by file.
+ *
+ * @return non-zero if there are problems creating the file.
+ */
+int swift_create_striped_file(const char *filename, int offset, int count,
+ int *usedoffset) {
+ int rc = 0;
+
+#ifdef HAVE_LUSTREAPI
+ *usedoffset = offset;
+ rc = llapi_file_create(filename, 0 /* Default block size */, offset, count,
+ LLAPI_LAYOUT_RAID0 /* Pattern default */);
+ if (rc != 0) {
+ rc = -rc;
+ message("Cannot create file %s : %s", filename, strerror(rc));
+ } else {
+
+ /* Recover the file offset of first OST in case it is changed from
+ * operational reasons. */
+ /* Yuk, needs extra space for array os lov_user_ost_data. */
+ size_t sizelum = sizeof(struct lov_user_md) +
+ LOV_MAX_STRIPE_COUNT * sizeof(struct lov_user_ost_data);
+ struct lov_user_md *lum = (struct lov_user_md *)malloc(sizelum);
+
+ rc = llapi_file_get_stripe(filename, lum);
+ rc = -rc;
+ if (rc == 0) {
+ *usedoffset = lum->lmm_objects[0].l_ost_idx;
+ } else {
+ /* Shouldn't be fatal. */
+ *usedoffset = offset;
+ }
+ free(lum);
+ }
+#endif
+ return rc;
+}
+
+/**
+ * @brief Scan for the available OSTs for a given file path.
+ *
+ * The OSTs will be sorted by free space on exit and may be further selected
+ * to remove OSTs that are too full for use or cannot be written to. If too
+ * many OSTs are rejected it will be considered to be a parameter error and
+ * all OSTs, sorted by free space, will be returned.
+ * We don't want to flood the OSTs with RPC calls so only one MPI rank
+ * should make this call.
+ *
+ * @param ost_infos pointer to empty OST storage struct. Will contain the
+ * selected free-space ordered OSTs found on exit. The OST
+ * count will remain at zero if anything fails. Note this
+ * will need to be freed as usual regardless.
+ * @param filepath path to a file on the lustre file system. Must not exist.
+ * The containing directory must exist and be part of the
+ * lustre file system.
+ * @param minfree minimum free space to allow in MiB. -1 for use a guess based
+ * on size of the current process, 0 to disable selection.
+ * @param writetest whether to check if the OSTs are writable. If used
+ * the path must be that of a non-existent file on the
+ * file system that is writable by the process.
+ * @param verbose if true information about the OSTs and the selections made
+ * will be output.
+ *
+ */
+void swift_ost_select(struct swift_ost_store *ost_infos, const char *filepath,
+ int minfree, int writetest, int verbose) {
+
+ /* Initialise the struct. */
+ swift_ost_store_init(ost_infos);
+
+ /* Get directory of filepath. */
+ char *filepathc = strdup(filepath);
+ char *dirp = dirname(filepathc);
+
+ /* Scan for all OSTs. */
+ int rc = swift_ost_scan(dirp, ost_infos);
+ free(dirp);
+
+ /* If does not succeed we do nothing, probably not a lustre mount. */
+ if (rc == 0) {
+ if (verbose) swift_ost_store_print(ost_infos, 1);
+
+ /* Make a copy so we can undo any changes. */
+ struct swift_ost_store ost_infos_full;
+ swift_ost_store_copy(ost_infos, &ost_infos_full);
+
+ /* Cull these so we do not use OSTs with too little free space. Also sorts
+ * into most free space order. If given a value use that, otherwise we use
+ * the resident set size of the process, dumps and restarts are always
+ * smaller than that. */
+ if (minfree != 0) {
+ if (minfree < 0) {
+
+ /* No guarantee this will work, hopefully will return 0 in those cases
+ * and we do nothing. */
+ long size, resident, shared, text, library, data, dirty;
+ memuse_use(&size, &resident, &shared, &text, &data, &library, &dirty);
+
+ /* KiB into MiB. */
+ minfree = (int)(resident / 1024.0);
+ }
+
+ /* And cull and sort. */
+ swift_ost_cull(ost_infos, minfree);
+ if (verbose)
+ message("Rejected %d OSTs using free space threshold %d (MiB)",
+ ost_infos->fullcount - ost_infos->count, minfree);
+ }
+
+ if (writetest != 0) {
+ /* Test writing to all OSTs and remove any that are not writable. We do
+ * this by creating our file on every OST and checking it was created on
+ * it. */
+ int usedindex = 0;
+ int removed = 0;
+ for (int i = ost_infos->count - 1; i >= 0; i--) {
+ usedindex = ost_infos->infos[i].index;
+ rc = swift_create_striped_file(filepath, ost_infos->infos[i].index, 1,
+ &usedindex);
+
+ if (rc != 0) {
+ /* Failed so not likely to succeed next time. Probably file
+ * exists, there is nothing we should do about that, the existing
+ * stripe will be reused, along with the space of the existing file.
+ */
+ message("Failed testing file creation on OSTs, aborting test");
+ break;
+ }
+
+ if (usedindex != ost_infos->infos[i].index) {
+ /* Differing OST indices, so not what we asked for, bye. */
+ swift_ost_remove(ost_infos, ost_infos->infos[i].index);
+ removed++;
+ }
+ unlink(filepath);
+ }
+ if (verbose) message("Rejected %d OSTs as readonly", removed);
+ }
+
+ /* Safety first. If we have too few OSTs left after the above we will
+ * make the choice to do nothing. */
+ if ((ost_infos->fullcount * 0.25 > ost_infos->count) ||
+ ost_infos->count < 2) {
+ message("Too many OSTs have been rejected (%d of %d).",
+ ost_infos->fullcount - ost_infos->count, ost_infos->fullcount);
+ message("Assuming OST rejection is flawed and skipping.");
+ swift_ost_store_copy(&ost_infos_full, ost_infos);
+
+ /* Still good to use a sorted list. */
+ swift_ost_cull(ost_infos, 0);
+ }
+ swift_ost_store_free(&ost_infos_full);
+ if (verbose) swift_ost_store_print(ost_infos, 1);
+
+ } else {
+
+ /* If the scan failed we do nothing, this is probably not a lustre mount. */
+ message("Lustre OST scan failed, is this a lustre mount?");
+ }
+}
diff --git a/src/swift_lustre_api.h b/src/swift_lustre_api.h
new file mode 100644
index 0000000000000000000000000000000000000000..a625a6223164af86a006a0ea08bbd5721a18ad29
--- /dev/null
+++ b/src/swift_lustre_api.h
@@ -0,0 +1,64 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2025 Peter W. Draper (p.w.draper@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+#ifndef SWIFT_LUSTRE_API_H
+#define SWIFT_LUSTRE_API_H
+
+/* For size_t and FILE. */
+#include <stdlib.h>
+
+/* Structure to store information about an OST. */
+struct swift_ost_info {
+ int index; /* OST index */
+ size_t size; /* Size in bytes */
+ size_t used; /* Used in bytes */
+};
+
+/* Structure to store a scan of all the OSTs for a mount point. */
+struct swift_ost_store {
+ struct swift_ost_info *infos;
+ int count; /* Count of active OSTs */
+ int fullcount; /* Count of OSTs available (only different when culled) */
+ int size; /* Space available for storing OST infos */
+};
+
+/* Public functions. */
+
+/* OST scanning and selection. */
+void swift_ost_select(struct swift_ost_store *ost_infos, const char *path,
+ int minfree, int writetest, int verbose);
+int swift_ost_scan(const char *path, struct swift_ost_store *ost_infos);
+void swift_ost_cull(struct swift_ost_store *ost_infos, int minfree);
+void swift_ost_remove(struct swift_ost_store *ost_infos, int index);
+int swift_ost_next(struct swift_ost_store *ost_infos, int *arrayindex,
+ int count);
+
+/* OST store. */
+void swift_ost_store_init(struct swift_ost_store *ost_infos);
+void swift_ost_store_alloc(struct swift_ost_store *ost_infos, int size);
+void swift_ost_store_copy(struct swift_ost_store *ost_infos_src,
+ struct swift_ost_store *ost_infos_dst);
+void swift_ost_store_free(struct swift_ost_store *ost_infos);
+void swift_ost_store_print(struct swift_ost_store *ost_infos, int verbose);
+void swift_ost_store_write(FILE *file, struct swift_ost_store *ost_infos);
+
+/* File striping. */
+int swift_create_striped_file(const char *filename, int offset, int count,
+ int *usedoffset);
+
+#endif /* SWIFT_LUSTRE_API_H */