Merge branch 'Pressure-Energy-SPH' into 'master'

Add Pressure-Energy (P-U) SPH

Closes #197

See merge request !540

.gitignore

@@ -26,17 +26,20 @@ examples/swift_mpi
 examples/*/*.xmf
 examples/*/*.h5
 examples/*/*.png
+examples/*/*.mp4
 examples/*/*.txt
 examples/*/*.dot
+examples/*/restart/*
 examples/*/used_parameters.yml
 examples/*/*/*.xmf
 examples/*/*/*.png
+examples/*/*/*.mp4
 examples/*/*/*.txt
 examples/*/*/*.dot
 examples/*/*/*.rst
 examples/*/*/*.hdf5
 examples/*/snapshots*
-examples/*/restart
+examples/*/restart/*
 examples/*/*/used_parameters.yml
 examples/*/*.mpg
 examples/*/gravity_checks_*.dat
@@ -298,3 +301,6 @@ sympy-plots-for-*.tex/
 
 # macOS
 *.DS_Store
+
+#ctags
+*tags

configure.ac

@@ -855,7 +855,7 @@ fi
 # Hydro scheme.
 AC_ARG_WITH([hydro],
    [AS_HELP_STRING([--with-hydro=<scheme>],
-      [Hydro dynamics to use @<:@gadget2, minimal, hopkins, default, gizmo-mfv, gizmo-mfm, shadowfax, minimal-multi-mat, debug default: gadget2@:>@]
+      [Hydro dynamics to use @<:@gadget2, minimal, pressure-entropy, pressure-energy, default, gizmo-mfv, gizmo-mfm, shadowfax, minimal-multi-mat, debug default: gadget2@:>@]
    )],
    [with_hydro="$withval"],
    [with_hydro="gadget2"]
@@ -876,9 +876,12 @@ case "$with_hydro" in
    minimal)
       AC_DEFINE([MINIMAL_SPH], [1], [Minimal SPH])
    ;;
-   hopkins)
+   pressure-entropy)
       AC_DEFINE([HOPKINS_PE_SPH], [1], [Pressure-Entropy SPH])
    ;;
+   pressure-energy)
+      AC_DEFINE([HOPKINS_PU_SPH], [1], [Pressure-Energy SPH])
+   ;;
    default)
       AC_DEFINE([DEFAULT_SPH], [1], [Default SPH])
    ;;

examples/KelvinHelmholtz_2D/kelvinHelmholtz.yml

@@ -8,8 +8,8 @@ InternalUnitSystem:
   
 # Parameters governing the time integration
 TimeIntegration:
-  time_begin: 0.    # The starting time of the simulation (in internal units).
-  time_end:   1.5   # The end time of the simulation (in internal units).
+  time_begin: 0.0    # The starting time of the simulation (in internal units).
+  time_end:   4.5   # The end time of the simulation (in internal units).
   dt_min:     1e-6  # The minimal time-step size of the simulation (in internal units).
   dt_max:     1e-2  # The maximal time-step size of the simulation (in internal units).
 
@@ -17,7 +17,7 @@ TimeIntegration:
 Snapshots:
   basename:            kelvinHelmholtz  # Common part of the name of output files
   time_first:          0.               # Time of the first output (in internal units)
-  delta_time:          0.25      # Time difference between consecutive outputs (in internal units)
+  delta_time:          0.01      # Time difference between consecutive outputs (in internal units)
 
 # Parameters governing the conserved quantities statistics
 Statistics:

examples/KelvinHelmholtz_2D/makeIC.py

@@ -24,7 +24,7 @@ import sys
 # Generates a swift IC file for the Kelvin-Helmholtz vortex in a periodic box
 
 # Parameters
-L2    = 128       # Particles along one edge in the low-density region
+L2    = 256       # Particles along one edge in the low-density region
 gamma = 5./3.     # Gas adiabatic index
 P1    = 2.5       # Central region pressure
 P2    = 2.5       # Outskirts pressure

examples/KelvinHelmholtz_2D/makeMovie.py

+"""
+Makes a movie of the KH 2D data.
+
+You will need to run your movie with far higher time-resolution than usual to
+get a nice movie; around 450 snapshots over 6s is required.
+
+Edit this file near the bottom with the number of snaps you have.
+
+Written by Josh Borrow (joshua.borrow@durham.ac.uk)
+"""
+
+import os
+import h5py as h5
+import numpy as np
+import scipy.interpolate as si
+
+
+def load_and_extract(filename):
+    """
+    Load the data and extract relevant info.
+    """
+
+    with h5.File(filename, "r") as f:
+        x, y, _ = f["PartType0/Coordinates"][...].T
+        density = f["PartType0/Density"][...]
+
+    return x, y, density
+
+
+def make_plot(filename, array, nx, ny, dx, dy):
+    """
+    Load the data and plop it on the grid using nearest
+    neighbour searching for finding the 'correct' value of
+    the density.
+    """
+
+    data_x, data_y, density = load_and_extract(filename)
+
+    # Make the grid
+    x = np.linspace(*dx, nx)
+    y = np.linspace(*dy, ny)
+
+    xv, yv = np.meshgrid(x, y)
+
+    mesh = si.griddata((data_x, data_y), density, (xv, yv), method="nearest")
+
+    array.set_array(mesh)
+
+    return array,
+
+
+def frame(n, *args):
+    """
+    Make a single frame. Requires the global variables plot and dpi.
+    """
+
+    global plot, dpi
+
+    fn = "{}_{:04d}.hdf5".format(filename, n)
+
+    return make_plot(fn, plot, dpi, dpi, (0, 1), (0, 1))
+
+
+if __name__ == "__main__":
+    import matplotlib
+    matplotlib.use("Agg")
+
+    from tqdm import tqdm
+    from matplotlib.animation import FuncAnimation
+    from scipy.stats import gaussian_kde
+
+    import matplotlib.pyplot as plt
+
+    filename = "kelvinhelmholtz"
+    dpi = 512
+
+
+    # Look for the number of files in the directory.
+    i = 0
+    while True:
+        if os.path.isfile("{}_{:04d}.hdf5".format(filename, i)):
+            i += 1
+        else:
+            break
+
+        if i > 10000:
+            raise FileNotFoundError(
+                "Could not find the snapshots in the directory")
+
+    frames = tqdm(np.arange(0, i))
+
+    # Creation of first frame
+    fig, ax = plt.subplots(1, 1, figsize=(1, 1), frameon=False)
+
+    data_x, data_y, density = load_and_extract("kelvinhelmholtz_0000.hdf5")
+
+    x = np.linspace(0, 1, dpi)
+    y = np.linspace(0, 1, dpi)
+    xv, yv = np.meshgrid(x, y)
+
+    mesh = si.griddata((data_x, data_y), density, (xv, yv), method="nearest")
+    
+    # Global variable for set_array
+    plot = ax.imshow(mesh, extent=[0, 1, 0, 1], animated=True, interpolation="none")
+
+    anim = FuncAnimation(fig, frame, frames, interval=40, blit=False)
+
+    # Remove all whitespace
+    fig.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=None, hspace=None)
+
+    # Actually make the movie
+    anim.save("khmovie.mp4", dpi=dpi, bitrate=4096)

examples/KelvinHelmholtz_2D/run.sh

@@ -8,7 +8,8 @@ then
 fi
 
 # Run SWIFT
-../swift -s -t 1 kelvinHelmholtz.yml 2>&1 | tee output.log
+../swift -s -t 4 kelvinHelmholtz.yml 2>&1 | tee output.log
 
 # Plot the solution
 python plotSolution.py 6
+python makeMovie.py

src/debug.c

@@ -46,6 +46,8 @@
 #include "./hydro/Gadget2/hydro_debug.h"
 #elif defined(HOPKINS_PE_SPH)
 #include "./hydro/PressureEntropy/hydro_debug.h"
+#elif defined(HOPKINS_PU_SPH)
+#include "./hydro/PressureEnergy/hydro_debug.h"
 #elif defined(DEFAULT_SPH)
 #include "./hydro/Default/hydro_debug.h"
 #elif defined(GIZMO_MFV_SPH)

src/hydro.h

@@ -41,6 +41,10 @@
 #include "./hydro/PressureEntropy/hydro.h"
 #include "./hydro/PressureEntropy/hydro_iact.h"
 #define SPH_IMPLEMENTATION "Pressure-Entropy SPH (Hopkins 2013)"
+#elif defined(HOPKINS_PU_SPH)
+#include "./hydro/PressureEnergy/hydro.h"
+#include "./hydro/PressureEnergy/hydro_iact.h"
+#define SPH_IMPLEMENTATION "Pressure-Energy SPH (Hopkins 2013)"
 #elif defined(DEFAULT_SPH)
 #include "./hydro/Default/hydro.h"
 #include "./hydro/Default/hydro_iact.h"

src/hydro/PressureEnergy/hydro_debug.h

+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Coypright (c) 2016 Matthieu Schaller (matthieu.schaller@durham.ac.uk) &
+ *                    Josh Borrow (joshua.borrow@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+#ifndef SWIFT_PRESSURE_ENERGY_HYDRO_DEBUG_H
+#define SWIFT_PRESSURE_ENERGY_HYDRO_DEBUG_H
+/**
+ * @file PressureEnergy/hydro_debug.h
+ * @brief P-U conservative implementation of SPH (Debugging routines)
+ */
+
+__attribute__((always_inline)) INLINE static void hydro_debug_particle(
+    const struct part* p, const struct xpart* xp) {
+  printf(
+      "x=[%.3e,%.3e,%.3e], "
+      "v=[%.3e,%.3e,%.3e],v_full=[%.3e,%.3e,%.3e] \n a=[%.3e,%.3e,%.3e], "
+      "u=%.3e, du/dt=%.3e v_sig=%.3e, P=%.3e\n"
+      "h=%.3e, dh/dt=%.3e wcount=%d, m=%.3e, dh_drho=%.3e, rho=%.3e, \n"
+      "p_dh=%.3e, p_bar=%.3e \n"
+      "time_bin=%d\n",
+      p->x[0], p->x[1], p->x[2], p->v[0], p->v[1], p->v[2], xp->v_full[0],
+      xp->v_full[1], xp->v_full[2], p->a_hydro[0], p->a_hydro[1], p->a_hydro[2],
+      p->u, p->u_dt, p->force.v_sig, hydro_get_comoving_pressure(p), p->h,
+      p->force.h_dt, (int)p->density.wcount, p->mass, p->density.rho_dh, p->rho,
+      p->density.pressure_bar_dh, p->pressure_bar, p->time_bin);
+}
+
+#endif /* SWIFT_MINIMAL_HYDRO_DEBUG_H */

src/hydro/PressureEnergy/hydro_iact.h

+/*******************************************************************************
+ * This file is part* of SWIFT.
+ * Copyright (c) 2016 Matthieu Schaller (matthieu.schaller@durham.ac.uk) &
+ *                    Josh Borrow (joshua.borrow@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+#ifndef SWIFT_MINIMAL_HYDRO_IACT_H
+#define SWIFT_MINIMAL_HYDRO_IACT_H
+
+/**
+ * @file PressureEnergy/hydro_iact.h
+ * @brief P-U implementation of SPH (Neighbour loop equations)
+ *
+ * The thermal variable is the internal energy (u). A simple constant
+ * viscosity term with a Balsara switch is implemented.
+ *
+ * No thermal conduction term is implemented.
+ *
+ * See PressureEnergy/hydro.h for references.
+ */
+
+#include "adiabatic_index.h"
+#include "minmax.h"
+
+/**
+ * @brief Density interaction between two part*icles.
+ *
+ * @param r2 Comoving square distance between the two part*icles.
+ * @param dx Comoving vector separating both part*icles (pi - pj).
+ * @param hi Comoving smoothing-length of part*icle i.
+ * @param hj Comoving smoothing-length of part*icle j.
+ * @param pi First part*icle.
+ * @param pj Second part*icle.
+ * @param a Current scale factor.
+ * @param H Current Hubble parameter.
+ */
+__attribute__((always_inline)) INLINE static void runner_iact_density(
+    float r2, const float* dx, float hi, float hj, struct part* pi,
+    struct part* pj, float a, float H) {
+
+  float wi, wj, wi_dx, wj_dx;
+  float dv[3], curlvr[3];
+
+  const float r = sqrtf(r2);
+
+  /* Get the masses. */
+  const float mi = pi->mass;
+  const float mj = pj->mass;
+
+  /* Compute density of pi. */
+  const float hi_inv = 1.f / hi;
+  const float ui = r * hi_inv;
+
+  kernel_deval(ui, &wi, &wi_dx);
+
+  pi->rho += mj * wi;
+  pi->density.rho_dh -= mj * (hydro_dimension * wi + ui * wi_dx);
+
+  pi->pressure_bar += mj * wi * pj->u;
+  pi->density.pressure_bar_dh -=
+      mj * pj->u * (hydro_dimension * wi + ui * wi_dx);
+  pi->density.wcount += wi;
+  pi->density.wcount_dh -= (hydro_dimension * wi + ui * wi_dx);
+
+  /* Compute density of pj. */
+  const float hj_inv = 1.f / hj;
+  const float uj = r * hj_inv;
+  kernel_deval(uj, &wj, &wj_dx);
+
+  pj->rho += mi * wj;
+  pj->density.rho_dh -= mi * (hydro_dimension * wj + uj * wj_dx);
+  pj->pressure_bar += mi * wj * pi->u;
+  pj->density.pressure_bar_dh -=
+      mi * pi->u * (hydro_dimension * wj + uj * wj_dx);
+  pj->density.wcount += wj;
+  pj->density.wcount_dh -= (hydro_dimension * wj + uj * wj_dx);
+
+  /* Now we need to compute the div terms */
+  const float r_inv = 1.f / r;
+  const float faci = mj * wi_dx * r_inv;
+  const float facj = mi * wj_dx * r_inv;
+
+  /* Compute dv dot r */
+  dv[0] = pi->v[0] - pj->v[0];
+  dv[1] = pi->v[1] - pj->v[1];
+  dv[2] = pi->v[2] - pj->v[2];
+  const float dvdr = dv[0] * dx[0] + dv[1] * dx[1] + dv[2] * dx[2];
+
+  pi->density.div_v -= faci * dvdr;
+  pj->density.div_v -= facj * dvdr;
+
+  /* Compute dv cross r */
+  curlvr[0] = dv[1] * dx[2] - dv[2] * dx[1];
+  curlvr[1] = dv[2] * dx[0] - dv[0] * dx[2];
+  curlvr[2] = dv[0] * dx[1] - dv[1] * dx[0];
+
+  pi->density.rot_v[0] += faci * curlvr[0];
+  pi->density.rot_v[1] += faci * curlvr[1];
+  pi->density.rot_v[2] += faci * curlvr[2];
+
+  /* Negative because of the change in sign of dx & dv. */
+  pj->density.rot_v[0] += facj * curlvr[0];
+  pj->density.rot_v[1] += facj * curlvr[1];
+  pj->density.rot_v[2] += facj * curlvr[2];
+}
+
+/**
+ * @brief Density interaction between two part*icles (non-symmetric).
+ *
+ * @param r2 Comoving square distance between the two part*icles.
+ * @param dx Comoving vector separating both part*icles (pi - pj).
+ * @param hi Comoving smoothing-length of part*icle i.
+ * @param hj Comoving smoothing-length of part*icle j.
+ * @param pi First part*icle.
+ * @param pj Second part*icle (not updated).
+ * @param a Current scale factor.
+ * @param H Current Hubble parameter.
+ */
+__attribute__((always_inline)) INLINE static void runner_iact_nonsym_density(
+    float r2, const float* dx, float hi, float hj, struct part* pi,
+    const struct part* pj, float a, float H) {
+
+  float wi, wi_dx;
+  float dv[3], curlvr[3];
+
+  /* Get the masses. */
+  const float mj = pj->mass;
+
+  /* Get r and r inverse. */
+  const float r = sqrtf(r2);
+
+  const float h_inv = 1.f / hi;
+  const float ui = r * h_inv;
+  kernel_deval(ui, &wi, &wi_dx);
+
+  pi->rho += mj * wi;
+  pi->density.rho_dh -= mj * (hydro_dimension * wi + ui * wi_dx);
+
+  pi->pressure_bar += mj * wi * pj->u;
+
+  pi->density.pressure_bar_dh -=
+      mj * pj->u * (hydro_dimension * wi + ui * wi_dx);
+  pi->density.wcount += wi;
+  pi->density.wcount_dh -= (hydro_dimension * wi + ui * wi_dx);
+
+  const float r_inv = 1.f / r;
+  const float faci = mj * wi_dx * r_inv;
+
+  /* Compute dv dot r */
+  dv[0] = pi->v[0] - pj->v[0];
+  dv[1] = pi->v[1] - pj->v[1];
+  dv[2] = pi->v[2] - pj->v[2];
+  const float dvdr = dv[0] * dx[0] + dv[1] * dx[1] + dv[2] * dx[2];
+
+  pi->density.div_v -= faci * dvdr;
+
+  /* Compute dv cross r */
+  curlvr[0] = dv[1] * dx[2] - dv[2] * dx[1];
+  curlvr[1] = dv[2] * dx[0] - dv[0] * dx[2];
+  curlvr[2] = dv[0] * dx[1] - dv[1] * dx[0];
+
+  pi->density.rot_v[0] += faci * curlvr[0];
+  pi->density.rot_v[1] += faci * curlvr[1];
+  pi->density.rot_v[2] += faci * curlvr[2];
+}
+
+/**
+ * @brief Force interaction between two part*icles.
+ *
+ * @param r2 Comoving square distance between the two part*icles.
+ * @param dx Comoving vector separating both part*icles (pi - pj).
+ * @param hi Comoving smoothing-length of part*icle i.
+ * @param hj Comoving smoothing-length of part*icle j.
+ * @param pi First part*icle.
+ * @param pj Second part*icle.
+ * @param a Current scale factor.
+ * @param H Current Hubble parameter.
+ */
+__attribute__((always_inline)) INLINE static void runner_iact_force(
+    float r2, const float* dx, float hi, float hj, struct part* pi,
+    struct part* pj, float a, float H) {
+
+  /* Cosmological factors entering the EoMs */
+  const float fac_mu = pow_three_gamma_minus_five_over_two(a);
+  const float a2_Hubble = a * a * H;
+
+  const float r = sqrtf(r2);
+  const float r_inv = 1.0f / r;
+
+  /* Recover some data */
+  const float mj = pj->mass;
+  const float mi = pi->mass;
+
+  const float miui = mi * pi->u;
+  const float mjuj = mj * pj->u;
+
+  const float rhoi = pi->rho;
+  const float rhoj = pj->rho;
+  /* Compute gradient terms */
+  const float f_ij = 1.f - (pi->force.f / mjuj);
+  const float f_ji = 1.f - (pj->force.f / miui);
+
+  /* Get the kernel for hi. */
+  const float hi_inv = 1.0f / hi;
+  const float hid_inv = pow_dimension_plus_one(hi_inv); /* 1/h^(d+1) */
+  const float xi = r * hi_inv;
+  float wi, wi_dx;
+  kernel_deval(xi, &wi, &wi_dx);
+  const float wi_dr = hid_inv * wi_dx;
+
+  /* Get the kernel for hj. */
+  const float hj_inv = 1.0f / hj;
+  const float hjd_inv = pow_dimension_plus_one(hj_inv); /* 1/h^(d+1) */
+  const float xj = r * hj_inv;
+  float wj, wj_dx;
+  kernel_deval(xj, &wj, &wj_dx);
+  const float wj_dr = hjd_inv * wj_dx;
+
+  /* Compute dv dot r. */
+  const float dvdr = (pi->v[0] - pj->v[0]) * dx[0] +
+                     (pi->v[1] - pj->v[1]) * dx[1] +
+                     (pi->v[2] - pj->v[2]) * dx[2] + a2_Hubble * r2;
+
+  /* Are the part*icles moving towards each others ? */
+  const float omega_ij = min(dvdr, 0.f);
+  const float mu_ij = fac_mu * r_inv * omega_ij; /* This is 0 or negative */
+
+  /* Compute sound speeds and signal velocity */
+  const float ci = pi->force.soundspeed;
+  const float cj = pj->force.soundspeed;
+  const float v_sig = ci + cj - 3.f * mu_ij;
+
+  /* Balsara term */
+  const float balsara_i = pi->force.balsara;
+  const float balsara_j = pj->force.balsara;
+
+  /* Construct the full viscosity term */
+  const float rho_ij = 0.5f * (rhoi + rhoj);
+  const float visc = -0.25f * const_viscosity_alpha * v_sig * mu_ij *
+                     (balsara_i + balsara_j) / rho_ij;
+
+  /* Convolve with the kernel */
+  const float visc_acc_term = 0.5f * visc * (wi_dr + wj_dr) * r_inv;
+
+  /* SPH acceleration term */
+  const float sph_acc_term =
+      pj->u * pi->u * hydro_gamma_minus_one * hydro_gamma_minus_one *
+      ((f_ij / pi->pressure_bar) * wi_dr + (f_ji / pj->pressure_bar) * wj_dr) *
+      r_inv;
+
+  /* Assemble the acceleration */
+  const float acc = sph_acc_term + visc_acc_term;
+
+  /* Use the force Luke ! */
+  pi->a_hydro[0] -= mj * acc * dx[0];
+  pi->a_hydro[1] -= mj * acc * dx[1];
+  pi->a_hydro[2] -= mj * acc * dx[2];
+
+  pj->a_hydro[0] += mi * acc * dx[0];
+  pj->a_hydro[1] += mi * acc * dx[1];
+  pj->a_hydro[2] += mi * acc * dx[2];
+
+  /* Get the time derivative for u. */
+  const float sph_du_term_i = hydro_gamma_minus_one * hydro_gamma_minus_one *
+                              pj->u * pi->u * (f_ij / pi->pressure_bar) *
+                              wi_dr * dvdr * r_inv;
+  const float sph_du_term_j = hydro_gamma_minus_one * hydro_gamma_minus_one *
+                              pi->u * pj->u * (f_ji / pj->pressure_bar) *
+                              wj_dr * dvdr * r_inv;
+
+  /* Viscosity term */
+  const float visc_du_term = 0.5f * visc_acc_term * dvdr;
+
+  /* Assemble the energy equation term */
+  const float du_dt_i = sph_du_term_i + visc_du_term;
+  const float du_dt_j = sph_du_term_j + visc_du_term;
+
+  /* Internal energy time derivative */
+  pi->u_dt += du_dt_i * mj;
+  pj->u_dt += du_dt_j * mi;
+
+  /* Get the time derivative for h. */
+  pi->force.h_dt -= mj * dvdr * r_inv / rhoj * wi_dr;
+  pj->force.h_dt -= mi * dvdr * r_inv / rhoi * wj_dr;
+
+  /* Update the signal velocity. */
+  pi->force.v_sig = max(pi->force.v_sig, v_sig);
+  pj->force.v_sig = max(pj->force.v_sig, v_sig);
+}
+
+/**
+ * @brief Force interaction between two part*icles (non-symmetric).
+ *
+ * @param r2 Comoving square distance between the two part*icles.
+ * @param dx Comoving vector separating both part*icles (pi - pj).
+ * @param hi Comoving smoothing-length of part*icle i.
+ * @param hj Comoving smoothing-length of part*icle j.
+ * @param pi First part*icle.
+ * @param pj Second part*icle (not updated).
+ * @param a Current scale factor.
+ * @param H Current Hubble parameter.
+ */
+__attribute__((always_inline)) INLINE static void runner_iact_nonsym_force(
+    float r2, const float* dx, float hi, float hj, struct part* pi,
+    const struct part* pj, float a, float H) {
+
+  /* Cosmological factors entering the EoMs */
+  const float fac_mu = pow_three_gamma_minus_five_over_two(a);
+  const float a2_Hubble = a * a * H;
+
+  const float r = sqrtf(r2);
+  const float r_inv = 1.0f / r;
+
+  /* Recover some data */