diff --git a/.gitignore b/.gitignore index ba0028d163dae64143b0c43b465152e357d042b0..b44b5189190bd6593368c9ce8fbbd19d1d34d6ce 100644 --- a/.gitignore +++ b/.gitignore @@ -22,9 +22,6 @@ doc/Doxyfile examples/swift examples/swift_mpi -examples/*.xmf -examples/used_parameters.yml -examples/energy.txt examples/*/*.xmf examples/*/*.hdf5 examples/*/*.png diff --git a/INSTALL.swift b/INSTALL.swift index 8e1635b0715426512503fd9dcde32f59a7ad1b62..90096bc1b88d34ab86c04f623f1d3f04ca5a2997 100644 --- a/INSTALL.swift +++ b/INSTALL.swift @@ -143,6 +143,8 @@ before you can build it. - DOXYGEN: the doxygen library is required to create the SWIFT API documentation. + - python: Examples and solution script use python and rely on the + numpy library version 1.8.2 or higher. SWIFT Coding style diff --git a/README b/README index 9ef773cd85b408ff822b3652c3fd5507e6d95d01..bec538bfc35239945d60487d66ed2e02b5d363a2 100644 --- a/README +++ b/README @@ -15,24 +15,26 @@ Usage: swift [OPTION]... PARAMFILE swift_mpi [OPTION]... PARAMFILE Valid options are: - -a Pin runners using processor affinity - -c Run with cosmological time integration - -C Run with cooling + -a Pin runners using processor affinity. + -c Run with cosmological time integration. + -C Run with cooling. -d Dry run. Read the parameter file, allocate memory but does not read the particles from ICs and exit before the start of time integration. Allows user to check validy of parameter and IC files as well as memory limits. - -D Always drift all particles even the ones far from active particles. - -e Enable floating-point exceptions (debugging mode) - -f {int} Overwrite the CPU frequency (Hz) to be used for time measurements - -g Run with an external gravitational potential - -G Run with self-gravity + -D Always drift all particles even the ones far from active particles. This emulates + Gadget-[23] and GIZMO's default behaviours. + -e Enable floating-point exceptions (debugging mode). + -f {int} Overwrite the CPU frequency (Hz) to be used for time measurements. + -g Run with an external gravitational potential. + -G Run with self-gravity. -n {int} Execute a fixed number of time steps. When unset use the time_end parameter to stop. - -s Run with SPH + -s Run with hydrodynamics. + -S Run with stars. -t {int} The number of threads to use on each MPI rank. Defaults to 1 if not specified. -v [12] Increase the level of verbosity 1: MPI-rank 0 writes 2: All MPI-ranks write - -y {int} Time-step frequency at which task graphs are dumped - -h Print this help message and exit + -y {int} Time-step frequency at which task graphs are dumped. + -h Print this help message and exit. See the file examples/parameter_example.yml for an example of parameter file. diff --git a/configure.ac b/configure.ac index 4b6308e96b81bbfd0a9256bb4914f1356fbfa6f8..224b7f722fda80a653bd47914a0ffda3881e49f1 100644 --- a/configure.ac +++ b/configure.ac @@ -16,7 +16,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. # Init the project. -AC_INIT([SWIFT],[0.4.0],[https://gitlab.cosma.dur.ac.uk/swift/swiftsim]) +AC_INIT([SWIFT],[0.5.0],[https://gitlab.cosma.dur.ac.uk/swift/swiftsim]) swift_config_flags="$*" # Need to define this, instead of using fifth argument of AC_INIT, until 2.64. @@ -105,8 +105,9 @@ if test "$enable_mpi" = "yes"; then AC_PATH_PROG([MPIRUN],[mpirun],[notfound]) fi if test "$MPIRUN" = "notfound"; then + # This may not be fatal (some systems do not allow mpirun on + # development nodes)., so push on. AC_MSG_WARN([Cannot find mpirun command on PATH, thread support may not be correct]) - enable_mpi="no" else # Special options we know about. # Intel: -mt_mpi @@ -200,6 +201,20 @@ if test "$enable_debugging_checks" = "yes"; then AC_DEFINE([SWIFT_DEBUG_CHECKS],1,[Enable expensive debugging]) fi +# Check if gravity force checks are on for some particles. +AC_ARG_ENABLE([gravity-force-checks], + [AS_HELP_STRING([--enable-gravity-force-checks], + [Activate expensive brute-force gravity checks for a fraction 1/N of all particles @<:@N@:>@] + )], + [gravity_force_checks="$enableval"], + [gravity_force_checks="no"] +) +if test "$gravity_force_checks" == "yes"; then + AC_MSG_ERROR(Need to specify the fraction of particles to check when using --enable-gravity-force-checks!) +elif test "$gravity_force_checks" != "no"; then + AC_DEFINE_UNQUOTED([SWIFT_GRAVITY_FORCE_CHECKS], [$enableval] ,[Enable gravity brute-force checks]) +fi + # Define HAVE_POSIX_MEMALIGN if it works. AX_FUNC_POSIX_MEMALIGN @@ -637,13 +652,13 @@ AC_ARG_WITH([hydro-dimension], ) case "$with_dimension" in 1) - AC_DEFINE([HYDRO_DIMENSION_1D], [1], [1D analysis]) + AC_DEFINE([HYDRO_DIMENSION_1D], [1], [1D solver]) ;; 2) - AC_DEFINE([HYDRO_DIMENSION_2D], [2], [2D analysis]) + AC_DEFINE([HYDRO_DIMENSION_2D], [2], [2D solver]) ;; 3) - AC_DEFINE([HYDRO_DIMENSION_3D], [3], [3D analysis]) + AC_DEFINE([HYDRO_DIMENSION_3D], [3], [3D solver]) ;; *) AC_MSG_ERROR([Dimensionality must be 1, 2 or 3]) @@ -751,7 +766,7 @@ esac # External potential AC_ARG_WITH([ext-potential], [AS_HELP_STRING([--with-ext-potential=<pot>], - [external potential @<:@none, point-mass, isothermal, softened-isothermal, disc-patch default: none@:>@] + [external potential @<:@none, point-mass, isothermal, softened-isothermal, disc-patch, sine-wave default: none@:>@] )], [with_potential="$withval"], [with_potential="none"] @@ -769,6 +784,9 @@ case "$with_potential" in disc-patch) AC_DEFINE([EXTERNAL_POTENTIAL_DISC_PATCH], [1], [Disc-patch external potential]) ;; + sine-wave) + AC_DEFINE([EXTERNAL_POTENTIAL_SINE_WAVE], [1], [Sine wave external potential in 1D]) + ;; *) AC_MSG_ERROR([Unknown external potential: $with_potential]) ;; @@ -821,8 +839,10 @@ AC_MSG_RESULT([ Riemann solver : $with_riemann Cooling function : $with_cooling External potential : $with_potential + Task debugging : $enable_task_debugging Debugging checks : $enable_debugging_checks + Gravity checks : $gravity_force_checks ]) # Make sure the latest git revision string gets included diff --git a/doc/Doxyfile.in b/doc/Doxyfile.in index 2a5aeba7d1db0b1e1e56a9a6eed3059aba6a09ff..0df1f91194b6d1e7e98cb1b75be7d3eaaca7fc32 100644 --- a/doc/Doxyfile.in +++ b/doc/Doxyfile.in @@ -762,6 +762,7 @@ WARN_LOGFILE = INPUT = @top_srcdir@ @top_srcdir@/src @top_srcdir@/tests @top_srcdir@/examples INPUT += @top_srcdir@/src/hydro/Minimal INPUT += @top_srcdir@/src/gravity/Default +INPUT += @top_srcdir@/src/stars/Default INPUT += @top_srcdir@/src/riemann INPUT += @top_srcdir@/src/potential/point_mass INPUT += @top_srcdir@/src/cooling/const_du diff --git a/examples/BigCosmoVolume/makeIC.py b/examples/BigCosmoVolume/makeIC.py index c141337c06fb28aa4049e2823fcc7cd3e9d5513c..8f83b564a3f747d164fd03b7dddf3cd9c609d9eb 100644 --- a/examples/BigCosmoVolume/makeIC.py +++ b/examples/BigCosmoVolume/makeIC.py @@ -107,7 +107,7 @@ if n_copy > 1: for i in range(n_copy): for j in range(n_copy): for k in range(n_copy): - coords = np.append(coords, coords_tile + np.array([ i * boxSize, j * boxSize, k * boxSize ]), axis=0) + coords = np.append(coords, coords_tile + np.array([ i * boxSize[0], j * boxSize[1], k * boxSize[2] ]), axis=0) v = np.append(v, v_tile, axis=0) m = np.append(m, m_tile) h = np.append(h, h_tile) diff --git a/examples/CoolingHalo/makeIC.py b/examples/CoolingHalo/makeIC.py index 0b542e200da709e2cc7f668ab8b62b94e0bf95ee..3ec1be6f7b5e568ebe8e0fefe508ef8287edb29c 100644 --- a/examples/CoolingHalo/makeIC.py +++ b/examples/CoolingHalo/makeIC.py @@ -227,7 +227,7 @@ ds[()] = u u = np.zeros(1) # Particle IDs -ids = 1 + np.linspace(0, N, N, endpoint=False, dtype='L') +ids = 1 + np.linspace(0, N, N, endpoint=False) ds = grp.create_dataset('ParticleIDs', (N, ), 'L') ds[()] = ids diff --git a/examples/CoolingHaloWithSpin/makeIC.py b/examples/CoolingHaloWithSpin/makeIC.py index a6d57868ad7542498b27007a5c3ef9234b9feb84..2cf3127c743f61756b3ff6c4a7738c83d185f9cd 100644 --- a/examples/CoolingHaloWithSpin/makeIC.py +++ b/examples/CoolingHaloWithSpin/makeIC.py @@ -233,7 +233,7 @@ ds[()] = u u = np.zeros(1) # Particle IDs -ids = 1 + np.linspace(0, N, N, endpoint=False, dtype='L') +ids = 1 + np.linspace(0, N, N, endpoint=False) ds = grp.create_dataset('ParticleIDs', (N, ), 'L') ds[()] = ids diff --git a/examples/DiscPatch/GravityOnly/makeIC.py b/examples/DiscPatch/GravityOnly/makeIC.py index 42cd26e235deb17a899a65050ef5caa9c832c59c..6e4e148392eb7ca2fbf8c29c3f737d029916c59b 100644 --- a/examples/DiscPatch/GravityOnly/makeIC.py +++ b/examples/DiscPatch/GravityOnly/makeIC.py @@ -150,7 +150,7 @@ ds[()] = m m = numpy.zeros(1) -ids = 1 + numpy.linspace(0, numPart, numPart, endpoint=False, dtype='L') +ids = 1 + numpy.linspace(0, numPart, numPart, endpoint=False) ds = grp1.create_dataset('ParticleIDs', (numPart, ), 'L') ds[()] = ids diff --git a/examples/DiscPatch/HydroStatic/makeIC.py b/examples/DiscPatch/HydroStatic/makeIC.py index e2846d82a8cfa8bf08de83632b19ae2e7818f3c1..6ba1ccd06fed84ca728aadaa5922dbba536b6881 100644 --- a/examples/DiscPatch/HydroStatic/makeIC.py +++ b/examples/DiscPatch/HydroStatic/makeIC.py @@ -205,7 +205,7 @@ if (entropy_flag == 1): else: ds[()] = u -ids = 1 + numpy.linspace(0, numGas, numGas, endpoint=False, dtype='L') +ids = 1 + numpy.linspace(0, numGas, numGas, endpoint=False) ds = grp0.create_dataset('ParticleIDs', (numGas, ), 'L') ds[()] = ids diff --git a/examples/ExternalPointMass/energy_plot.py b/examples/ExternalPointMass/energy_plot.py index a75fcb835d33b3695170aab822092556f12db7d1..25640bcb5af2966dcd57efbe1a814bb18ac4f263 100644 --- a/examples/ExternalPointMass/energy_plot.py +++ b/examples/ExternalPointMass/energy_plot.py @@ -91,6 +91,9 @@ for i in range(402): E_tot_snap[i] = E_kin_snap[i] + E_pot_snap[i] Lz_snap[i] = np.sum(Lz) +print "Starting energy:", E_kin_stats[0], E_pot_stats[0], E_tot_stats[0] +print "Ending energy:", E_kin_stats[-1], E_pot_stats[-1], E_tot_stats[-1] + # Plot energy evolution figure() plot(time_stats, E_kin_stats, "r-", lw=0.5, label="Kinetic energy") diff --git a/examples/GreshoVortex_2D/plotSolution.py b/examples/GreshoVortex_2D/plotSolution.py index 050bca39a6b7d6f985d630e057a475945471086a..7a86daa6a4e5e1dd80888ceac9a6eb6b08dff443 100644 --- a/examples/GreshoVortex_2D/plotSolution.py +++ b/examples/GreshoVortex_2D/plotSolution.py @@ -31,6 +31,7 @@ P0 = 0. # Constant additional pressure (should have no impact on the d import matplotlib matplotlib.use("Agg") from pylab import * +from scipy import stats import h5py # Plot parameters @@ -104,6 +105,26 @@ u = sim["/PartType0/InternalEnergy"][:] S = sim["/PartType0/Entropy"][:] P = sim["/PartType0/Pressure"][:] +# Bin te data +r_bin_edge = np.arange(0., 1., 0.02) +r_bin = 0.5*(r_bin_edge[1:] + r_bin_edge[:-1]) +rho_bin,_,_ = stats.binned_statistic(r, rho, statistic='mean', bins=r_bin_edge) +v_bin,_,_ = stats.binned_statistic(r, v_phi, statistic='mean', bins=r_bin_edge) +P_bin,_,_ = stats.binned_statistic(r, P, statistic='mean', bins=r_bin_edge) +S_bin,_,_ = stats.binned_statistic(r, S, statistic='mean', bins=r_bin_edge) +u_bin,_,_ = stats.binned_statistic(r, u, statistic='mean', bins=r_bin_edge) +rho2_bin,_,_ = stats.binned_statistic(r, rho**2, statistic='mean', bins=r_bin_edge) +v2_bin,_,_ = stats.binned_statistic(r, v_phi**2, statistic='mean', bins=r_bin_edge) +P2_bin,_,_ = stats.binned_statistic(r, P**2, statistic='mean', bins=r_bin_edge) +S2_bin,_,_ = stats.binned_statistic(r, S**2, statistic='mean', bins=r_bin_edge) +u2_bin,_,_ = stats.binned_statistic(r, u**2, statistic='mean', bins=r_bin_edge) +rho_sigma_bin = np.sqrt(rho2_bin - rho_bin**2) +v_sigma_bin = np.sqrt(v2_bin - v_bin**2) +P_sigma_bin = np.sqrt(P2_bin - P_bin**2) +S_sigma_bin = np.sqrt(S2_bin - S_bin**2) +u_sigma_bin = np.sqrt(u2_bin - u_bin**2) + + # Plot the interesting quantities figure() @@ -113,6 +134,7 @@ subplot(231) plot(r, v_phi, '.', color='r', ms=0.5) plot(solution_r, solution_v_phi, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, v_bin, yerr=v_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) plot([0.2, 0.2], [-100, 100], ':', color='k', alpha=0.4, lw=1.2) plot([0.4, 0.4], [-100, 100], ':', color='k', alpha=0.4, lw=1.2) xlabel("${\\rm{Radius}}~r$", labelpad=0) @@ -125,6 +147,7 @@ subplot(232) plot(r, rho, '.', color='r', ms=0.5) plot(solution_r, solution_rho, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, rho_bin, yerr=rho_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) plot([0.2, 0.2], [-100, 100], ':', color='k', alpha=0.4, lw=1.2) plot([0.4, 0.4], [-100, 100], ':', color='k', alpha=0.4, lw=1.2) xlabel("${\\rm{Radius}}~r$", labelpad=0) @@ -138,6 +161,7 @@ subplot(233) plot(r, P, '.', color='r', ms=0.5) plot(solution_r, solution_P, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, P_bin, yerr=P_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) plot([0.2, 0.2], [-100, 100], ':', color='k', alpha=0.4, lw=1.2) plot([0.4, 0.4], [-100, 100], ':', color='k', alpha=0.4, lw=1.2) xlabel("${\\rm{Radius}}~r$", labelpad=0) @@ -150,6 +174,7 @@ subplot(234) plot(r, u, '.', color='r', ms=0.5) plot(solution_r, solution_u, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, u_bin, yerr=u_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) plot([0.2, 0.2], [-100, 100], ':', color='k', alpha=0.4, lw=1.2) plot([0.4, 0.4], [-100, 100], ':', color='k', alpha=0.4, lw=1.2) xlabel("${\\rm{Radius}}~r$", labelpad=0) @@ -163,6 +188,7 @@ subplot(235) plot(r, S, '.', color='r', ms=0.5) plot(solution_r, solution_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, S_bin, yerr=S_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) plot([0.2, 0.2], [-100, 100], ':', color='k', alpha=0.4, lw=1.2) plot([0.4, 0.4], [-100, 100], ':', color='k', alpha=0.4, lw=1.2) xlabel("${\\rm{Radius}}~r$", labelpad=0) diff --git a/examples/HydrostaticHalo/density_profile.py b/examples/HydrostaticHalo/density_profile.py index 52bebb9ffefa77dae66af155fb31fed539dcde13..d0afd399f951cf3b727e869ca8571a3a802c2e8d 100644 --- a/examples/HydrostaticHalo/density_profile.py +++ b/examples/HydrostaticHalo/density_profile.py @@ -28,7 +28,7 @@ unit_mass_cgs = float(params.attrs["InternalUnitSystem:UnitMass_in_cgs"]) unit_length_cgs = float(params.attrs["InternalUnitSystem:UnitLength_in_cgs"]) unit_velocity_cgs = float(params.attrs["InternalUnitSystem:UnitVelocity_in_cgs"]) unit_time_cgs = unit_length_cgs / unit_velocity_cgs -v_c = float(params.attrs["SoftenedIsothermalPotential:vrot"]) +v_c = float(params.attrs["IsothermalPotential:vrot"]) v_c_cgs = v_c * unit_velocity_cgs #lambda_cgs = float(params.attrs["LambdaCooling:lambda_cgs"]) #X_H = float(params.attrs["LambdaCooling:hydrogen_mass_abundance"]) diff --git a/examples/HydrostaticHalo/internal_energy_profile.py b/examples/HydrostaticHalo/internal_energy_profile.py index a1b2bda314a66eb965974d34519f66c544ee8aed..ea52cf8fc5fd098a46f05eaa58494529a868000c 100644 --- a/examples/HydrostaticHalo/internal_energy_profile.py +++ b/examples/HydrostaticHalo/internal_energy_profile.py @@ -46,7 +46,7 @@ unit_mass_cgs = float(params.attrs["InternalUnitSystem:UnitMass_in_cgs"]) unit_length_cgs = float(params.attrs["InternalUnitSystem:UnitLength_in_cgs"]) unit_velocity_cgs = float(params.attrs["InternalUnitSystem:UnitVelocity_in_cgs"]) unit_time_cgs = unit_length_cgs / unit_velocity_cgs -v_c = float(params.attrs["SoftenedIsothermalPotential:vrot"]) +v_c = float(params.attrs["IsothermalPotential:vrot"]) v_c_cgs = v_c * unit_velocity_cgs #lambda_cgs = float(params.attrs["LambdaCooling:lambda_cgs"]) #X_H = float(params.attrs["LambdaCooling:hydrogen_mass_abundance"]) diff --git a/examples/HydrostaticHalo/makeIC.py b/examples/HydrostaticHalo/makeIC.py index f33387e18dd0ab523684227f5c745b5c8b807b7f..d5081ac84473edc87857c6872278b4d0ca6389b1 100644 --- a/examples/HydrostaticHalo/makeIC.py +++ b/examples/HydrostaticHalo/makeIC.py @@ -227,7 +227,7 @@ ds[()] = u u = np.zeros(1) # Particle IDs -ids = 1 + np.linspace(0, N, N, endpoint=False, dtype='L') +ids = 1 + np.linspace(0, N, N, endpoint=False) ds = grp.create_dataset('ParticleIDs', (N, ), 'L') ds[()] = ids diff --git a/examples/HydrostaticHalo/velocity_profile.py b/examples/HydrostaticHalo/velocity_profile.py index f6a7350b9731d660b2092266d4d6ad3730bab48c..9133195d942233514148aa419003ee0ab7923494 100644 --- a/examples/HydrostaticHalo/velocity_profile.py +++ b/examples/HydrostaticHalo/velocity_profile.py @@ -46,7 +46,7 @@ unit_mass_cgs = float(params.attrs["InternalUnitSystem:UnitMass_in_cgs"]) unit_length_cgs = float(params.attrs["InternalUnitSystem:UnitLength_in_cgs"]) unit_velocity_cgs = float(params.attrs["InternalUnitSystem:UnitVelocity_in_cgs"]) unit_time_cgs = unit_length_cgs / unit_velocity_cgs -v_c = float(params.attrs["SoftenedIsothermalPotential:vrot"]) +v_c = float(params.attrs["IsothermalPotential:vrot"]) v_c_cgs = v_c * unit_velocity_cgs header = f["Header"] N = header.attrs["NumPart_Total"][0] diff --git a/examples/IsothermalPotential/makeIC.py b/examples/IsothermalPotential/makeIC.py index 7d1c5361f9a255365517226e49c55a8a50c4d6ce..27ddf15fe63d69d4172b927cfca8b8662d11ea4e 100644 --- a/examples/IsothermalPotential/makeIC.py +++ b/examples/IsothermalPotential/makeIC.py @@ -138,7 +138,7 @@ ds = grp1.create_dataset('Masses', (numPart,), 'f') ds[()] = m m = numpy.zeros(1) -ids = 1 + numpy.linspace(0, numPart, numPart, endpoint=False, dtype='L') +ids = 1 + numpy.linspace(0, numPart, numPart, endpoint=False) ds = grp1.create_dataset('ParticleIDs', (numPart, ), 'L') ds[()] = ids diff --git a/examples/IsothermalPotential/run.sh b/examples/IsothermalPotential/run.sh index 976fbddc01cf7a3dcbb114d437ddb8f03b4d54bd..a5f03f32f82e27660d0a950335d731cf0ff7401d 100755 --- a/examples/IsothermalPotential/run.sh +++ b/examples/IsothermalPotential/run.sh @@ -4,7 +4,7 @@ if [ ! -e Isothermal.hdf5 ] then echo "Generating initial conditions for the isothermal potential box example..." - python makeIC.py 1000 1 + python makeIC.py 1000 0 fi rm -rf Isothermal_*.hdf5 diff --git a/examples/MultiTypes/makeIC.py b/examples/MultiTypes/makeIC.py index 229450b67c02258553b588483d7cbd4fef887817..41a5ef5f2ffc4073ef8a4e93a130b43fcbe2c1f5 100644 --- a/examples/MultiTypes/makeIC.py +++ b/examples/MultiTypes/makeIC.py @@ -36,110 +36,154 @@ eta = 1.2349 # 48 ngbs with cubic spline kernel rhoDM = 1. Ldm = int(sys.argv[2]) # Number of particles along one axis -fileName = "multiTypes.hdf5" +massStars = 0.1 +Lstars = int(sys.argv[3]) # Number of particles along one axis + +fileBaseName = "multiTypes" +num_files = int(sys.argv[4]) #--------------------------------------------------- -numGas = Lgas**3 -massGas = boxSize**3 * rhoGas / numGas +numGas_tot = Lgas**3 +massGas = boxSize**3 * rhoGas / numGas_tot internalEnergy = P / ((gamma - 1.)*rhoGas) -numDM = Ldm**3 -massDM = boxSize**3 * rhoDM / numDM +numDM_tot = Ldm**3 +massDM = boxSize**3 * rhoDM / numDM_tot + +numStars_tot = Lstars**3 +massStars = massDM * massStars + #-------------------------------------------------- -#File -file = h5py.File(fileName, 'w') - -# Header -grp = file.create_group("/Header") -grp.attrs["BoxSize"] = boxSize -grp.attrs["NumPart_Total"] = [numGas, numDM, 0, 0, 0, 0] -grp.attrs["NumPart_Total_HighWord"] = [0, 0, 0, 0, 0, 0] -grp.attrs["NumPart_ThisFile"] = [numGas, numDM, 0, 0, 0, 0] -grp.attrs["Time"] = 0.0 -grp.attrs["NumFilesPerSnapshot"] = 1 -grp.attrs["MassTable"] = [0.0, massDM, 0.0, 0.0, 0.0, 0.0] -grp.attrs["Flag_Entropy_ICs"] = 0 -grp.attrs["Dimension"] = 3 - -#Runtime parameters -grp = file.create_group("/RuntimePars") -grp.attrs["PeriodicBoundariesOn"] = periodic - -#Units -grp = file.create_group("/Units") -grp.attrs["Unit length in cgs (U_L)"] = 1. -grp.attrs["Unit mass in cgs (U_M)"] = 1. -grp.attrs["Unit time in cgs (U_t)"] = 1. -grp.attrs["Unit current in cgs (U_I)"] = 1. -grp.attrs["Unit temperature in cgs (U_T)"] = 1. - - -# Gas Particle group -grp = file.create_group("/PartType0") - -v = zeros((numGas, 3)) -ds = grp.create_dataset('Velocities', (numGas, 3), 'f') -ds[()] = v -v = zeros(1) - -m = full((numGas, 1), massGas) -ds = grp.create_dataset('Masses', (numGas,1), 'f') -ds[()] = m -m = zeros(1) - -h = full((numGas, 1), eta * boxSize / Lgas) -ds = grp.create_dataset('SmoothingLength', (numGas,1), 'f') -ds[()] = h -h = zeros(1) - -u = full((numGas, 1), internalEnergy) -ds = grp.create_dataset('InternalEnergy', (numGas,1), 'f') -ds[()] = u -u = zeros(1) - -ids = linspace(0, numGas, numGas, endpoint=False).reshape((numGas,1)) -ds = grp.create_dataset('ParticleIDs', (numGas, 1), 'L') -ds[()] = ids + 1 -x = ids % Lgas; -y = ((ids - x) / Lgas) % Lgas; -z = (ids - x - Lgas * y) / Lgas**2; -coords = zeros((numGas, 3)) -coords[:,0] = z[:,0] * boxSize / Lgas + boxSize / (2*Lgas) -coords[:,1] = y[:,0] * boxSize / Lgas + boxSize / (2*Lgas) -coords[:,2] = x[:,0] * boxSize / Lgas + boxSize / (2*Lgas) -ds = grp.create_dataset('Coordinates', (numGas, 3), 'd') -ds[()] = coords - - - - - -# DM Particle group -grp = file.create_group("/PartType1") - -v = zeros((numDM, 3)) -ds = grp.create_dataset('Velocities', (numDM, 3), 'f') -ds[()] = v -v = zeros(1) - -m = full((numDM, 1), massDM) -ds = grp.create_dataset('Masses', (numDM,1), 'f') -ds[()] = m -m = zeros(1) - -ids = linspace(0, numDM, numDM, endpoint=False).reshape((numDM,1)) -ds = grp.create_dataset('ParticleIDs', (numDM, 1), 'L') -ds[()] = ids + Lgas**3 + 1 -x = ids % Ldm; -y = ((ids - x) / Ldm) % Ldm; -z = (ids - x - Ldm * y) / Ldm**2; -coords = zeros((numDM, 3)) -coords[:,0] = z[:,0] * boxSize / Ldm + boxSize / (2*Ldm) -coords[:,1] = y[:,0] * boxSize / Ldm + boxSize / (2*Ldm) -coords[:,2] = x[:,0] * boxSize / Ldm + boxSize / (2*Ldm) -ds = grp.create_dataset('Coordinates', (numDM, 3), 'd') -ds[()] = coords - -file.close() +offsetGas = 0 +offsetDM = 0 +offsetStars = 0 + +for n in range(num_files): + + # File name + if num_files == 1: + fileName = fileBaseName + ".hdf5" + else: + fileName = fileBaseName + ".%d.hdf5"%n + + # File + file = h5py.File(fileName, 'w') + + # Number of particles + numGas = numGas_tot / num_files + numDM = numDM_tot / num_files + numStars = numStars_tot / num_files + + if n == num_files - 1: + numGas += numGas_tot % num_files + numDM += numDM_tot % num_files + numStars += numStars_tot % num_files + + + # Header + grp = file.create_group("/Header") + grp.attrs["BoxSize"] = boxSize + grp.attrs["NumPart_Total"] = [numGas_tot, numDM_tot, 0, 0, numStars_tot, 0] + grp.attrs["NumPart_Total_HighWord"] = [0, 0, 0, 0, 0, 0] + grp.attrs["NumPart_ThisFile"] = [numGas, numDM, 0, 0, numStars, 0] + grp.attrs["Time"] = 0.0 + grp.attrs["NumFilesPerSnapshot"] = num_files + grp.attrs["MassTable"] = [0.0, massDM, 0.0, 0.0, 0.0, 0.0] + grp.attrs["Flag_Entropy_ICs"] = 0 + grp.attrs["Dimension"] = 3 + + #Runtime parameters + grp = file.create_group("/RuntimePars") + grp.attrs["PeriodicBoundariesOn"] = periodic + + #Units + grp = file.create_group("/Units") + grp.attrs["Unit length in cgs (U_L)"] = 1. + grp.attrs["Unit mass in cgs (U_M)"] = 1. + grp.attrs["Unit time in cgs (U_t)"] = 1. + grp.attrs["Unit current in cgs (U_I)"] = 1. + grp.attrs["Unit temperature in cgs (U_T)"] = 1. + + + # Gas Particle group + grp = file.create_group("/PartType0") + + v = zeros((numGas, 3)) + ds = grp.create_dataset('Velocities', (numGas, 3), 'f', data=v) + + m = full((numGas, 1), massGas) + ds = grp.create_dataset('Masses', (numGas,1), 'f', data=m) + + h = full((numGas, 1), eta * boxSize / Lgas) + ds = grp.create_dataset('SmoothingLength', (numGas,1), 'f', data=h) + + u = full((numGas, 1), internalEnergy) + ds = grp.create_dataset('InternalEnergy', (numGas,1), 'f', data=u) + + ids = linspace(offsetGas, offsetGas+numGas, numGas, endpoint=False).reshape((numGas,1)) + ds = grp.create_dataset('ParticleIDs', (numGas, 1), 'L', data=ids+1) + x = ids % Lgas; + y = ((ids - x) / Lgas) % Lgas; + z = (ids - x - Lgas * y) / Lgas**2; + coords = zeros((numGas, 3)) + coords[:,0] = z[:,0] * boxSize / Lgas + boxSize / (2*Lgas) + coords[:,1] = y[:,0] * boxSize / Lgas + boxSize / (2*Lgas) + coords[:,2] = x[:,0] * boxSize / Lgas + boxSize / (2*Lgas) + ds = grp.create_dataset('Coordinates', (numGas, 3), 'd', data=coords) + + + + # DM Particle group + grp = file.create_group("/PartType1") + + v = zeros((numDM, 3)) + ds = grp.create_dataset('Velocities', (numDM, 3), 'f', data=v) + + m = full((numDM, 1), massDM) + ds = grp.create_dataset('Masses', (numDM,1), 'f', data=m) + + ids = linspace(offsetDM, offsetDM+numDM, numDM, endpoint=False).reshape((numDM,1)) + ds = grp.create_dataset('ParticleIDs', (numDM, 1), 'L', data=ids + numGas_tot + 1) + ds[()] = ids + Lgas**3 + 1 + x = ids % Ldm; + y = ((ids - x) / Ldm) % Ldm; + z = (ids - x - Ldm * y) / Ldm**2; + coords = zeros((numDM, 3)) + coords[:,0] = z[:,0] * boxSize / Ldm + boxSize / (2*Ldm) + coords[:,1] = y[:,0] * boxSize / Ldm + boxSize / (2*Ldm) + coords[:,2] = x[:,0] * boxSize / Ldm + boxSize / (2*Ldm) + ds = grp.create_dataset('Coordinates', (numDM, 3), 'd', data=coords) + + + + # Star Particle group + grp = file.create_group("/PartType4") + + v = zeros((numStars, 3)) + ds = grp.create_dataset('Velocities', (numStars, 3), 'f', data=v) + + m = full((numStars, 1), massStars) + ds = grp.create_dataset('Masses', (numStars,1), 'f', data=m) + + ids = linspace(0, numStars, numStars, endpoint=False).reshape((numStars,1)) + ds = grp.create_dataset('ParticleIDs', (numStars, 1), 'L', data=ids + numGas_tot + numDM_tot + 1) + x = ids % Ldm; + y = ((ids - x) / Ldm) % Ldm; + z = (ids - x - Ldm * y) / Ldm**2; + coords = zeros((numStars, 3)) + coords[:,0] = z[:,0] * boxSize / Ldm + boxSize / (2*Ldm) + coords[:,1] = y[:,0] * boxSize / Ldm + boxSize / (2*Ldm) + coords[:,2] = x[:,0] * boxSize / Ldm + boxSize / (2*Ldm) + ds = grp.create_dataset('Coordinates', (numStars, 3), 'd', data=coords) + + + + # Shift stuff + offsetGas += numGas + offsetDM += numDM + offsetStars += numStars + + file.close() + diff --git a/examples/MultiTypes/multiTypes.yml b/examples/MultiTypes/multiTypes.yml index 4d54f95fcdd09464b03d0f9987398cd2710b2e44..05d45595e9ec43b0849ed574f6b9922c19021a33 100644 --- a/examples/MultiTypes/multiTypes.yml +++ b/examples/MultiTypes/multiTypes.yml @@ -32,6 +32,7 @@ SPH: # Parameters related to the initial conditions InitialConditions: file_name: ./multiTypes.hdf5 # The file to read + replicate: 2 # Replicate all particles twice along each axis # External potential parameters PointMassPotential: diff --git a/examples/MultiTypes/run.sh b/examples/MultiTypes/run.sh index 57465ce0ba6dde3988359df990f2a93323dbc617..38cba70393861539f18bf9fa360d51f46dd3367d 100755 --- a/examples/MultiTypes/run.sh +++ b/examples/MultiTypes/run.sh @@ -4,7 +4,7 @@ if [ ! -e multiTypes.hdf5 ] then echo "Generating initial conditions for the multitype box example..." - python makeIC.py 50 60 + python makeIC.py 9 13 7 1 fi -../swift -s -g -t 16 multiTypes.yml 2>&1 | tee output.log +../swift -s -g -S -t 1 multiTypes.yml 2>&1 | tee output.log diff --git a/examples/Noh_1D/makeIC.py b/examples/Noh_1D/makeIC.py new file mode 100644 index 0000000000000000000000000000000000000000..176f3517455db7a8b0994ac7d1e65fb9cb7419d4 --- /dev/null +++ b/examples/Noh_1D/makeIC.py @@ -0,0 +1,90 @@ +############################################################################### + # This file is part of SWIFT. + # Copyright (c) 2016 Matthieu Schaller (matthieu.schaller@durham.ac.uk) + # + # This program is free software: you can redistribute it and/or modify + # it under the terms of the GNU Lesser General Public License as published + # by the Free Software Foundation, either version 3 of the License, or + # (at your option) any later version. + # + # This program is distributed in the hope that it will be useful, + # but WITHOUT ANY WARRANTY; without even the implied warranty of + # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + # GNU General Public License for more details. + # + # You should have received a copy of the GNU Lesser General Public License + # along with this program. If not, see <http://www.gnu.org/licenses/>. + # + ############################################################################## + +import h5py +from numpy import * + +# Generates a swift IC file for the 1D Noh problem test in a periodic box + +# Parameters +numPart = 1001 +gamma = 5./3. # Gas adiabatic index +rho0 = 1. # Background density +P0 = 1.e-6 # Background pressure +fileName = "noh.hdf5" + +#--------------------------------------------------- +coords = zeros((numPart, 3)) +h = zeros(numPart) +vol = 2. + +for i in range(numPart): + coords[i,0] = i * vol/numPart + vol/(2.*numPart) + h[i] = 1.2348 * vol / numPart + +# Generate extra arrays +v = zeros((numPart, 3)) +ids = linspace(1, numPart, numPart) +m = zeros(numPart) +u = zeros(numPart) + +m[:] = rho0 * vol / numPart +u[:] = P0 / (rho0 * (gamma - 1)) +v[coords[:,0]<vol/2. ,0] = 1 +v[coords[:,0]>vol/2. ,0] = -1 + +#-------------------------------------------------- + +#File +file = h5py.File(fileName, 'w') + +# Header +grp = file.create_group("/Header") +grp.attrs["BoxSize"] = [vol, vol, vol] +grp.attrs["NumPart_Total"] = [numPart, 0, 0, 0, 0, 0] +grp.attrs["NumPart_Total_HighWord"] = [0, 0, 0, 0, 0, 0] +grp.attrs["NumPart_ThisFile"] = [numPart, 0, 0, 0, 0, 0] +grp.attrs["Time"] = 0.0 +grp.attrs["NumFilesPerSnapshot"] = 1 +grp.attrs["MassTable"] = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +grp.attrs["Flag_Entropy_ICs"] = 0 +grp.attrs["Dimension"] = 1 + +#Runtime parameters +grp = file.create_group("/RuntimePars") +grp.attrs["PeriodicBoundariesOn"] = 1 + +#Units +grp = file.create_group("/Units") +grp.attrs["Unit length in cgs (U_L)"] = 1. +grp.attrs["Unit mass in cgs (U_M)"] = 1. +grp.attrs["Unit time in cgs (U_t)"] = 1. +grp.attrs["Unit current in cgs (U_I)"] = 1. +grp.attrs["Unit temperature in cgs (U_T)"] = 1. + +#Particle group +grp = file.create_group("/PartType0") +grp.create_dataset('Coordinates', data=coords, dtype='d') +grp.create_dataset('Velocities', data=v, dtype='f') +grp.create_dataset('Masses', data=m, dtype='f') +grp.create_dataset('SmoothingLength', data=h, dtype='f') +grp.create_dataset('InternalEnergy', data=u, dtype='f') +grp.create_dataset('ParticleIDs', data=ids, dtype='L') + +file.close() diff --git a/examples/Noh_1D/noh.yml b/examples/Noh_1D/noh.yml new file mode 100644 index 0000000000000000000000000000000000000000..911ebbdc1ca82f9b5cf2f70841d848008cbc5f6c --- /dev/null +++ b/examples/Noh_1D/noh.yml @@ -0,0 +1,35 @@ +# Define the system of units to use internally. +InternalUnitSystem: + UnitMass_in_cgs: 1 # Grams + UnitLength_in_cgs: 1 # Centimeters + UnitVelocity_in_cgs: 1 # Centimeters per second + UnitCurrent_in_cgs: 1 # Amperes + UnitTemp_in_cgs: 1 # Kelvin + +# Parameters governing the time integration +TimeIntegration: + time_begin: 0. # The starting time of the simulation (in internal units). + time_end: 0.6 # The end time of the simulation (in internal units). + dt_min: 1e-7 # The minimal time-step size of the simulation (in internal units). + dt_max: 1e-3 # The maximal time-step size of the simulation (in internal units). + +# Parameters governing the snapshots +Snapshots: + basename: noh # Common part of the name of output files + time_first: 0. # Time of the first output (in internal units) + delta_time: 5e-2 # Time difference between consecutive outputs (in internal units) + +# Parameters governing the conserved quantities statistics +Statistics: + delta_time: 1e-5 # Time between statistics output + +# Parameters for the hydrodynamics scheme +SPH: + resolution_eta: 1.2348 # Target smoothing length in units of the mean inter-particle separation (1.2348 == 48Ngbs with the cubic spline kernel). + delta_neighbours: 0.1 # The tolerance for the targetted number of neighbours. + CFL_condition: 0.1 # Courant-Friedrich-Levy condition for time integration. + +# Parameters related to the initial conditions +InitialConditions: + file_name: ./noh.hdf5 # The file to read + diff --git a/examples/Noh_1D/plotSolution.py b/examples/Noh_1D/plotSolution.py new file mode 100644 index 0000000000000000000000000000000000000000..f4916af6e6066d21f76c28b5acef41e1907a83fd --- /dev/null +++ b/examples/Noh_1D/plotSolution.py @@ -0,0 +1,175 @@ +############################################################################### + # This file is part of SWIFT. + # Copyright (c) 2016 Matthieu Schaller (matthieu.schaller@durham.ac.uk) + # + # This program is free software: you can redistribute it and/or modify + # it under the terms of the GNU Lesser General Public License as published + # by the Free Software Foundation, either version 3 of the License, or + # (at your option) any later version. + # + # This program is distributed in the hope that it will be useful, + # but WITHOUT ANY WARRANTY; without even the implied warranty of + # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + # GNU General Public License for more details. + # + # You should have received a copy of the GNU Lesser General Public License + # along with this program. If not, see <http://www.gnu.org/licenses/>. + # + ############################################################################## + +# Computes the analytical solution of the Noh problem and plots the SPH answer + + +# Parameters +gas_gamma = 5./3. # Polytropic index +rho0 = 1. # Background density +P0 = 1.e-6 # Background pressure +v0 = 1 + +import matplotlib +matplotlib.use("Agg") +from pylab import * +import h5py + +# Plot parameters +params = {'axes.labelsize': 10, +'axes.titlesize': 10, +'font.size': 12, +'legend.fontsize': 12, +'xtick.labelsize': 10, +'ytick.labelsize': 10, +'text.usetex': True, + 'figure.figsize' : (9.90,6.45), +'figure.subplot.left' : 0.045, +'figure.subplot.right' : 0.99, +'figure.subplot.bottom' : 0.05, +'figure.subplot.top' : 0.99, +'figure.subplot.wspace' : 0.15, +'figure.subplot.hspace' : 0.12, +'lines.markersize' : 6, +'lines.linewidth' : 3., +'text.latex.unicode': True +} +rcParams.update(params) +rc('font',**{'family':'sans-serif','sans-serif':['Times']}) + + +snap = int(sys.argv[1]) + + +# Read the simulation data +sim = h5py.File("noh_%03d.hdf5"%snap, "r") +boxSize = sim["/Header"].attrs["BoxSize"][0] +time = sim["/Header"].attrs["Time"][0] +scheme = sim["/HydroScheme"].attrs["Scheme"] +kernel = sim["/HydroScheme"].attrs["Kernel function"] +neighbours = sim["/HydroScheme"].attrs["Kernel target N_ngb"] +eta = sim["/HydroScheme"].attrs["Kernel eta"] +git = sim["Code"].attrs["Git Revision"] + +x = sim["/PartType0/Coordinates"][:,0] +v = sim["/PartType0/Velocities"][:,0] +u = sim["/PartType0/InternalEnergy"][:] +S = sim["/PartType0/Entropy"][:] +P = sim["/PartType0/Pressure"][:] +rho = sim["/PartType0/Density"][:] + +N = 1001 # Number of points +x_min = -1 +x_max = 1 + +x += x_min + +# --------------------------------------------------------------- +# Don't touch anything after this. +# --------------------------------------------------------------- +x_s = np.arange(0, 2., 2./N) - 1. +rho_s = np.ones(N) * rho0 +P_s = np.ones(N) * rho0 +v_s = np.ones(N) * v0 + +# Shock position +u0 = rho0 * P0 * (gas_gamma-1) +us = 0.5 * (gas_gamma - 1) * v0 +rs = us * time + +# Post-shock values +rho_s[np.abs(x_s) < rs] = rho0 * (gas_gamma + 1) / (gas_gamma - 1) +P_s[np.abs(x_s) < rs] = 0.5 * rho0 * v0**2 * (gas_gamma + 1) +v_s[np.abs(x_s) < rs] = 0. + +# Pre-shock values +rho_s[np.abs(x_s) >= rs] = rho0 +P_s[np.abs(x_s) >= rs] = 0 +v_s[x_s >= rs] = -v0 +v_s[x_s <= -rs] = v0 + +# Additional arrays +u_s = P_s / (rho_s * (gas_gamma - 1.)) #internal energy +s_s = P_s / rho_s**gas_gamma # entropic function + +# Plot the interesting quantities +figure() + +# Velocity profile -------------------------------- +subplot(231) +plot(x, v, '.', color='r', ms=4.0) +plot(x_s, v_s, '--', color='k', alpha=0.8, lw=1.2) +xlabel("${\\rm{Position}}~x$", labelpad=0) +ylabel("${\\rm{Velocity}}~v_x$", labelpad=-4) +xlim(-0.5, 0.5) +ylim(-1.2, 1.2) + +# Density profile -------------------------------- +subplot(232) +plot(x, rho, '.', color='r', ms=4.0) +plot(x_s, rho_s, '--', color='k', alpha=0.8, lw=1.2) +xlabel("${\\rm{Position}}~x$", labelpad=0) +ylabel("${\\rm{Density}}~\\rho$", labelpad=0) +xlim(-0.5, 0.5) +ylim(0.95, 4.4) + +# Pressure profile -------------------------------- +subplot(233) +plot(x, P, '.', color='r', ms=4.0) +plot(x_s, P_s, '--', color='k', alpha=0.8, lw=1.2) +xlabel("${\\rm{Position}}~x$", labelpad=0) +ylabel("${\\rm{Pressure}}~P$", labelpad=0) +xlim(-0.5, 0.5) +ylim(-0.05, 1.8) + +# Internal energy profile ------------------------- +subplot(234) +plot(x, u, '.', color='r', ms=4.0) +plot(x_s, u_s, '--', color='k', alpha=0.8, lw=1.2) +xlabel("${\\rm{Position}}~x$", labelpad=0) +ylabel("${\\rm{Internal~Energy}}~u$", labelpad=0) +xlim(-0.5, 0.5) +ylim(-0.05, 0.8) + +# Entropy profile --------------------------------- +subplot(235) +plot(x, S, '.', color='r', ms=4.0) +plot(x_s, s_s, '--', color='k', alpha=0.8, lw=1.2) +xlabel("${\\rm{Position}}~x$", labelpad=0) +ylabel("${\\rm{Entropy}}~S$", labelpad=-9) +xlim(-0.5, 0.5) +ylim(-0.05, 0.2) + +# Information ------------------------------------- +subplot(236, frameon=False) + +text(-0.49, 0.9, "Noh problem with $\\gamma=%.3f$ in 1D at $t=%.2f$"%(gas_gamma,time), fontsize=10) +text(-0.49, 0.8, "ICs:~~ $(P_0, \\rho_0, v_0) = (%1.2e, %.3f, %.3f)$"%(1e-6, 1., -1.), fontsize=10) +plot([-0.49, 0.1], [0.62, 0.62], 'k-', lw=1) +text(-0.49, 0.5, "$\\textsc{Swift}$ %s"%git, fontsize=10) +text(-0.49, 0.4, scheme, fontsize=10) +text(-0.49, 0.3, kernel, fontsize=10) +text(-0.49, 0.2, "$%.2f$ neighbours ($\\eta=%.3f$)"%(neighbours, eta), fontsize=10) +xlim(-0.5, 0.5) +ylim(0, 1) +xticks([]) +yticks([]) + + +savefig("Noh.png", dpi=200) diff --git a/examples/Noh_1D/run.sh b/examples/Noh_1D/run.sh new file mode 100755 index 0000000000000000000000000000000000000000..77788bfa8429e2fbf0502068baa70598acaaa791 --- /dev/null +++ b/examples/Noh_1D/run.sh @@ -0,0 +1,14 @@ +#!/bin/bash + + # Generate the initial conditions if they are not present. +if [ ! -e noh.hdf5 ] +then + echo "Generating initial conditions for the Noh problem..." + python makeIC.py +fi + +# Run SWIFT +../swift -s -t 1 noh.yml 2>&1 | tee output.log + +# Plot the solution +python plotSolution.py 12 diff --git a/examples/Noh_2D/getGlass.sh b/examples/Noh_2D/getGlass.sh new file mode 100755 index 0000000000000000000000000000000000000000..ae3c977064f5e7a408aa249c5fd9089b3c52ecb1 --- /dev/null +++ b/examples/Noh_2D/getGlass.sh @@ -0,0 +1,2 @@ +#!/bin/bash +wget http://virgodb.cosma.dur.ac.uk/swift-webstorage/ICs/glassPlane_128.hdf5 diff --git a/examples/Noh_2D/makeIC.py b/examples/Noh_2D/makeIC.py new file mode 100644 index 0000000000000000000000000000000000000000..f7239fa3cd188637df929f86451d20a9978bd1f5 --- /dev/null +++ b/examples/Noh_2D/makeIC.py @@ -0,0 +1,98 @@ + +############################################################################### + # This file is part of SWIFT. + # Copyright (c) 2016 Matthieu Schaller (matthieu.schaller@durham.ac.uk) + # + # This program is free software: you can redistribute it and/or modify + # it under the terms of the GNU Lesser General Public License as published + # by the Free Software Foundation, either version 3 of the License, or + # (at your option) any later version. + # + # This program is distributed in the hope that it will be useful, + # but WITHOUT ANY WARRANTY; without even the implied warranty of + # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + # GNU General Public License for more details. + # + # You should have received a copy of the GNU Lesser General Public License + # along with this program. If not, see <http://www.gnu.org/licenses/>. + # + ############################################################################## + +import h5py +from numpy import * + +# Generates a swift IC file for the 2D Noh problem in a periodic box + +# Parameters +gamma = 5./3. # Gas adiabatic index +gamma = 5./3. # Gas adiabatic index +rho0 = 1. # Background density +P0 = 1.e-6 # Background pressure +fileName = "noh.hdf5" + +#--------------------------------------------------- +glass = h5py.File("glassPlane_128.hdf5", "r") + +vol = 4. + +pos = glass["/PartType0/Coordinates"][:,:] * sqrt(vol) +h = glass["/PartType0/SmoothingLength"][:] * sqrt(vol) +numPart = size(h) + +# Generate extra arrays +v = zeros((numPart, 3)) +ids = linspace(1, numPart, numPart) + +m = zeros(numPart) +u = zeros(numPart) +m[:] = rho0 * vol / numPart +u[:] = P0 / (rho0 * (gamma - 1)) + +# Make radial velocities +#r = sqrt((pos[:,0]-1.)**2 + (pos[:,1]-1.)**2) +#theta = arctan2((pos[:,1]-1.), (pos[:,0]-1.)) +v[:,0] = -(pos[:,0] - 1.) +v[:,1] = -(pos[:,1] - 1.) + +norm_v = sqrt(v[:,0]**2 + v[:,1]**2) +v[:,0] /= norm_v +v[:,1] /= norm_v + +#File +file = h5py.File(fileName, 'w') + +# Header +grp = file.create_group("/Header") +grp.attrs["BoxSize"] = [sqrt(vol), sqrt(vol), 1.] +grp.attrs["NumPart_Total"] = [numPart, 0, 0, 0, 0, 0] +grp.attrs["NumPart_Total_HighWord"] = [0, 0, 0, 0, 0, 0] +grp.attrs["NumPart_ThisFile"] = [numPart, 0, 0, 0, 0, 0] +grp.attrs["Time"] = 0.0 +grp.attrs["NumFilesPerSnapshot"] = 1 +grp.attrs["MassTable"] = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +grp.attrs["Flag_Entropy_ICs"] = 0 +grp.attrs["Dimension"] = 2 + +#Runtime parameters +grp = file.create_group("/RuntimePars") +grp.attrs["PeriodicBoundariesOn"] = 1 + +#Units +grp = file.create_group("/Units") +grp.attrs["Unit length in cgs (U_L)"] = 1. +grp.attrs["Unit mass in cgs (U_M)"] = 1. +grp.attrs["Unit time in cgs (U_t)"] = 1. +grp.attrs["Unit current in cgs (U_I)"] = 1. +grp.attrs["Unit temperature in cgs (U_T)"] = 1. + +#Particle group +grp = file.create_group("/PartType0") +grp.create_dataset('Coordinates', data=pos, dtype='d') +grp.create_dataset('Velocities', data=v, dtype='f') +grp.create_dataset('Masses', data=m, dtype='f') +grp.create_dataset('SmoothingLength', data=h, dtype='f') +grp.create_dataset('InternalEnergy', data=u, dtype='f') +grp.create_dataset('ParticleIDs', data=ids, dtype='L') + + +file.close() diff --git a/examples/Noh_2D/noh.yml b/examples/Noh_2D/noh.yml new file mode 100644 index 0000000000000000000000000000000000000000..911ebbdc1ca82f9b5cf2f70841d848008cbc5f6c --- /dev/null +++ b/examples/Noh_2D/noh.yml @@ -0,0 +1,35 @@ +# Define the system of units to use internally. +InternalUnitSystem: + UnitMass_in_cgs: 1 # Grams + UnitLength_in_cgs: 1 # Centimeters + UnitVelocity_in_cgs: 1 # Centimeters per second + UnitCurrent_in_cgs: 1 # Amperes + UnitTemp_in_cgs: 1 # Kelvin + +# Parameters governing the time integration +TimeIntegration: + time_begin: 0. # The starting time of the simulation (in internal units). + time_end: 0.6 # The end time of the simulation (in internal units). + dt_min: 1e-7 # The minimal time-step size of the simulation (in internal units). + dt_max: 1e-3 # The maximal time-step size of the simulation (in internal units). + +# Parameters governing the snapshots +Snapshots: + basename: noh # Common part of the name of output files + time_first: 0. # Time of the first output (in internal units) + delta_time: 5e-2 # Time difference between consecutive outputs (in internal units) + +# Parameters governing the conserved quantities statistics +Statistics: + delta_time: 1e-5 # Time between statistics output + +# Parameters for the hydrodynamics scheme +SPH: + resolution_eta: 1.2348 # Target smoothing length in units of the mean inter-particle separation (1.2348 == 48Ngbs with the cubic spline kernel). + delta_neighbours: 0.1 # The tolerance for the targetted number of neighbours. + CFL_condition: 0.1 # Courant-Friedrich-Levy condition for time integration. + +# Parameters related to the initial conditions +InitialConditions: + file_name: ./noh.hdf5 # The file to read + diff --git a/examples/Noh_2D/plotSolution.py b/examples/Noh_2D/plotSolution.py new file mode 100644 index 0000000000000000000000000000000000000000..a01a712efd412488aea09c3f3c4e8d68323fc916 --- /dev/null +++ b/examples/Noh_2D/plotSolution.py @@ -0,0 +1,198 @@ +############################################################################### + # This file is part of SWIFT. + # Copyright (c) 2016 Matthieu Schaller (matthieu.schaller@durham.ac.uk) + # + # This program is free software: you can redistribute it and/or modify + # it under the terms of the GNU Lesser General Public License as published + # by the Free Software Foundation, either version 3 of the License, or + # (at your option) any later version. + # + # This program is distributed in the hope that it will be useful, + # but WITHOUT ANY WARRANTY; without even the implied warranty of + # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + # GNU General Public License for more details. + # + # You should have received a copy of the GNU Lesser General Public License + # along with this program. If not, see <http://www.gnu.org/licenses/>. + # + ############################################################################## + +# Computes the analytical solution of the Noh problem and plots the SPH answer + + +# Parameters +gas_gamma = 5./3. # Polytropic index +rho0 = 1. # Background density +P0 = 1.e-6 # Background pressure +v0 = 1 + +import matplotlib +matplotlib.use("Agg") +from pylab import * +from scipy import stats +import h5py + +# Plot parameters +params = {'axes.labelsize': 10, +'axes.titlesize': 10, +'font.size': 12, +'legend.fontsize': 12, +'xtick.labelsize': 10, +'ytick.labelsize': 10, +'text.usetex': True, + 'figure.figsize' : (9.90,6.45), +'figure.subplot.left' : 0.045, +'figure.subplot.right' : 0.99, +'figure.subplot.bottom' : 0.05, +'figure.subplot.top' : 0.99, +'figure.subplot.wspace' : 0.15, +'figure.subplot.hspace' : 0.12, +'lines.markersize' : 6, +'lines.linewidth' : 3., +'text.latex.unicode': True +} +rcParams.update(params) +rc('font',**{'family':'sans-serif','sans-serif':['Times']}) + + +snap = int(sys.argv[1]) + +# Read the simulation data +sim = h5py.File("noh_%03d.hdf5"%snap, "r") +boxSize = sim["/Header"].attrs["BoxSize"][0] +time = sim["/Header"].attrs["Time"][0] +scheme = sim["/HydroScheme"].attrs["Scheme"] +kernel = sim["/HydroScheme"].attrs["Kernel function"] +neighbours = sim["/HydroScheme"].attrs["Kernel target N_ngb"] +eta = sim["/HydroScheme"].attrs["Kernel eta"] +git = sim["Code"].attrs["Git Revision"] + +x = sim["/PartType0/Coordinates"][:,0] +y = sim["/PartType0/Coordinates"][:,1] +vx = sim["/PartType0/Velocities"][:,0] +vy = sim["/PartType0/Velocities"][:,1] +u = sim["/PartType0/InternalEnergy"][:] +S = sim["/PartType0/Entropy"][:] +P = sim["/PartType0/Pressure"][:] +rho = sim["/PartType0/Density"][:] + +r = np.sqrt((x-1)**2 + (y-1)**2) +v = -np.sqrt(vx**2 + vy**2) + +# Bin te data +r_bin_edge = np.arange(0., 1., 0.02) +r_bin = 0.5*(r_bin_edge[1:] + r_bin_edge[:-1]) +rho_bin,_,_ = stats.binned_statistic(r, rho, statistic='mean', bins=r_bin_edge) +v_bin,_,_ = stats.binned_statistic(r, v, statistic='mean', bins=r_bin_edge) +P_bin,_,_ = stats.binned_statistic(r, P, statistic='mean', bins=r_bin_edge) +S_bin,_,_ = stats.binned_statistic(r, S, statistic='mean', bins=r_bin_edge) +u_bin,_,_ = stats.binned_statistic(r, u, statistic='mean', bins=r_bin_edge) +rho2_bin,_,_ = stats.binned_statistic(r, rho**2, statistic='mean', bins=r_bin_edge) +v2_bin,_,_ = stats.binned_statistic(r, v**2, statistic='mean', bins=r_bin_edge) +P2_bin,_,_ = stats.binned_statistic(r, P**2, statistic='mean', bins=r_bin_edge) +S2_bin,_,_ = stats.binned_statistic(r, S**2, statistic='mean', bins=r_bin_edge) +u2_bin,_,_ = stats.binned_statistic(r, u**2, statistic='mean', bins=r_bin_edge) +rho_sigma_bin = np.sqrt(rho2_bin - rho_bin**2) +v_sigma_bin = np.sqrt(v2_bin - v_bin**2) +P_sigma_bin = np.sqrt(P2_bin - P_bin**2) +S_sigma_bin = np.sqrt(S2_bin - S_bin**2) +u_sigma_bin = np.sqrt(u2_bin - u_bin**2) + +# Analytic solution +N = 1000 # Number of points + +x_s = np.arange(0, 2., 2./N) - 1. +rho_s = np.ones(N) * rho0 +P_s = np.ones(N) * rho0 +v_s = np.ones(N) * v0 + +# Shock position +u0 = rho0 * P0 * (gas_gamma-1) +us = 0.5 * (gas_gamma - 1) * v0 +rs = us * time + +# Post-shock values +rho_s[np.abs(x_s) < rs] = rho0 * ((gas_gamma + 1) / (gas_gamma - 1))**2 +P_s[np.abs(x_s) < rs] = 0.5 * rho0 * v0**2 * (gas_gamma + 1)**2 / (gas_gamma-1) +v_s[np.abs(x_s) < rs] = 0. + +# Pre-shock values +rho_s[np.abs(x_s) >= rs] = rho0 * (1 + v0 * time/np.abs(x_s[np.abs(x_s) >=rs])) +P_s[np.abs(x_s) >= rs] = 0 +v_s[x_s >= rs] = -v0 +v_s[x_s <= -rs] = v0 + +# Additional arrays +u_s = P_s / (rho_s * (gas_gamma - 1.)) #internal energy +s_s = P_s / rho_s**gas_gamma # entropic function + +# Plot the interesting quantities +figure() + +# Velocity profile -------------------------------- +subplot(231) +plot(r, v, '.', color='r', ms=0.5, alpha=0.2) +plot(x_s, v_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, v_bin, yerr=v_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) +xlabel("${\\rm{Radius}}~r$", labelpad=0) +ylabel("${\\rm{Velocity}}~v_r$", labelpad=-4) +xlim(0, 0.5) +ylim(-1.2, 0.4) + +# Density profile -------------------------------- +subplot(232) +plot(r, rho, '.', color='r', ms=0.5, alpha=0.2) +plot(x_s, rho_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, rho_bin, yerr=rho_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) +xlabel("${\\rm{Radius}}~r$", labelpad=0) +ylabel("${\\rm{Density}}~\\rho$", labelpad=0) +xlim(0, 0.5) +ylim(0.95, 19) + +# Pressure profile -------------------------------- +subplot(233) +plot(r, P, '.', color='r', ms=0.5, alpha=0.2) +plot(x_s, P_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, P_bin, yerr=P_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) +xlabel("${\\rm{Radius}}~r$", labelpad=0) +ylabel("${\\rm{Pressure}}~P$", labelpad=0) +xlim(0, 0.5) +ylim(-0.5, 11) + +# Internal energy profile ------------------------- +subplot(234) +plot(r, u, '.', color='r', ms=0.5, alpha=0.2) +plot(x_s, u_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, u_bin, yerr=u_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) +xlabel("${\\rm{Radius}}~r$", labelpad=0) +ylabel("${\\rm{Internal~Energy}}~u$", labelpad=0) +xlim(0, 0.5) +ylim(-0.05, 0.8) + +# Entropy profile --------------------------------- +subplot(235) +plot(r, S, '.', color='r', ms=0.5, alpha=0.2) +plot(x_s, s_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, S_bin, yerr=S_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) +xlabel("${\\rm{Radius}}~r$", labelpad=0) +ylabel("${\\rm{Entropy}}~S$", labelpad=-9) +xlim(0, 0.5) +ylim(-0.05, 0.2) + +# Information ------------------------------------- +subplot(236, frameon=False) + +text(-0.49, 0.9, "Noh problem with $\\gamma=%.3f$ in 2D at $t=%.2f$"%(gas_gamma,time), fontsize=10) +text(-0.49, 0.8, "ICs:~~ $(P_0, \\rho_0, v_0) = (%1.2e, %.3f, %.3f)$"%(1e-6, 1., -1.), fontsize=10) +plot([-0.49, 0.1], [0.62, 0.62], 'k-', lw=1) +text(-0.49, 0.5, "$\\textsc{Swift}$ %s"%git, fontsize=10) +text(-0.49, 0.4, scheme, fontsize=10) +text(-0.49, 0.3, kernel, fontsize=10) +text(-0.49, 0.2, "$%.2f$ neighbours ($\\eta=%.3f$)"%(neighbours, eta), fontsize=10) +xlim(-0.5, 0.5) +ylim(0, 1) +xticks([]) +yticks([]) + + +savefig("Noh.png", dpi=200) diff --git a/examples/Noh_2D/run.sh b/examples/Noh_2D/run.sh new file mode 100755 index 0000000000000000000000000000000000000000..cff200801018e04ea560bd2c3fbd84057aec2d7c --- /dev/null +++ b/examples/Noh_2D/run.sh @@ -0,0 +1,19 @@ +#!/bin/bash + + # Generate the initial conditions if they are not present. +if [ ! -e glassPlane_128.hdf5 ] +then + echo "Fetching initial glass file for the Noh problem..." + ./getGlass.sh +fi +if [ ! -e noh.hdf5 ] +then + echo "Generating initial conditions for the Noh problem..." + python makeIC.py +fi + +# Run SWIFT +../swift -s -t 2 noh.yml 2>&1 | tee output.log + +# Plot the solution +python plotSolution.py 12 diff --git a/examples/Noh_3D/getGlass.sh b/examples/Noh_3D/getGlass.sh new file mode 100755 index 0000000000000000000000000000000000000000..d5c5f590ac37c9c9431d626a2ea61b0c12c1513c --- /dev/null +++ b/examples/Noh_3D/getGlass.sh @@ -0,0 +1,2 @@ +#!/bin/bash +wget http://virgodb.cosma.dur.ac.uk/swift-webstorage/ICs/glassCube_64.hdf5 diff --git a/examples/Noh_3D/makeIC.py b/examples/Noh_3D/makeIC.py new file mode 100644 index 0000000000000000000000000000000000000000..0c25a5c8b3e967185cf16bae4b1f21c215266def --- /dev/null +++ b/examples/Noh_3D/makeIC.py @@ -0,0 +1,100 @@ + +############################################################################### + # This file is part of SWIFT. + # Copyright (c) 2016 Matthieu Schaller (matthieu.schaller@durham.ac.uk) + # + # This program is free software: you can redistribute it and/or modify + # it under the terms of the GNU Lesser General Public License as published + # by the Free Software Foundation, either version 3 of the License, or + # (at your option) any later version. + # + # This program is distributed in the hope that it will be useful, + # but WITHOUT ANY WARRANTY; without even the implied warranty of + # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + # GNU General Public License for more details. + # + # You should have received a copy of the GNU Lesser General Public License + # along with this program. If not, see <http://www.gnu.org/licenses/>. + # + ############################################################################## + +import h5py +from numpy import * + +# Generates a swift IC file for the 3D Noh problem in a periodic box + +# Parameters +gamma = 5./3. # Gas adiabatic index +gamma = 5./3. # Gas adiabatic index +rho0 = 1. # Background density +P0 = 1.e-6 # Background pressure +fileName = "noh.hdf5" + +#--------------------------------------------------- +glass = h5py.File("glassCube_64.hdf5", "r") + +vol = 8. + +pos = glass["/PartType0/Coordinates"][:,:] * vol**(1./3.) +h = glass["/PartType0/SmoothingLength"][:] * vol**(1./3.) +numPart = size(h) + +# Generate extra arrays +v = zeros((numPart, 3)) +ids = linspace(1, numPart, numPart) + +m = zeros(numPart) +u = zeros(numPart) +m[:] = rho0 * vol / numPart +u[:] = P0 / (rho0 * (gamma - 1)) + +# Make radial velocities +#r = sqrt((pos[:,0]-1.)**2 + (pos[:,1]-1.)**2) +#theta = arctan2((pos[:,1]-1.), (pos[:,0]-1.)) +v[:,0] = -(pos[:,0] - 1.) +v[:,1] = -(pos[:,1] - 1.) +v[:,2] = -(pos[:,2] - 1.) + +norm_v = sqrt(v[:,0]**2 + v[:,1]**2 + v[:,2]**2) +v[:,0] /= norm_v +v[:,1] /= norm_v +v[:,2] /= norm_v + +#File +file = h5py.File(fileName, 'w') + +# Header +grp = file.create_group("/Header") +grp.attrs["BoxSize"] = [vol**(1./3.), vol**(1./3.), vol**(1./3.)] +grp.attrs["NumPart_Total"] = [numPart, 0, 0, 0, 0, 0] +grp.attrs["NumPart_Total_HighWord"] = [0, 0, 0, 0, 0, 0] +grp.attrs["NumPart_ThisFile"] = [numPart, 0, 0, 0, 0, 0] +grp.attrs["Time"] = 0.0 +grp.attrs["NumFilesPerSnapshot"] = 1 +grp.attrs["MassTable"] = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +grp.attrs["Flag_Entropy_ICs"] = 0 +grp.attrs["Dimension"] = 3 + +#Runtime parameters +grp = file.create_group("/RuntimePars") +grp.attrs["PeriodicBoundariesOn"] = 1 + +#Units +grp = file.create_group("/Units") +grp.attrs["Unit length in cgs (U_L)"] = 1. +grp.attrs["Unit mass in cgs (U_M)"] = 1. +grp.attrs["Unit time in cgs (U_t)"] = 1. +grp.attrs["Unit current in cgs (U_I)"] = 1. +grp.attrs["Unit temperature in cgs (U_T)"] = 1. + +#Particle group +grp = file.create_group("/PartType0") +grp.create_dataset('Coordinates', data=pos, dtype='d') +grp.create_dataset('Velocities', data=v, dtype='f') +grp.create_dataset('Masses', data=m, dtype='f') +grp.create_dataset('SmoothingLength', data=h, dtype='f') +grp.create_dataset('InternalEnergy', data=u, dtype='f') +grp.create_dataset('ParticleIDs', data=ids, dtype='L') + + +file.close() diff --git a/examples/Noh_3D/noh.yml b/examples/Noh_3D/noh.yml new file mode 100644 index 0000000000000000000000000000000000000000..911ebbdc1ca82f9b5cf2f70841d848008cbc5f6c --- /dev/null +++ b/examples/Noh_3D/noh.yml @@ -0,0 +1,35 @@ +# Define the system of units to use internally. +InternalUnitSystem: + UnitMass_in_cgs: 1 # Grams + UnitLength_in_cgs: 1 # Centimeters + UnitVelocity_in_cgs: 1 # Centimeters per second + UnitCurrent_in_cgs: 1 # Amperes + UnitTemp_in_cgs: 1 # Kelvin + +# Parameters governing the time integration +TimeIntegration: + time_begin: 0. # The starting time of the simulation (in internal units). + time_end: 0.6 # The end time of the simulation (in internal units). + dt_min: 1e-7 # The minimal time-step size of the simulation (in internal units). + dt_max: 1e-3 # The maximal time-step size of the simulation (in internal units). + +# Parameters governing the snapshots +Snapshots: + basename: noh # Common part of the name of output files + time_first: 0. # Time of the first output (in internal units) + delta_time: 5e-2 # Time difference between consecutive outputs (in internal units) + +# Parameters governing the conserved quantities statistics +Statistics: + delta_time: 1e-5 # Time between statistics output + +# Parameters for the hydrodynamics scheme +SPH: + resolution_eta: 1.2348 # Target smoothing length in units of the mean inter-particle separation (1.2348 == 48Ngbs with the cubic spline kernel). + delta_neighbours: 0.1 # The tolerance for the targetted number of neighbours. + CFL_condition: 0.1 # Courant-Friedrich-Levy condition for time integration. + +# Parameters related to the initial conditions +InitialConditions: + file_name: ./noh.hdf5 # The file to read + diff --git a/examples/Noh_3D/plotSolution.py b/examples/Noh_3D/plotSolution.py new file mode 100644 index 0000000000000000000000000000000000000000..1742e13a5daeff392690a9804fb2831ef4304963 --- /dev/null +++ b/examples/Noh_3D/plotSolution.py @@ -0,0 +1,202 @@ +############################################################################### + # This file is part of SWIFT. + # Copyright (c) 2016 Matthieu Schaller (matthieu.schaller@durham.ac.uk) + # + # This program is free software: you can redistribute it and/or modify + # it under the terms of the GNU Lesser General Public License as published + # by the Free Software Foundation, either version 3 of the License, or + # (at your option) any later version. + # + # This program is distributed in the hope that it will be useful, + # but WITHOUT ANY WARRANTY; without even the implied warranty of + # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + # GNU General Public License for more details. + # + # You should have received a copy of the GNU Lesser General Public License + # along with this program. If not, see <http://www.gnu.org/licenses/>. + # + ############################################################################## + +# Computes the analytical solution of the Noh problem and plots the SPH answer + + +# Parameters +gas_gamma = 5./3. # Polytropic index +rho0 = 1. # Background density +P0 = 1.e-6 # Background pressure +v0 = 1 + +import matplotlib +matplotlib.use("Agg") +from pylab import * +from scipy import stats +import h5py + +# Plot parameters +params = {'axes.labelsize': 10, +'axes.titlesize': 10, +'font.size': 12, +'legend.fontsize': 12, +'xtick.labelsize': 10, +'ytick.labelsize': 10, +'text.usetex': True, + 'figure.figsize' : (9.90,6.45), +'figure.subplot.left' : 0.045, +'figure.subplot.right' : 0.99, +'figure.subplot.bottom' : 0.05, +'figure.subplot.top' : 0.99, +'figure.subplot.wspace' : 0.15, +'figure.subplot.hspace' : 0.12, +'lines.markersize' : 6, +'lines.linewidth' : 3., +'text.latex.unicode': True +} +rcParams.update(params) +rc('font',**{'family':'sans-serif','sans-serif':['Times']}) + + +snap = int(sys.argv[1]) + + +# Read the simulation data +sim = h5py.File("noh_%03d.hdf5"%snap, "r") +boxSize = sim["/Header"].attrs["BoxSize"][0] +time = sim["/Header"].attrs["Time"][0] +scheme = sim["/HydroScheme"].attrs["Scheme"] +kernel = sim["/HydroScheme"].attrs["Kernel function"] +neighbours = sim["/HydroScheme"].attrs["Kernel target N_ngb"] +eta = sim["/HydroScheme"].attrs["Kernel eta"] +git = sim["Code"].attrs["Git Revision"] + +x = sim["/PartType0/Coordinates"][:,0] +y = sim["/PartType0/Coordinates"][:,1] +z = sim["/PartType0/Coordinates"][:,2] +vx = sim["/PartType0/Velocities"][:,0] +vy = sim["/PartType0/Velocities"][:,1] +vz = sim["/PartType0/Velocities"][:,2] +u = sim["/PartType0/InternalEnergy"][:] +S = sim["/PartType0/Entropy"][:] +P = sim["/PartType0/Pressure"][:] +rho = sim["/PartType0/Density"][:] + +r = np.sqrt((x-1)**2 + (y-1)**2 + (z-1)**2) +v = -np.sqrt(vx**2 + vy**2 + vz**2) + +# Bin te data +r_bin_edge = np.arange(0., 1., 0.02) +r_bin = 0.5*(r_bin_edge[1:] + r_bin_edge[:-1]) +rho_bin,_,_ = stats.binned_statistic(r, rho, statistic='mean', bins=r_bin_edge) +v_bin,_,_ = stats.binned_statistic(r, v, statistic='mean', bins=r_bin_edge) +P_bin,_,_ = stats.binned_statistic(r, P, statistic='mean', bins=r_bin_edge) +S_bin,_,_ = stats.binned_statistic(r, S, statistic='mean', bins=r_bin_edge) +u_bin,_,_ = stats.binned_statistic(r, u, statistic='mean', bins=r_bin_edge) +rho2_bin,_,_ = stats.binned_statistic(r, rho**2, statistic='mean', bins=r_bin_edge) +v2_bin,_,_ = stats.binned_statistic(r, v**2, statistic='mean', bins=r_bin_edge) +P2_bin,_,_ = stats.binned_statistic(r, P**2, statistic='mean', bins=r_bin_edge) +S2_bin,_,_ = stats.binned_statistic(r, S**2, statistic='mean', bins=r_bin_edge) +u2_bin,_,_ = stats.binned_statistic(r, u**2, statistic='mean', bins=r_bin_edge) +rho_sigma_bin = np.sqrt(rho2_bin - rho_bin**2) +v_sigma_bin = np.sqrt(v2_bin - v_bin**2) +P_sigma_bin = np.sqrt(P2_bin - P_bin**2) +S_sigma_bin = np.sqrt(S2_bin - S_bin**2) +u_sigma_bin = np.sqrt(u2_bin - u_bin**2) + + +# Analytic solution +N = 1000 # Number of points + +x_s = np.arange(0, 2., 2./N) - 1. +rho_s = np.ones(N) * rho0 +P_s = np.ones(N) * rho0 +v_s = np.ones(N) * v0 + +# Shock position +u0 = rho0 * P0 * (gas_gamma-1) +us = 0.5 * (gas_gamma - 1) * v0 +rs = us * time + +# Post-shock values +rho_s[np.abs(x_s) < rs] = rho0 * ((gas_gamma + 1) / (gas_gamma - 1))**3 +P_s[np.abs(x_s) < rs] = 0.5 * rho0 * v0**2 * (gas_gamma + 1)**3 / (gas_gamma-1)**2 +v_s[np.abs(x_s) < rs] = 0. + +# Pre-shock values +rho_s[np.abs(x_s) >= rs] = rho0 * (1 + v0 * time/np.abs(x_s[np.abs(x_s) >=rs]))**2 +P_s[np.abs(x_s) >= rs] = 0 +v_s[x_s >= rs] = -v0 +v_s[x_s <= -rs] = v0 + +# Additional arrays +u_s = P_s / (rho_s * (gas_gamma - 1.)) #internal energy +s_s = P_s / rho_s**gas_gamma # entropic function + +# Plot the interesting quantities +figure() + +# Velocity profile -------------------------------- +subplot(231) +plot(r, v, '.', color='r', ms=0.5, alpha=0.2) +plot(x_s, v_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, v_bin, yerr=v_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) +xlabel("${\\rm{Radius}}~r$", labelpad=0) +ylabel("${\\rm{Velocity}}~v_r$", labelpad=-4) +xlim(0, 0.5) +ylim(-1.2, 0.4) + +# Density profile -------------------------------- +subplot(232) +plot(r, rho, '.', color='r', ms=0.5, alpha=0.2) +plot(x_s, rho_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, rho_bin, yerr=rho_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) +xlabel("${\\rm{Radius}}~r$", labelpad=0) +ylabel("${\\rm{Density}}~\\rho$", labelpad=0) +xlim(0, 0.5) +ylim(0.95, 71) + +# Pressure profile -------------------------------- +subplot(233) +plot(r, P, '.', color='r', ms=0.5, alpha=0.2) +plot(x_s, P_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, P_bin, yerr=P_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) +xlabel("${\\rm{Radius}}~r$", labelpad=0) +ylabel("${\\rm{Pressure}}~P$", labelpad=0) +xlim(0, 0.5) +ylim(-0.5, 25) + +# Internal energy profile ------------------------- +subplot(234) +plot(r, u, '.', color='r', ms=0.5, alpha=0.2) +plot(x_s, u_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, u_bin, yerr=u_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) +xlabel("${\\rm{Radius}}~r$", labelpad=0) +ylabel("${\\rm{Internal~Energy}}~u$", labelpad=0) +xlim(0, 0.5) +ylim(-0.05, 0.8) + +# Entropy profile --------------------------------- +subplot(235) +plot(r, S, '.', color='r', ms=0.5, alpha=0.2) +plot(x_s, s_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, S_bin, yerr=S_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) +xlabel("${\\rm{Radius}}~r$", labelpad=0) +ylabel("${\\rm{Entropy}}~S$", labelpad=-9) +xlim(0, 0.5) +ylim(-0.05, 0.2) + +# Information ------------------------------------- +subplot(236, frameon=False) + +text(-0.49, 0.9, "Noh problem with $\\gamma=%.3f$ in 3D at $t=%.2f$"%(gas_gamma,time), fontsize=10) +text(-0.49, 0.8, "ICs:~~ $(P_0, \\rho_0, v_0) = (%1.2e, %.3f, %.3f)$"%(1e-6, 1., -1.), fontsize=10) +plot([-0.49, 0.1], [0.62, 0.62], 'k-', lw=1) +text(-0.49, 0.5, "$\\textsc{Swift}$ %s"%git, fontsize=10) +text(-0.49, 0.4, scheme, fontsize=10) +text(-0.49, 0.3, kernel, fontsize=10) +text(-0.49, 0.2, "$%.2f$ neighbours ($\\eta=%.3f$)"%(neighbours, eta), fontsize=10) +xlim(-0.5, 0.5) +ylim(0, 1) +xticks([]) +yticks([]) + + +savefig("Noh.png", dpi=200) diff --git a/examples/Noh_3D/run.sh b/examples/Noh_3D/run.sh new file mode 100755 index 0000000000000000000000000000000000000000..b9e4fb145b2465433aa2bc0362aba19cc1267461 --- /dev/null +++ b/examples/Noh_3D/run.sh @@ -0,0 +1,19 @@ +#!/bin/bash + + # Generate the initial conditions if they are not present. +if [ ! -e glassCube_64.hdf5 ] +then + echo "Fetching initial glass file for the Noh problem..." + ./getGlass.sh +fi +if [ ! -e noh.hdf5 ] +then + echo "Generating initial conditions for the Noh problem..." + python makeIC.py +fi + +# Run SWIFT +../swift -s -t 2 noh.yml 2>&1 | tee output.log + +# Plot the solution +python plotSolution.py 12 diff --git a/examples/SedovBlast_2D/plotSolution.py b/examples/SedovBlast_2D/plotSolution.py index 69e4e1232dd5c14b06e8a705f4add391f1f597f0..d8c0c9791d1834cc2a5cf0103b46a49e20d2e8a3 100644 --- a/examples/SedovBlast_2D/plotSolution.py +++ b/examples/SedovBlast_2D/plotSolution.py @@ -35,6 +35,7 @@ gas_gamma = 5./3. # Gas polytropic index import matplotlib matplotlib.use("Agg") from pylab import * +from scipy import stats import h5py # Plot parameters @@ -84,6 +85,24 @@ S = sim["/PartType0/Entropy"][:] P = sim["/PartType0/Pressure"][:] rho = sim["/PartType0/Density"][:] +# Bin te data +r_bin_edge = np.arange(0., 0.5, 0.01) +r_bin = 0.5*(r_bin_edge[1:] + r_bin_edge[:-1]) +rho_bin,_,_ = stats.binned_statistic(r, rho, statistic='mean', bins=r_bin_edge) +v_bin,_,_ = stats.binned_statistic(r, v_r, statistic='mean', bins=r_bin_edge) +P_bin,_,_ = stats.binned_statistic(r, P, statistic='mean', bins=r_bin_edge) +S_bin,_,_ = stats.binned_statistic(r, S, statistic='mean', bins=r_bin_edge) +u_bin,_,_ = stats.binned_statistic(r, u, statistic='mean', bins=r_bin_edge) +rho2_bin,_,_ = stats.binned_statistic(r, rho**2, statistic='mean', bins=r_bin_edge) +v2_bin,_,_ = stats.binned_statistic(r, v_r**2, statistic='mean', bins=r_bin_edge) +P2_bin,_,_ = stats.binned_statistic(r, P**2, statistic='mean', bins=r_bin_edge) +S2_bin,_,_ = stats.binned_statistic(r, S**2, statistic='mean', bins=r_bin_edge) +u2_bin,_,_ = stats.binned_statistic(r, u**2, statistic='mean', bins=r_bin_edge) +rho_sigma_bin = np.sqrt(rho2_bin - rho_bin**2) +v_sigma_bin = np.sqrt(v2_bin - v_bin**2) +P_sigma_bin = np.sqrt(P2_bin - P_bin**2) +S_sigma_bin = np.sqrt(S2_bin - S_bin**2) +u_sigma_bin = np.sqrt(u2_bin - u_bin**2) # Now, work our the solution.... @@ -213,8 +232,9 @@ figure() # Velocity profile -------------------------------- subplot(231) -plot(r, v_r, '.', color='r', ms=1.) +plot(r, v_r, '.', color='r', ms=0.5, alpha=0.2) plot(r_s, v_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, v_bin, yerr=v_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) xlabel("${\\rm{Radius}}~r$", labelpad=0) ylabel("${\\rm{Radial~velocity}}~v_r$", labelpad=0) xlim(0, 1.3 * r_shock) @@ -222,8 +242,9 @@ ylim(-0.2, 3.8) # Density profile -------------------------------- subplot(232) -plot(r, rho, '.', color='r', ms=1.) +plot(r, rho, '.', color='r', ms=0.5, alpha=0.2) plot(r_s, rho_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, rho_bin, yerr=rho_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) xlabel("${\\rm{Radius}}~r$", labelpad=0) ylabel("${\\rm{Density}}~\\rho$", labelpad=2) xlim(0, 1.3 * r_shock) @@ -231,8 +252,9 @@ ylim(-0.2, 5.2) # Pressure profile -------------------------------- subplot(233) -plot(r, P, '.', color='r', ms=1.) +plot(r, P, '.', color='r', ms=0.5, alpha=0.2) plot(r_s, P_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, P_bin, yerr=P_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) xlabel("${\\rm{Radius}}~r$", labelpad=0) ylabel("${\\rm{Pressure}}~P$", labelpad=0) xlim(0, 1.3 * r_shock) @@ -240,8 +262,9 @@ ylim(-1, 12.5) # Internal energy profile ------------------------- subplot(234) -plot(r, u, '.', color='r', ms=1.) +plot(r, u, '.', color='r', ms=0.5, alpha=0.2) plot(r_s, u_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, u_bin, yerr=u_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) xlabel("${\\rm{Radius}}~r$", labelpad=0) ylabel("${\\rm{Internal~Energy}}~u$", labelpad=0) xlim(0, 1.3 * r_shock) @@ -249,8 +272,9 @@ ylim(-2, 22) # Entropy profile --------------------------------- subplot(235) -plot(r, S, '.', color='r', ms=1.) +plot(r, S, '.', color='r', ms=0.5, alpha=0.2) plot(r_s, s_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, S_bin, yerr=S_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) xlabel("${\\rm{Radius}}~r$", labelpad=0) ylabel("${\\rm{Entropy}}~S$", labelpad=0) xlim(0, 1.3 * r_shock) diff --git a/examples/SedovBlast_3D/plotSolution.py b/examples/SedovBlast_3D/plotSolution.py index 8eca94c98acd7e081cdfc2c785aa1d19d601e81e..6e90a9a43524b3cdb279054764b71fd1b546b366 100644 --- a/examples/SedovBlast_3D/plotSolution.py +++ b/examples/SedovBlast_3D/plotSolution.py @@ -35,6 +35,7 @@ gas_gamma = 5./3. # Gas polytropic index import matplotlib matplotlib.use("Agg") from pylab import * +from scipy import stats import h5py # Plot parameters @@ -85,6 +86,25 @@ S = sim["/PartType0/Entropy"][:] P = sim["/PartType0/Pressure"][:] rho = sim["/PartType0/Density"][:] +# Bin te data +r_bin_edge = np.arange(0., 0.5, 0.01) +r_bin = 0.5*(r_bin_edge[1:] + r_bin_edge[:-1]) +rho_bin,_,_ = stats.binned_statistic(r, rho, statistic='mean', bins=r_bin_edge) +v_bin,_,_ = stats.binned_statistic(r, v_r, statistic='mean', bins=r_bin_edge) +P_bin,_,_ = stats.binned_statistic(r, P, statistic='mean', bins=r_bin_edge) +S_bin,_,_ = stats.binned_statistic(r, S, statistic='mean', bins=r_bin_edge) +u_bin,_,_ = stats.binned_statistic(r, u, statistic='mean', bins=r_bin_edge) +rho2_bin,_,_ = stats.binned_statistic(r, rho**2, statistic='mean', bins=r_bin_edge) +v2_bin,_,_ = stats.binned_statistic(r, v_r**2, statistic='mean', bins=r_bin_edge) +P2_bin,_,_ = stats.binned_statistic(r, P**2, statistic='mean', bins=r_bin_edge) +S2_bin,_,_ = stats.binned_statistic(r, S**2, statistic='mean', bins=r_bin_edge) +u2_bin,_,_ = stats.binned_statistic(r, u**2, statistic='mean', bins=r_bin_edge) +rho_sigma_bin = np.sqrt(rho2_bin - rho_bin**2) +v_sigma_bin = np.sqrt(v2_bin - v_bin**2) +P_sigma_bin = np.sqrt(P2_bin - P_bin**2) +S_sigma_bin = np.sqrt(S2_bin - S_bin**2) +u_sigma_bin = np.sqrt(u2_bin - u_bin**2) + # Now, work our the solution.... @@ -214,8 +234,9 @@ figure() # Velocity profile -------------------------------- subplot(231) -plot(r, v_r, '.', color='r', ms=1.) +plot(r, v_r, '.', color='r', ms=0.5, alpha=0.2) plot(r_s, v_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, v_bin, yerr=v_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) xlabel("${\\rm{Radius}}~r$", labelpad=0) ylabel("${\\rm{Radial~velocity}}~v_r$", labelpad=0) xlim(0, 1.3 * r_shock) @@ -223,8 +244,9 @@ ylim(-0.2, 3.8) # Density profile -------------------------------- subplot(232) -plot(r, rho, '.', color='r', ms=1.) +plot(r, rho, '.', color='r', ms=0.5, alpha=0.2) plot(r_s, rho_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, rho_bin, yerr=rho_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) xlabel("${\\rm{Radius}}~r$", labelpad=0) ylabel("${\\rm{Density}}~\\rho$", labelpad=2) xlim(0, 1.3 * r_shock) @@ -232,8 +254,9 @@ ylim(-0.2, 5.2) # Pressure profile -------------------------------- subplot(233) -plot(r, P, '.', color='r', ms=1.) +plot(r, P, '.', color='r', ms=0.5, alpha=0.2) plot(r_s, P_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, P_bin, yerr=P_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) xlabel("${\\rm{Radius}}~r$", labelpad=0) ylabel("${\\rm{Pressure}}~P$", labelpad=0) xlim(0, 1.3 * r_shock) @@ -241,8 +264,9 @@ ylim(-1, 12.5) # Internal energy profile ------------------------- subplot(234) -plot(r, u, '.', color='r', ms=1.) +plot(r, u, '.', color='r', ms=0.5, alpha=0.2) plot(r_s, u_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, u_bin, yerr=u_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) xlabel("${\\rm{Radius}}~r$", labelpad=0) ylabel("${\\rm{Internal~Energy}}~u$", labelpad=0) xlim(0, 1.3 * r_shock) @@ -250,8 +274,9 @@ ylim(-2, 22) # Entropy profile --------------------------------- subplot(235) -plot(r, S, '.', color='r', ms=1.) +plot(r, S, '.', color='r', ms=0.5, alpha=0.2) plot(r_s, s_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(r_bin, S_bin, yerr=S_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) xlabel("${\\rm{Radius}}~r$", labelpad=0) ylabel("${\\rm{Entropy}}~S$", labelpad=0) xlim(0, 1.3 * r_shock) diff --git a/examples/SineWavePotential_1D/makeIC.py b/examples/SineWavePotential_1D/makeIC.py new file mode 100644 index 0000000000000000000000000000000000000000..321af0714219dfa0c7cbb3d80845881dcbb8416d --- /dev/null +++ b/examples/SineWavePotential_1D/makeIC.py @@ -0,0 +1,99 @@ +############################################################################### +# This file is part of SWIFT. +# Copyright (c) 2017 Bert Vandenbroucke (bert.vandenbroucke@gmail.com) +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +############################################################################## + +# Generates a distorted 1D grid with a density profile that balances out the +# external sine wave potential if run with an isothermal equation of state. + +import numpy as np +import h5py + +# constant thermal energy +# the definition below assumes the same thermal energy is defined in const.h, +# and also that the code was configured with an adiabatic index of 5./3. +uconst = 20.2615290634 +cs2 = 2.*uconst/3. +A = 10. + +fileName = "sineWavePotential.hdf5" +numPart = 100 +boxSize = 1. + +coords = np.zeros((numPart, 3)) +v = np.zeros((numPart, 3)) +m = np.zeros(numPart) + 1. +h = np.zeros(numPart) + 2./numPart +u = np.zeros(numPart) + uconst +ids = np.arange(numPart, dtype = 'L') +rho = np.zeros(numPart) + +# first set the positions, as we try to do a reasonable volume estimate to +# set the masses +for i in range(numPart): + coords[i,0] = (i+np.random.random())/numPart + +for i in range(numPart): + # reasonable mass estimate (actually not really good, but better than assuming + # a constant volume) + if i == 0: + V = 0.5*(coords[1,0]-coords[-1,0]+1.) + elif i == numPart-1: + V = 0.5*(coords[0,0]+1.-coords[-2,0]) + else: + V = 0.5*(coords[i+1,0] - coords[i-1,0]) + rho[i] = 1000.*np.exp(-0.5*A/np.pi/cs2*np.cos(2.*np.pi*coords[i,0])) + m[i] = rho[i]*V + +#File +file = h5py.File(fileName, 'w') + +# Header +grp = file.create_group("/Header") +grp.attrs["BoxSize"] = boxSize +grp.attrs["NumPart_Total"] = [numPart, 0, 0, 0, 0, 0] +grp.attrs["NumPart_Total_HighWord"] = [0, 0, 0, 0, 0, 0] +grp.attrs["NumPart_ThisFile"] = [numPart, 0, 0, 0, 0, 0] +grp.attrs["Time"] = 0.0 +grp.attrs["NumFilesPerSnapshot"] = 1 +grp.attrs["MassTable"] = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +grp.attrs["Flag_Entropy_ICs"] = 0 +grp.attrs["Dimension"] = 1 + +#Runtime parameters +grp = file.create_group("/RuntimePars") +grp.attrs["PeriodicBoundariesOn"] = 1 + +#Units +grp = file.create_group("/Units") +grp.attrs["Unit length in cgs (U_L)"] = 1. +grp.attrs["Unit mass in cgs (U_M)"] = 1. +grp.attrs["Unit time in cgs (U_t)"] = 1. +grp.attrs["Unit current in cgs (U_I)"] = 1. +grp.attrs["Unit temperature in cgs (U_T)"] = 1. + +#Particle group +grp = file.create_group("/PartType0") +grp.create_dataset('Coordinates', data=coords, dtype='d') +grp.create_dataset('Velocities', data=v, dtype='f') +grp.create_dataset('Masses', data=m, dtype='f') +grp.create_dataset('SmoothingLength', data=h, dtype='f') +grp.create_dataset('InternalEnergy', data=u, dtype='f') +grp.create_dataset('ParticleIDs', data=ids, dtype='L') +grp.create_dataset('Density', data=rho, dtype='f') + +file.close() diff --git a/examples/SineWavePotential_1D/plotSolution.py b/examples/SineWavePotential_1D/plotSolution.py new file mode 100644 index 0000000000000000000000000000000000000000..65e981e4648fe0fe5d1da6cf3e753fb8a34f0fb4 --- /dev/null +++ b/examples/SineWavePotential_1D/plotSolution.py @@ -0,0 +1,77 @@ +############################################################################### +# This file is part of SWIFT. +# Copyright (c) 2017 Bert Vandenbroucke (bert.vandenbroucke@gmail.com) +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +############################################################################## + +# Plots some quantities for the snapshot file which is passed on as a command +# line argument (full name) + +import numpy as np +import h5py +import sys +import pylab as pl + +# these should be the same as in makeIC.py +uconst = 20.2615290634 +cs2 = 2.*uconst/3. +A = 10. + +if len(sys.argv) < 2: + print "Need to provide a filename argument!" + exit() + +fileName = sys.argv[1] + +file = h5py.File(fileName, 'r') +coords = np.array(file["/PartType0/Coordinates"]) +rho = np.array(file["/PartType0/Density"]) +u = np.array(file["/PartType0/InternalEnergy"]) +agrav = np.array(file["/PartType0/GravAcceleration"]) +m = np.array(file["/PartType0/Masses"]) +ids = np.array(file["/PartType0/ParticleIDs"]) + +x = np.linspace(0., 1., 1000) +rho_x = 1000.*np.exp(-0.5*A/np.pi/cs2*np.cos(2.*np.pi*x)) + +P = cs2*rho + +ids_reverse = np.argsort(ids) + +gradP = np.zeros(P.shape) +for i in range(len(P)): + iself = int(ids[i]) + corr = 0. + im1 = iself-1 + if im1 < 0: + im1 = len(P)-1 + corr = 1. + ip1 = iself+1 + if ip1 == len(P): + ip1 = 0 + corr = 1. + idxp1 = ids_reverse[ip1] + idxm1 = ids_reverse[im1] + gradP[i] = (P[idxp1]-P[idxm1])/(coords[idxp1,0]-coords[idxm1,0]) + +fig, ax = pl.subplots(2, 2) + +ax[0][0].plot(coords[:,0], rho, "r.", markersize = 0.5) +ax[0][0].plot(x, rho_x, "g-") +ax[0][1].plot(coords[:,0], gradP/rho, "b.") +ax[1][0].plot(coords[:,0], agrav[:,0], "g.", markersize = 0.5) +ax[1][1].plot(coords[:,0], m, "y.") +pl.savefig("{fileName}.png".format(fileName = fileName[:-5])) diff --git a/examples/SineWavePotential_1D/run.sh b/examples/SineWavePotential_1D/run.sh new file mode 100755 index 0000000000000000000000000000000000000000..077cf1c0cc64ef7a85cfd0e67f8f490b0de4ba37 --- /dev/null +++ b/examples/SineWavePotential_1D/run.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +if [ ! -e sineWavePotential.hdf5 ] +then + echo "Generating initial conditions for the 1D SineWavePotential example..." + python makeIC.py +fi + +../swift -g -s -t 2 sineWavePotential.yml 2>&1 | tee output.log + +for f in sineWavePotential_*.hdf5 +do + python plotSolution.py $f +done diff --git a/examples/SineWavePotential_1D/sineWavePotential.yml b/examples/SineWavePotential_1D/sineWavePotential.yml new file mode 100644 index 0000000000000000000000000000000000000000..1f98b47a0e3c72e77a2024aebf4dcd863ed02e9b --- /dev/null +++ b/examples/SineWavePotential_1D/sineWavePotential.yml @@ -0,0 +1,39 @@ +# Define the system of units to use internally. +InternalUnitSystem: + UnitMass_in_cgs: 1. + UnitLength_in_cgs: 1. + UnitVelocity_in_cgs: 1. + UnitCurrent_in_cgs: 1. + UnitTemp_in_cgs: 1. + +# Parameters governing the time integration +TimeIntegration: + time_begin: 0. # The starting time of the simulation (in internal units). + time_end: 10. # The end time of the simulation (in internal units). + dt_min: 1e-6 # The minimal time-step size of the simulation (in internal units). + dt_max: 1e-2 # The maximal time-step size of the simulation (in internal units). + +# Parameters governing the conserved quantities statistics +Statistics: + delta_time: 1e-2 # Time between statistics output + +# Parameters governing the snapshots +Snapshots: + basename: sineWavePotential # Common part of the name of output files + time_first: 0. # Time of the first output (in internal units) + delta_time: 1. # Time difference between consecutive outputs (in internal units) + +# Parameters for the hydrodynamics scheme +SPH: + resolution_eta: 1.2349 # Target smoothing length in units of the mean inter-particle separation (1.2349 == 48Ngbs with the cubic spline kernel). + delta_neighbours: 0.1 # The tolerance for the targetted number of neighbours. + CFL_condition: 0.1 # Courant-Friedrich-Levy condition for time integration. + +# Parameters related to the initial conditions +InitialConditions: + file_name: sineWavePotential.hdf5 # The file to read + +# External potential parameters +SineWavePotential: + amplitude: 10. + timestep_limit: 1. diff --git a/examples/SineWavePotential_2D/makeIC.py b/examples/SineWavePotential_2D/makeIC.py new file mode 100644 index 0000000000000000000000000000000000000000..62ae89f8f52bff9c0db37cd537f286ab817da3fe --- /dev/null +++ b/examples/SineWavePotential_2D/makeIC.py @@ -0,0 +1,95 @@ +############################################################################### +# This file is part of SWIFT. +# Copyright (c) 2017 Bert Vandenbroucke (bert.vandenbroucke@gmail.com) +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +############################################################################## + +# Generates a distorted 1D grid with a density profile that balances out the +# external sine wave potential if run with an isothermal equation of state. + +import numpy as np +import h5py + +# constant thermal energy +# the definition below assumes the same thermal energy is defined in const.h, +# and also that the code was configured with an adiabatic index of 5./3. +uconst = 20.2615290634 +cs2 = 2.*uconst/3. +A = 10. + +fileName = "sineWavePotential.hdf5" +numPart_1D = 50 +boxSize = [1., 1.] +numPart = numPart_1D**2 + +coords = np.zeros((numPart, 3)) +v = np.zeros((numPart, 3)) +m = np.zeros(numPart) + 1. +h = np.zeros(numPart) + 2./numPart +u = np.zeros(numPart) + uconst +ids = np.arange(numPart, dtype = 'L') +rho = np.zeros(numPart) + +# first set the positions, as we try to do a reasonable volume estimate to +# set the masses +for i in range(numPart_1D): + for j in range(numPart_1D): + coords[numPart_1D*i+j,0] = (i+0.5)/numPart_1D + coords[numPart_1D*i+j,1] = (j+0.5)/numPart_1D + +V = 1./numPart +for i in range(numPart): + rho[i] = 1000.*np.exp(-0.5*A/np.pi/cs2*np.cos(2.*np.pi*coords[i,0])) + m[i] = rho[i]*V + +#File +file = h5py.File(fileName, 'w') + +# Header +grp = file.create_group("/Header") +grp.attrs["BoxSize"] = boxSize +grp.attrs["NumPart_Total"] = [numPart, 0, 0, 0, 0, 0] +grp.attrs["NumPart_Total_HighWord"] = [0, 0, 0, 0, 0, 0] +grp.attrs["NumPart_ThisFile"] = [numPart, 0, 0, 0, 0, 0] +grp.attrs["Time"] = 0.0 +grp.attrs["NumFilesPerSnapshot"] = 1 +grp.attrs["MassTable"] = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +grp.attrs["Flag_Entropy_ICs"] = 0 +grp.attrs["Dimension"] = 2 + +#Runtime parameters +grp = file.create_group("/RuntimePars") +grp.attrs["PeriodicBoundariesOn"] = 1 + +#Units +grp = file.create_group("/Units") +grp.attrs["Unit length in cgs (U_L)"] = 1. +grp.attrs["Unit mass in cgs (U_M)"] = 1. +grp.attrs["Unit time in cgs (U_t)"] = 1. +grp.attrs["Unit current in cgs (U_I)"] = 1. +grp.attrs["Unit temperature in cgs (U_T)"] = 1. + +#Particle group +grp = file.create_group("/PartType0") +grp.create_dataset('Coordinates', data=coords, dtype='d') +grp.create_dataset('Velocities', data=v, dtype='f') +grp.create_dataset('Masses', data=m, dtype='f') +grp.create_dataset('SmoothingLength', data=h, dtype='f') +grp.create_dataset('InternalEnergy', data=u, dtype='f') +grp.create_dataset('ParticleIDs', data=ids, dtype='L') +grp.create_dataset('Density', data=rho, dtype='f') + +file.close() diff --git a/examples/SineWavePotential_2D/plotSolution.py b/examples/SineWavePotential_2D/plotSolution.py new file mode 100644 index 0000000000000000000000000000000000000000..ee02f59c404db87a790465d2786e6296525e36b0 --- /dev/null +++ b/examples/SineWavePotential_2D/plotSolution.py @@ -0,0 +1,83 @@ +############################################################################### +# This file is part of SWIFT. +# Copyright (c) 2017 Bert Vandenbroucke (bert.vandenbroucke@gmail.com) +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +############################################################################## + +# Plots some quantities for the snapshot file which is passed on as a command +# line argument (full name) + +import numpy as np +import h5py +import sys +import pylab as pl + +# these should be the same as in makeIC.py +uconst = 20.2615290634 +cs2 = 2.*uconst/3. +A = 10. + +if len(sys.argv) < 2: + print "Need to provide a filename argument!" + exit() + +fileName = sys.argv[1] + +file = h5py.File(fileName, 'r') +coords = np.array(file["/PartType0/Coordinates"]) +rho = np.array(file["/PartType0/Density"]) +u = np.array(file["/PartType0/InternalEnergy"]) +agrav = np.array(file["/PartType0/GravAcceleration"]) +m = np.array(file["/PartType0/Masses"]) +ids = np.array(file["/PartType0/ParticleIDs"]) + +# ids_reverse gives the index original particle 0 now has in the particle arrays +# and so on +# note that this will only work if the particles do not move away too much from +# there initial positions +ids_reverse = np.argsort(ids) + +x = np.linspace(0., 1., 1000) +rho_x = 1000.*np.exp(-0.5*A/np.pi/cs2*np.cos(2.*np.pi*x)) + +P = cs2*rho + +n1D = int(np.sqrt(len(P))) +gradP = np.zeros(P.shape) +for i in range(len(P)): + iself = int(ids[i]/n1D) + jself = int(ids[i]-n1D*iself) + corr = 0. + im1 = iself-1 + if im1 < 0: + im1 = n1D-1 + corr = 1. + ip1 = iself+1 + if ip1 == n1D: + ip1 = 0 + corr = 1. + idxp1 = ids_reverse[ip1*n1D+jself] + idxm1 = ids_reverse[im1*n1D+jself] + gradP[i] = (P[idxp1]-P[idxm1])/(coords[idxp1,0]-coords[idxm1,0]+corr) + +fig, ax = pl.subplots(2, 2) + +ax[0][0].plot(coords[:,0], rho, "r.", markersize = 0.5) +ax[0][0].plot(x, rho_x, "g-") +ax[0][1].plot(coords[:,0], gradP/rho, "b.") +ax[1][0].plot(coords[:,0], agrav[:,0], "g.", markersize = 0.5) +ax[1][1].plot(coords[:,0], m, "y.") +pl.savefig("{fileName}.png".format(fileName = fileName[:-5])) diff --git a/examples/SineWavePotential_2D/run.sh b/examples/SineWavePotential_2D/run.sh new file mode 100755 index 0000000000000000000000000000000000000000..077cf1c0cc64ef7a85cfd0e67f8f490b0de4ba37 --- /dev/null +++ b/examples/SineWavePotential_2D/run.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +if [ ! -e sineWavePotential.hdf5 ] +then + echo "Generating initial conditions for the 1D SineWavePotential example..." + python makeIC.py +fi + +../swift -g -s -t 2 sineWavePotential.yml 2>&1 | tee output.log + +for f in sineWavePotential_*.hdf5 +do + python plotSolution.py $f +done diff --git a/examples/SineWavePotential_2D/sineWavePotential.yml b/examples/SineWavePotential_2D/sineWavePotential.yml new file mode 100644 index 0000000000000000000000000000000000000000..f5a565ce4db46676c1f1f340f00025decea19644 --- /dev/null +++ b/examples/SineWavePotential_2D/sineWavePotential.yml @@ -0,0 +1,39 @@ +# Define the system of units to use internally. +InternalUnitSystem: + UnitMass_in_cgs: 1. + UnitLength_in_cgs: 1. + UnitVelocity_in_cgs: 1. + UnitCurrent_in_cgs: 1. + UnitTemp_in_cgs: 1. + +# Parameters governing the time integration +TimeIntegration: + time_begin: 0. # The starting time of the simulation (in internal units). + time_end: 1. # The end time of the simulation (in internal units). + dt_min: 1e-6 # The minimal time-step size of the simulation (in internal units). + dt_max: 1.e-2 # The maximal time-step size of the simulation (in internal units). + +# Parameters governing the conserved quantities statistics +Statistics: + delta_time: 1e-2 # Time between statistics output + +# Parameters governing the snapshots +Snapshots: + basename: sineWavePotential # Common part of the name of output files + time_first: 0. # Time of the first output (in internal units) + delta_time: 0.1 # Time difference between consecutive outputs (in internal units) + +# Parameters for the hydrodynamics scheme +SPH: + resolution_eta: 1.2349 # Target smoothing length in units of the mean inter-particle separation (1.2349 == 48Ngbs with the cubic spline kernel). + delta_neighbours: 0.1 # The tolerance for the targetted number of neighbours. + CFL_condition: 0.1 # Courant-Friedrich-Levy condition for time integration. + +# Parameters related to the initial conditions +InitialConditions: + file_name: sineWavePotential.hdf5 # The file to read + +# External potential parameters +SineWavePotential: + amplitude: 10. + timestep_limit: 1. diff --git a/examples/SineWavePotential_3D/makeIC.py b/examples/SineWavePotential_3D/makeIC.py new file mode 100644 index 0000000000000000000000000000000000000000..4833ec1b055e27b63751136f0491e972fb9e492a --- /dev/null +++ b/examples/SineWavePotential_3D/makeIC.py @@ -0,0 +1,106 @@ +############################################################################### +# This file is part of SWIFT. +# Copyright (c) 2017 Bert Vandenbroucke (bert.vandenbroucke@gmail.com) +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +############################################################################## + +# Generates a distorted 1D grid with a density profile that balances out the +# external sine wave potential if run with an isothermal equation of state. + +import numpy as np +import h5py + +# constant thermal energy +# the definition below assumes the same thermal energy is defined in const.h, +# and also that the code was configured with an adiabatic index of 5./3. +uconst = 20.2615290634 +cs2 = 2.*uconst/3. +A = 10. + +fileName = "sineWavePotential.hdf5" +numPart_1D = 20 +boxSize = [1., 1.] +numPart = numPart_1D**3 + +coords = np.zeros((numPart, 3)) +v = np.zeros((numPart, 3)) +m = np.zeros(numPart) + 1. +h = np.zeros(numPart) + 2./numPart +u = np.zeros(numPart) + uconst +ids = np.arange(numPart, dtype = 'L') +rho = np.zeros(numPart) + +# first set the positions, as we try to do a reasonable volume estimate to +# set the masses +for i in range(numPart_1D): +# coords[i,0] = (i+np.random.random())/numPart + for j in range(numPart_1D): + for k in range(numPart_1D): + coords[numPart_1D**2*i+numPart_1D*j+k,0] = (i+0.5)/numPart_1D + coords[numPart_1D**2*i+numPart_1D*j+k,1] = (j+0.5)/numPart_1D + coords[numPart_1D**2*i+numPart_1D*j+k,2] = (k+0.5)/numPart_1D + +V = 1./numPart +for i in range(numPart): + # reasonable mass estimate (actually not really good, but better than assuming + # a constant volume) +# if i == 0: +# V = 0.5*(coords[1,0]-coords[-1,0]+1.) +# elif i == numPart-1: +# V = 0.5*(coords[0,0]+1.-coords[-2,0]) +# else: +# V = 0.5*(coords[i+1,0] - coords[i-1,0]) + rho[i] = 1000.*np.exp(-0.5*A/np.pi/cs2*np.cos(2.*np.pi*coords[i,0])) + m[i] = rho[i]*V + +#File +file = h5py.File(fileName, 'w') + +# Header +grp = file.create_group("/Header") +grp.attrs["BoxSize"] = boxSize +grp.attrs["NumPart_Total"] = [numPart, 0, 0, 0, 0, 0] +grp.attrs["NumPart_Total_HighWord"] = [0, 0, 0, 0, 0, 0] +grp.attrs["NumPart_ThisFile"] = [numPart, 0, 0, 0, 0, 0] +grp.attrs["Time"] = 0.0 +grp.attrs["NumFilesPerSnapshot"] = 1 +grp.attrs["MassTable"] = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +grp.attrs["Flag_Entropy_ICs"] = 0 +grp.attrs["Dimension"] = 3 + +#Runtime parameters +grp = file.create_group("/RuntimePars") +grp.attrs["PeriodicBoundariesOn"] = 1 + +#Units +grp = file.create_group("/Units") +grp.attrs["Unit length in cgs (U_L)"] = 1. +grp.attrs["Unit mass in cgs (U_M)"] = 1. +grp.attrs["Unit time in cgs (U_t)"] = 1. +grp.attrs["Unit current in cgs (U_I)"] = 1. +grp.attrs["Unit temperature in cgs (U_T)"] = 1. + +#Particle group +grp = file.create_group("/PartType0") +grp.create_dataset('Coordinates', data=coords, dtype='d') +grp.create_dataset('Velocities', data=v, dtype='f') +grp.create_dataset('Masses', data=m, dtype='f') +grp.create_dataset('SmoothingLength', data=h, dtype='f') +grp.create_dataset('InternalEnergy', data=u, dtype='f') +grp.create_dataset('ParticleIDs', data=ids, dtype='L') +grp.create_dataset('Density', data=rho, dtype='f') + +file.close() diff --git a/examples/SineWavePotential_3D/plotSolution.py b/examples/SineWavePotential_3D/plotSolution.py new file mode 100644 index 0000000000000000000000000000000000000000..13cae037b64eff4ad4fec0003bf0f5ad3ce94896 --- /dev/null +++ b/examples/SineWavePotential_3D/plotSolution.py @@ -0,0 +1,84 @@ +############################################################################### +# This file is part of SWIFT. +# Copyright (c) 2017 Bert Vandenbroucke (bert.vandenbroucke@gmail.com) +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# +############################################################################## + +# Plots some quantities for the snapshot file which is passed on as a command +# line argument (full name) + +import numpy as np +import h5py +import sys +import pylab as pl + +# these should be the same as in makeIC.py +uconst = 20.2615290634 +cs2 = 2.*uconst/3. +A = 10. + +if len(sys.argv) < 2: + print "Need to provide a filename argument!" + exit() + +fileName = sys.argv[1] + +file = h5py.File(fileName, 'r') +coords = np.array(file["/PartType0/Coordinates"]) +rho = np.array(file["/PartType0/Density"]) +u = np.array(file["/PartType0/InternalEnergy"]) +agrav = np.array(file["/PartType0/GravAcceleration"]) +m = np.array(file["/PartType0/Masses"]) +ids = np.array(file["/PartType0/ParticleIDs"]) + +# ids_reverse gives the index original particle 0 now has in the particle arrays +# and so on +# note that this will only work if the particles do not move away too much from +# there initial positions +ids_reverse = np.argsort(ids) + +x = np.linspace(0., 1., 1000) +rho_x = 1000.*np.exp(-0.5*A/np.pi/cs2*np.cos(2.*np.pi*x)) + +P = cs2*rho + +n1D = np.ceil(len(P)**(1./3.)) +gradP = np.zeros(P.shape) +for i in range(len(P)): + iself = int(ids[i]/n1D/n1D) + jself = int(int(ids[i]-n1D*iself)/n1D) + kself = int(ids[i]-n1D**2*iself-n1D*jself) + corr = 0. + im1 = iself-1 + if im1 < 0: + im1 = n1D-1 + corr = 1. + ip1 = iself+1 + if ip1 == n1D: + ip1 = 0 + corr = 1. + idxp1 = ids_reverse[ip1*n1D**2+jself*n1D+kself] + idxm1 = ids_reverse[im1*n1D**2+jself*n1D+kself] + gradP[i] = (P[idxp1]-P[idxm1])/(coords[idxp1,0]-coords[idxm1,0]+corr) + +fig, ax = pl.subplots(2, 2) + +ax[0][0].plot(coords[:,0], rho, "r.", markersize = 0.5) +ax[0][0].plot(x, rho_x, "g-") +ax[0][1].plot(coords[:,0], gradP/rho, "b.") +ax[1][0].plot(coords[:,0], agrav[:,0], "g.", markersize = 0.5) +ax[1][1].plot(coords[:,0], m, "y.") +pl.savefig("{fileName}.png".format(fileName = fileName[:-5])) diff --git a/examples/SineWavePotential_3D/run.sh b/examples/SineWavePotential_3D/run.sh new file mode 100755 index 0000000000000000000000000000000000000000..077cf1c0cc64ef7a85cfd0e67f8f490b0de4ba37 --- /dev/null +++ b/examples/SineWavePotential_3D/run.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +if [ ! -e sineWavePotential.hdf5 ] +then + echo "Generating initial conditions for the 1D SineWavePotential example..." + python makeIC.py +fi + +../swift -g -s -t 2 sineWavePotential.yml 2>&1 | tee output.log + +for f in sineWavePotential_*.hdf5 +do + python plotSolution.py $f +done diff --git a/examples/SineWavePotential_3D/sineWavePotential.yml b/examples/SineWavePotential_3D/sineWavePotential.yml new file mode 100644 index 0000000000000000000000000000000000000000..391383568a5a94bd492cb228da4a7d4d24db413f --- /dev/null +++ b/examples/SineWavePotential_3D/sineWavePotential.yml @@ -0,0 +1,39 @@ +# Define the system of units to use internally. +InternalUnitSystem: + UnitMass_in_cgs: 1. + UnitLength_in_cgs: 1. + UnitVelocity_in_cgs: 1. + UnitCurrent_in_cgs: 1. + UnitTemp_in_cgs: 1. + +# Parameters governing the time integration +TimeIntegration: + time_begin: 0. # The starting time of the simulation (in internal units). + time_end: 0.1 # The end time of the simulation (in internal units). + dt_min: 1e-6 # The minimal time-step size of the simulation (in internal units). + dt_max: 1.e-2 # The maximal time-step size of the simulation (in internal units). + +# Parameters governing the conserved quantities statistics +Statistics: + delta_time: 1e-2 # Time between statistics output + +# Parameters governing the snapshots +Snapshots: + basename: sineWavePotential # Common part of the name of output files + time_first: 0. # Time of the first output (in internal units) + delta_time: 0.01 # Time difference between consecutive outputs (in internal units) + +# Parameters for the hydrodynamics scheme +SPH: + resolution_eta: 1.2349 # Target smoothing length in units of the mean inter-particle separation (1.2349 == 48Ngbs with the cubic spline kernel). + delta_neighbours: 0.1 # The tolerance for the targetted number of neighbours. + CFL_condition: 0.1 # Courant-Friedrich-Levy condition for time integration. + +# Parameters related to the initial conditions +InitialConditions: + file_name: sineWavePotential.hdf5 # The file to read + +# External potential parameters +SineWavePotential: + amplitude: 10. + timestep_limit: 1. diff --git a/examples/SodShock_2D/plotSolution.py b/examples/SodShock_2D/plotSolution.py index 99ba7e9a6e9ae4b6d50688a1428f07e9a08b3b85..b4a203d93518d98ee87282f4ea46d045c4c3b38a 100644 --- a/examples/SodShock_2D/plotSolution.py +++ b/examples/SodShock_2D/plotSolution.py @@ -38,6 +38,7 @@ P_R = 0.1 # Pressure right state import matplotlib matplotlib.use("Agg") from pylab import * +from scipy import stats import h5py # Plot parameters @@ -86,13 +87,30 @@ rho = sim["/PartType0/Density"][:] N = 1000 # Number of points x_min = -1. x_max = 1. - x += x_min -# --------------------------------------------------------------- -# Don't touch anything after this. -# --------------------------------------------------------------- +# Bin te data +x_bin_edge = np.arange(-0.6, 0.6, 0.02) +x_bin = 0.5*(x_bin_edge[1:] + x_bin_edge[:-1]) +rho_bin,_,_ = stats.binned_statistic(x, rho, statistic='mean', bins=x_bin_edge) +v_bin,_,_ = stats.binned_statistic(x, v, statistic='mean', bins=x_bin_edge) +P_bin,_,_ = stats.binned_statistic(x, P, statistic='mean', bins=x_bin_edge) +S_bin,_,_ = stats.binned_statistic(x, S, statistic='mean', bins=x_bin_edge) +u_bin,_,_ = stats.binned_statistic(x, u, statistic='mean', bins=x_bin_edge) +rho2_bin,_,_ = stats.binned_statistic(x, rho**2, statistic='mean', bins=x_bin_edge) +v2_bin,_,_ = stats.binned_statistic(x, v**2, statistic='mean', bins=x_bin_edge) +P2_bin,_,_ = stats.binned_statistic(x, P**2, statistic='mean', bins=x_bin_edge) +S2_bin,_,_ = stats.binned_statistic(x, S**2, statistic='mean', bins=x_bin_edge) +u2_bin,_,_ = stats.binned_statistic(x, u**2, statistic='mean', bins=x_bin_edge) +rho_sigma_bin = np.sqrt(rho2_bin - rho_bin**2) +v_sigma_bin = np.sqrt(v2_bin - v_bin**2) +P_sigma_bin = np.sqrt(P2_bin - P_bin**2) +S_sigma_bin = np.sqrt(S2_bin - S_bin**2) +u_sigma_bin = np.sqrt(u2_bin - u_bin**2) + + +# Analytic solution c_L = sqrt(gas_gamma * P_L / rho_L) # Speed of the rarefaction wave c_R = sqrt(gas_gamma * P_R / rho_R) # Speed of the shock front @@ -225,8 +243,9 @@ figure() # Velocity profile -------------------------------- subplot(231) -plot(x, v, '.', color='r', ms=0.5) +plot(x, v, '.', color='r', ms=0.2) plot(x_s, v_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(x_bin, v_bin, yerr=v_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) xlabel("${\\rm{Position}}~x$", labelpad=0) ylabel("${\\rm{Velocity}}~v_x$", labelpad=0) xlim(-0.5, 0.5) @@ -234,8 +253,9 @@ ylim(-0.1, 0.95) # Density profile -------------------------------- subplot(232) -plot(x, rho, '.', color='r', ms=0.5) +plot(x, rho, '.', color='r', ms=0.2) plot(x_s, rho_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(x_bin, rho_bin, yerr=rho_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) xlabel("${\\rm{Position}}~x$", labelpad=0) ylabel("${\\rm{Density}}~\\rho$", labelpad=0) xlim(-0.5, 0.5) @@ -243,8 +263,9 @@ ylim(0.05, 1.1) # Pressure profile -------------------------------- subplot(233) -plot(x, P, '.', color='r', ms=0.5) +plot(x, P, '.', color='r', ms=0.2) plot(x_s, P_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(x_bin, P_bin, yerr=P_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) xlabel("${\\rm{Position}}~x$", labelpad=0) ylabel("${\\rm{Pressure}}~P$", labelpad=0) xlim(-0.5, 0.5) @@ -252,8 +273,9 @@ ylim(0.01, 1.1) # Internal energy profile ------------------------- subplot(234) -plot(x, u, '.', color='r', ms=0.5) +plot(x, u, '.', color='r', ms=0.2) plot(x_s, u_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(x_bin, u_bin, yerr=u_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) xlabel("${\\rm{Position}}~x$", labelpad=0) ylabel("${\\rm{Internal~Energy}}~u$", labelpad=0) xlim(-0.5, 0.5) @@ -261,8 +283,9 @@ ylim(0.8, 2.2) # Entropy profile --------------------------------- subplot(235) -plot(x, S, '.', color='r', ms=0.5) +plot(x, S, '.', color='r', ms=0.2) plot(x_s, s_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(x_bin, S_bin, yerr=S_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) xlabel("${\\rm{Position}}~x$", labelpad=0) ylabel("${\\rm{Entropy}}~S$", labelpad=0) xlim(-0.5, 0.5) diff --git a/examples/SodShock_3D/plotSolution.py b/examples/SodShock_3D/plotSolution.py index 23a16e6aed73a7281cf78a215940ccdcff722a79..3d9616af55a204db4be9df2e42b355e266944153 100644 --- a/examples/SodShock_3D/plotSolution.py +++ b/examples/SodShock_3D/plotSolution.py @@ -38,6 +38,7 @@ P_R = 0.1 # Pressure right state import matplotlib matplotlib.use("Agg") from pylab import * +from scipy import stats import h5py # Plot parameters @@ -83,16 +84,32 @@ S = sim["/PartType0/Entropy"][:] P = sim["/PartType0/Pressure"][:] rho = sim["/PartType0/Density"][:] -N = 1000 # Number of points x_min = -1. x_max = 1. - x += x_min - -# --------------------------------------------------------------- -# Don't touch anything after this. -# --------------------------------------------------------------- - +N = 1000 + +# Bin te data +x_bin_edge = np.arange(-0.6, 0.6, 0.02) +x_bin = 0.5*(x_bin_edge[1:] + x_bin_edge[:-1]) +rho_bin,_,_ = stats.binned_statistic(x, rho, statistic='mean', bins=x_bin_edge) +v_bin,_,_ = stats.binned_statistic(x, v, statistic='mean', bins=x_bin_edge) +P_bin,_,_ = stats.binned_statistic(x, P, statistic='mean', bins=x_bin_edge) +S_bin,_,_ = stats.binned_statistic(x, S, statistic='mean', bins=x_bin_edge) +u_bin,_,_ = stats.binned_statistic(x, u, statistic='mean', bins=x_bin_edge) +rho2_bin,_,_ = stats.binned_statistic(x, rho**2, statistic='mean', bins=x_bin_edge) +v2_bin,_,_ = stats.binned_statistic(x, v**2, statistic='mean', bins=x_bin_edge) +P2_bin,_,_ = stats.binned_statistic(x, P**2, statistic='mean', bins=x_bin_edge) +S2_bin,_,_ = stats.binned_statistic(x, S**2, statistic='mean', bins=x_bin_edge) +u2_bin,_,_ = stats.binned_statistic(x, u**2, statistic='mean', bins=x_bin_edge) +rho_sigma_bin = np.sqrt(rho2_bin - rho_bin**2) +v_sigma_bin = np.sqrt(v2_bin - v_bin**2) +P_sigma_bin = np.sqrt(P2_bin - P_bin**2) +S_sigma_bin = np.sqrt(S2_bin - S_bin**2) +u_sigma_bin = np.sqrt(u2_bin - u_bin**2) + + +# Analytic solution c_L = sqrt(gas_gamma * P_L / rho_L) # Speed of the rarefaction wave c_R = sqrt(gas_gamma * P_R / rho_R) # Speed of the shock front @@ -225,8 +242,9 @@ figure() # Velocity profile -------------------------------- subplot(231) -plot(x, v, '.', color='r', ms=0.5) +plot(x, v, '.', color='r', ms=0.5, alpha=0.2) plot(x_s, v_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(x_bin, v_bin, yerr=v_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) xlabel("${\\rm{Position}}~x$", labelpad=0) ylabel("${\\rm{Velocity}}~v_x$", labelpad=0) xlim(-0.5, 0.5) @@ -234,8 +252,9 @@ ylim(-0.1, 0.95) # Density profile -------------------------------- subplot(232) -plot(x, rho, '.', color='r', ms=0.5) +plot(x, rho, '.', color='r', ms=0.5, alpha=0.2) plot(x_s, rho_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(x_bin, rho_bin, yerr=rho_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) xlabel("${\\rm{Position}}~x$", labelpad=0) ylabel("${\\rm{Density}}~\\rho$", labelpad=0) xlim(-0.5, 0.5) @@ -243,8 +262,9 @@ ylim(0.05, 1.1) # Pressure profile -------------------------------- subplot(233) -plot(x, P, '.', color='r', ms=0.5) +plot(x, P, '.', color='r', ms=0.5, alpha=0.2) plot(x_s, P_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(x_bin, P_bin, yerr=P_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) xlabel("${\\rm{Position}}~x$", labelpad=0) ylabel("${\\rm{Pressure}}~P$", labelpad=0) xlim(-0.5, 0.5) @@ -252,8 +272,9 @@ ylim(0.01, 1.1) # Internal energy profile ------------------------- subplot(234) -plot(x, u, '.', color='r', ms=0.5) +plot(x, u, '.', color='r', ms=0.5, alpha=0.2) plot(x_s, u_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(x_bin, u_bin, yerr=u_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) xlabel("${\\rm{Position}}~x$", labelpad=0) ylabel("${\\rm{Internal~Energy}}~u$", labelpad=0) xlim(-0.5, 0.5) @@ -261,8 +282,9 @@ ylim(0.8, 2.2) # Entropy profile --------------------------------- subplot(235) -plot(x, S, '.', color='r', ms=0.5) +plot(x, S, '.', color='r', ms=0.5, alpha=0.2) plot(x_s, s_s, '--', color='k', alpha=0.8, lw=1.2) +errorbar(x_bin, S_bin, yerr=S_sigma_bin, fmt='.', ms=8.0, color='b', lw=1.2) xlabel("${\\rm{Position}}~x$", labelpad=0) ylabel("${\\rm{Entropy}}~S$", labelpad=0) xlim(-0.5, 0.5) diff --git a/examples/main.c b/examples/main.c index 4d511ce8bc4ad35afee4f86655cbabeda3eebc0a..8d4289c872ce34ba626cacffca4097c0093fe0c3 100644 --- a/examples/main.c +++ b/examples/main.c @@ -54,12 +54,12 @@ struct profiler prof; void print_help_message() { printf("\nUsage: swift [OPTION]... PARAMFILE\n"); - printf(" swift_mpi [OPTION]... PARAMFILE\n"); + printf(" swift_mpi [OPTION]... PARAMFILE\n\n"); printf("Valid options are:\n"); - printf(" %2s %8s %s\n", "-a", "", "Pin runners using processor affinity"); - printf(" %2s %8s %s\n", "-c", "", "Run with cosmological time integration"); - printf(" %2s %8s %s\n", "-C", "", "Run with cooling"); + printf(" %2s %8s %s\n", "-a", "", "Pin runners using processor affinity."); + printf(" %2s %8s %s\n", "-c", "", "Run with cosmological time integration."); + printf(" %2s %8s %s\n", "-C", "", "Run with cooling."); printf( " %2s %8s %s\n", "-d", "", "Dry run. Read the parameter file, allocate memory but does not read "); @@ -70,28 +70,32 @@ void print_help_message() { "Allows user to check validy of parameter and IC files as well as " "memory limits."); printf(" %2s %8s %s\n", "-D", "", - "Always drift all particles even the ones far from active particles."); + "Always drift all particles even the ones far from active particles. " + "This emulates"); + printf(" %2s %8s %s\n", "", "", + "Gadget-[23] and GIZMO's default behaviours."); printf(" %2s %8s %s\n", "-e", "", - "Enable floating-point exceptions (debugging mode)"); + "Enable floating-point exceptions (debugging mode)."); printf(" %2s %8s %s\n", "-f", "{int}", - "Overwrite the CPU frequency (Hz) to be used for time measurements"); + "Overwrite the CPU frequency (Hz) to be used for time measurements."); printf(" %2s %8s %s\n", "-g", "", - "Run with an external gravitational potential"); - printf(" %2s %8s %s\n", "-F", "", "Run with feedback "); - printf(" %2s %8s %s\n", "-G", "", "Run with self-gravity"); + "Run with an external gravitational potential."); + printf(" %2s %8s %s\n", "-F", "", "Run with feedback."); + printf(" %2s %8s %s\n", "-G", "", "Run with self-gravity."); printf(" %2s %8s %s\n", "-n", "{int}", "Execute a fixed number of time steps. When unset use the time_end " "parameter to stop."); - printf(" %2s %8s %s\n", "-s", "", "Run with SPH"); + printf(" %2s %8s %s\n", "-s", "", "Run with hydrodynamics."); + printf(" %2s %8s %s\n", "-S", "", "Run with stars."); printf(" %2s %8s %s\n", "-t", "{int}", "The number of threads to use on each MPI rank. Defaults to 1 if not " "specified."); - printf(" %2s %8s %s\n", "-v", "[12]", "Increase the level of verbosity"); + printf(" %2s %8s %s\n", "-v", "[12]", "Increase the level of verbosity."); printf(" %2s %8s %s\n", "", "", "1: MPI-rank 0 writes "); printf(" %2s %8s %s\n", "", "", "2: All MPI-ranks write"); printf(" %2s %8s %s\n", "-y", "{int}", - "Time-step frequency at which task graphs are dumped"); - printf(" %2s %8s %s\n", "-h", "", "Print this help message and exit"); + "Time-step frequency at which task graphs are dumped."); + printf(" %2s %8s %s\n", "-h", "", "Print this help message and exit."); printf( "\nSee the file parameter_example.yml for an example of " "parameter file.\n"); @@ -156,6 +160,7 @@ int main(int argc, char *argv[]) { int with_cooling = 0; int with_self_gravity = 0; int with_hydro = 0; + int with_stars = 0; int with_fp_exceptions = 0; int with_drift_all = 0; int verbose = 0; @@ -165,7 +170,7 @@ int main(int argc, char *argv[]) { /* Parse the parameters */ int c; - while ((c = getopt(argc, argv, "acCdDef:FgGhn:st:v:y:")) != -1) switch (c) { + while ((c = getopt(argc, argv, "acCdDef:FgGhn:sSt:v:y:")) != -1) switch (c) { case 'a': with_aff = 1; break; @@ -213,6 +218,9 @@ int main(int argc, char *argv[]) { case 's': with_hydro = 1; break; + case 'S': + with_stars = 1; + break; case 't': if (sscanf(optarg, "%d", &nr_threads) != 1) { if (myrank == 0) @@ -269,6 +277,9 @@ int main(int argc, char *argv[]) { /* Genesis 1.1: And then, there was time ! */ clocks_set_cpufreq(cpufreq); + /* How vocal are we ? */ + const int talking = (verbose == 1 && myrank == 0) || (verbose == 2); + if (myrank == 0 && dry_run) message( "Executing a dry run. No i/o or time integration will be performed."); @@ -281,7 +292,7 @@ int main(int argc, char *argv[]) { /* Report host name(s). */ #ifdef WITH_MPI - if (myrank == 0 || verbose > 1) { + if (talking) { message("Rank %d running on: %s", myrank, hostname()); } #else @@ -294,6 +305,15 @@ int main(int argc, char *argv[]) { message("WARNING: Debugging checks activated. Code will be slower !"); #endif +/* Do we have gravity accuracy checks ? */ +#ifdef SWIFT_GRAVITY_FORCE_CHECKS + if (myrank == 0) + message( + "WARNING: Checking 1/%d of all gpart for gravity accuracy. Code will " + "be slower !", + SWIFT_GRAVITY_FORCE_CHECKS); +#endif + /* Do we choke on FP-exceptions ? */ if (with_fp_exceptions) { feenableexcept(FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW); @@ -303,16 +323,16 @@ int main(int argc, char *argv[]) { /* How large are the parts? */ if (myrank == 0) { - message("sizeof(struct part) is %4zi bytes.", sizeof(struct part)); - message("sizeof(struct xpart) is %4zi bytes.", sizeof(struct xpart)); - message("sizeof(struct gpart) is %4zi bytes.", sizeof(struct gpart)); - message("sizeof(struct task) is %4zi bytes.", sizeof(struct task)); - message("sizeof(struct cell) is %4zi bytes.", sizeof(struct cell)); + message("sizeof(part) is %4zi bytes.", sizeof(struct part)); + message("sizeof(xpart) is %4zi bytes.", sizeof(struct xpart)); + message("sizeof(spart) is %4zi bytes.", sizeof(struct spart)); + message("sizeof(gpart) is %4zi bytes.", sizeof(struct gpart)); + message("sizeof(multipole) is %4zi bytes.", sizeof(struct multipole)); + message("sizeof(acc_tensor) is %4zi bytes.", sizeof(struct acc_tensor)); + message("sizeof(task) is %4zi bytes.", sizeof(struct task)); + message("sizeof(cell) is %4zi bytes.", sizeof(struct cell)); } - /* How vocal are we ? */ - const int talking = (verbose == 1 && myrank == 0) || (verbose == 2); - /* Read the parameter file */ struct swift_params *params = malloc(sizeof(struct swift_params)); if (params == NULL) error("Error allocating memory for the parameter file."); @@ -327,10 +347,10 @@ int main(int argc, char *argv[]) { MPI_Bcast(params, sizeof(struct swift_params), MPI_BYTE, 0, MPI_COMM_WORLD); #endif -/* Prepare the domain decomposition scheme */ + /* Prepare the domain decomposition scheme */ + enum repartition_type reparttype = REPART_NONE; #ifdef WITH_MPI struct partition initial_partition; - enum repartition_type reparttype; partition_init(&initial_partition, &reparttype, params, nr_nodes); /* Let's report what we did */ @@ -345,7 +365,7 @@ int main(int argc, char *argv[]) { #endif /* Initialize unit system and constants */ - struct UnitSystem us; + struct unit_system us; struct phys_const prog_const; units_init(&us, params, "InternalUnitSystem"); phys_const_init(&us, &prog_const); @@ -362,32 +382,44 @@ int main(int argc, char *argv[]) { struct hydro_props hydro_properties; if (with_hydro) hydro_props_init(&hydro_properties, params); + /* Initialise the gravity properties */ + struct gravity_props gravity_properties; + if (with_self_gravity) gravity_props_init(&gravity_properties, params); + /* Read particles and space information from (GADGET) ICs */ char ICfileName[200] = ""; parser_get_param_string(params, "InitialConditions:file_name", ICfileName); + const int replicate = + parser_get_opt_param_int(params, "InitialConditions:replicate", 1); if (myrank == 0) message("Reading ICs from file '%s'", ICfileName); fflush(stdout); + /* Get ready to read particles of all kinds */ struct part *parts = NULL; struct gpart *gparts = NULL; - size_t Ngas = 0, Ngpart = 0; + struct spart *sparts = NULL; + size_t Ngas = 0, Ngpart = 0, Nspart = 0; double dim[3] = {0., 0., 0.}; int periodic = 0; int flag_entropy_ICs = 0; if (myrank == 0) clocks_gettime(&tic); #if defined(WITH_MPI) #if defined(HAVE_PARALLEL_HDF5) - read_ic_parallel(ICfileName, &us, dim, &parts, &gparts, &Ngas, &Ngpart, - &periodic, &flag_entropy_ICs, myrank, nr_nodes, - MPI_COMM_WORLD, MPI_INFO_NULL, dry_run); + read_ic_parallel(ICfileName, &us, dim, &parts, &gparts, &sparts, &Ngas, + &Ngpart, &Nspart, &periodic, &flag_entropy_ICs, with_hydro, + (with_external_gravity || with_self_gravity), with_stars, + myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL, dry_run); #else - read_ic_serial(ICfileName, &us, dim, &parts, &gparts, &Ngas, &Ngpart, - &periodic, &flag_entropy_ICs, myrank, nr_nodes, MPI_COMM_WORLD, - MPI_INFO_NULL, dry_run); + read_ic_serial(ICfileName, &us, dim, &parts, &gparts, &sparts, &Ngas, &Ngpart, + &Nspart, &periodic, &flag_entropy_ICs, with_hydro, + (with_external_gravity || with_self_gravity), with_stars, + myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL, dry_run); #endif #else - read_ic_single(ICfileName, &us, dim, &parts, &gparts, &Ngas, &Ngpart, - &periodic, &flag_entropy_ICs, dry_run); + read_ic_single(ICfileName, &us, dim, &parts, &gparts, &sparts, &Ngas, &Ngpart, + &Nspart, &periodic, &flag_entropy_ICs, with_hydro, + (with_external_gravity || with_self_gravity), with_stars, + dry_run); #endif if (myrank == 0) { clocks_gettime(&toc); @@ -396,40 +428,40 @@ int main(int argc, char *argv[]) { fflush(stdout); } - /* Discard gparts if we don't have gravity - * (Better implementation of i/o will come)*/ - if (!with_external_gravity && !with_self_gravity) { - free(gparts); - gparts = NULL; - for (size_t k = 0; k < Ngas; ++k) parts[k].gpart = NULL; - Ngpart = 0; +#ifdef SWIFT_DEBUG_CHECKS + /* Check once and for all that we don't have unwanted links */ + if (!with_stars) { + for (size_t k = 0; k < Ngpart; ++k) + if (gparts[k].type == swift_type_star) error("Linking problem"); } if (!with_hydro) { - free(parts); - parts = NULL; for (size_t k = 0; k < Ngpart; ++k) - if (gparts[k].id_or_neg_offset < 0) error("Linking problem"); - Ngas = 0; + if (gparts[k].type == swift_type_gas) error("Linking problem"); } +#endif /* Get the total number of particles across all nodes. */ - long long N_total[2] = {0, 0}; + long long N_total[3] = {0, 0, 0}; #if defined(WITH_MPI) - long long N_long[2] = {Ngas, Ngpart}; - MPI_Reduce(&N_long, &N_total, 2, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD); + long long N_long[3] = {Ngas, Ngpart, Nspart}; + MPI_Reduce(&N_long, &N_total, 3, MPI_LONG_LONG_INT, MPI_SUM, 0, + MPI_COMM_WORLD); #else N_total[0] = Ngas; N_total[1] = Ngpart; + N_total[2] = Nspart; #endif if (myrank == 0) - message("Read %lld gas particles and %lld gparts from the ICs.", N_total[0], - N_total[1]); + message( + "Read %lld gas particles, %lld star particles and %lld gparts from the " + "ICs.", + N_total[0], N_total[2], N_total[1]); /* Initialize the space with these data. */ if (myrank == 0) clocks_gettime(&tic); struct space s; - space_init(&s, params, dim, parts, gparts, Ngas, Ngpart, periodic, - with_self_gravity, talking, dry_run); + space_init(&s, params, dim, parts, gparts, sparts, Ngas, Ngpart, Nspart, + periodic, replicate, with_self_gravity, talking, dry_run); if (myrank == 0) { clocks_gettime(&toc); message("space_init took %.3f %s.", clocks_diff(&tic, &toc), @@ -446,6 +478,7 @@ int main(int argc, char *argv[]) { s.cdim[1], s.cdim[2]); message("%zi parts in %i cells.", s.nr_parts, s.tot_cells); message("%zi gparts in %i cells.", s.nr_gparts, s.tot_cells); + message("%zi sparts in %i cells.", s.nr_sparts, s.tot_cells); message("maximum depth is %d.", s.maxdepth); fflush(stdout); } @@ -489,13 +522,15 @@ int main(int argc, char *argv[]) { if (with_cosmology) engine_policies |= engine_policy_cosmology; if (with_cooling) engine_policies |= engine_policy_cooling; if (with_sourceterms) engine_policies |= engine_policy_sourceterms; + if (with_stars) engine_policies |= engine_policy_stars; /* Initialize the engine with the space and policies. */ if (myrank == 0) clocks_gettime(&tic); struct engine e; engine_init(&e, &s, params, nr_nodes, myrank, nr_threads, with_aff, - engine_policies, talking, &us, &prog_const, &hydro_properties, - &potential, &cooling_func, &sourceterms); + engine_policies, talking, reparttype, &us, &prog_const, + &hydro_properties, &gravity_properties, &potential, &cooling_func, + &sourceterms); if (myrank == 0) { clocks_gettime(&toc); message("engine_init took %.3f %s.", clocks_diff(&tic, &toc), @@ -508,13 +543,30 @@ int main(int argc, char *argv[]) { for (k = 0; k < runner_hist_N; k++) runner_hist_bins[k] = 0; #endif +#if defined(WITH_MPI) + N_long[0] = s.nr_parts; + N_long[1] = s.nr_gparts; + N_long[2] = s.nr_sparts; + MPI_Reduce(&N_long, &N_total, 3, MPI_LONG_LONG_INT, MPI_SUM, 0, + MPI_COMM_WORLD); +#else + N_total[0] = s.nr_parts; + N_total[1] = s.nr_gparts; + N_total[2] = s.nr_sparts; +#endif + /* Get some info to the user. */ if (myrank == 0) { + long long N_DM = N_total[1] - N_total[2] - N_total[0]; + message( + "Running on %lld gas particles, %lld star particles and %lld DM " + "particles (%lld gravity particles)", + N_total[0], N_total[2], N_total[1] > 0 ? N_DM : 0, N_total[1]); message( - "Running on %lld gas particles and %lld DM particles from t=%.3e until " - "t=%.3e with %d threads and %d queues (dt_min=%.3e, dt_max=%.3e)...", - N_total[0], N_total[1], e.timeBegin, e.timeEnd, e.nr_threads, - e.sched.nr_queues, e.dt_min, e.dt_max); + "from t=%.3e until t=%.3e with %d threads and %d queues (dt_min=%.3e, " + "dt_max=%.3e)...", + e.timeBegin, e.timeEnd, e.nr_threads, e.sched.nr_queues, e.dt_min, + e.dt_max); fflush(stdout); } @@ -545,16 +597,12 @@ int main(int argc, char *argv[]) { /* Legend */ if (myrank == 0) - printf("# %6s %14s %14s %10s %10s %16s [%s]\n", "Step", "Time", "Time-step", - "Updates", "g-Updates", "Wall-clock time", clocks_getunit()); + printf("# %6s %14s %14s %10s %10s %10s %16s [%s]\n", "Step", "Time", + "Time-step", "Updates", "g-Updates", "s-Updates", "Wall-clock time", + clocks_getunit()); /* Main simulation loop */ - for (int j = 0; !engine_is_done(&e) && e.step != nsteps; j++) { - -/* Repartition the space amongst the nodes? */ -#ifdef WITH_MPI - if (j % 100 == 2) e.forcerepart = reparttype; -#endif + for (int j = 0; !engine_is_done(&e) && e.step - 1 != nsteps; j++) { /* Reset timers */ timers_reset(timers_mask_all); @@ -657,6 +705,7 @@ int main(int argc, char *argv[]) { #endif /* Write final output. */ + engine_drift_all(&e); engine_dump_snapshot(&e); #ifdef WITH_MPI diff --git a/examples/parameter_example.yml b/examples/parameter_example.yml index 9681e880e9f4b44d4f77195eecf5b8642266ce52..7c69d19bc215472c90bb1b91f23f46702afc64f2 100644 --- a/examples/parameter_example.yml +++ b/examples/parameter_example.yml @@ -47,7 +47,15 @@ SPH: CFL_condition: 0.1 # Courant-Friedrich-Levy condition for time integration. max_ghost_iterations: 30 # (Optional) Maximal number of iterations allowed to converge towards the smoothing length. max_volume_change: 2. # (Optional) Maximal allowed change of kernel volume over one time-step + h_max: 10. # (Optional) Maximal allowed smoothing length in internal units. Defaults to FLT_MAX if unspecified. +# Parameters for the self-gravity scheme +Gravity: + eta: 0.025 # Constant dimensionless multiplier for time integration. + epsilon: 0.1 # Softening length (in internal units). + a_smooth: 1.25 # (Optional) Smoothing scale in top-level cell sizes to smooth the long-range forces over (this is the default value). + r_cut: 4.5 # (Optional) Cut-off in number of top-level cells beyond which no FMM forces are computed (this is the default value). + # Parameters related to the initial conditions InitialConditions: file_name: SedovBlast/sedov.hdf5 # The file to read @@ -55,6 +63,7 @@ InitialConditions: shift_x: 0. # (Optional) A shift to apply to all particles read from the ICs (in internal units). shift_y: 0. shift_z: 0. + replicate: 2 # (Optional) Replicate all particles along each axis a given number of times. Default 1. # Parameters governing domain decomposition DomainDecomposition: diff --git a/src/Makefile.am b/src/Makefile.am index 5434241c0eb203f069901dd9be99640b91dd8e90..29c110218416cc4bece0d766516eeb0a97fe4810 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -45,7 +45,7 @@ include_HEADERS = space.h runner.h queue.h task.h lock.h cell.h part.h const.h \ physical_constants.h physical_constants_cgs.h potential.h version.h \ hydro_properties.h riemann.h threadpool.h cooling.h cooling_struct.h sourceterms.h \ sourceterms_struct.h statistics.h memswap.h cache.h runner_doiact_vec.h profiler.h \ - dump.h logger.h active.h timeline.h + dump.h logger.h active.h timeline.h xmf.h gravity_properties.h gravity_derivatives.h # Common source files AM_SOURCES = space.c runner.c queue.c task.c cell.c engine.c \ @@ -54,13 +54,14 @@ AM_SOURCES = space.c runner.c queue.c task.c cell.c engine.c \ kernel_hydro.c tools.c part.c partition.c clocks.c parser.c \ physical_constants.c potential.c hydro_properties.c \ runner_doiact_fft.c threadpool.c cooling.c sourceterms.c \ - statistics.c runner_doiact_vec.c profiler.c dump.c logger.c + statistics.c runner_doiact_vec.c profiler.c dump.c logger.c \ + part_type.c xmf.c gravity_properties.c gravity.c # Include files for distribution, not installation. nobase_noinst_HEADERS = align.h approx_math.h atomic.h cycle.h error.h inline.h kernel_hydro.h kernel_gravity.h \ kernel_long_gravity.h vector.h cache.h runner_doiact.h runner_doiact_vec.h runner_doiact_grav.h runner_doiact_fft.h \ runner_doiact_nosort.h units.h intrinsics.h minmax.h kick.h timestep.h drift.h adiabatic_index.h io_properties.h \ - dimension.h equation_of_state.h \ + dimension.h equation_of_state.h part_type.h \ gravity.h gravity_io.h \ gravity/Default/gravity.h gravity/Default/gravity_iact.h gravity/Default/gravity_io.h \ gravity/Default/gravity_debug.h gravity/Default/gravity_part.h \ @@ -78,8 +79,12 @@ nobase_noinst_HEADERS = align.h approx_math.h atomic.h cycle.h error.h inline.h hydro/Gizmo/hydro_debug.h hydro/Gizmo/hydro_part.h \ riemann/riemann_hllc.h riemann/riemann_trrs.h \ riemann/riemann_exact.h riemann/riemann_vacuum.h \ + stars.h stars_io.h \ + stars/Default/star.h stars/Default/star_iact.h stars/Default/star_io.h \ + stars/Default/star_debug.h stars/Default/star_part.h \ potential/none/potential.h potential/point_mass/potential.h \ potential/isothermal/potential.h potential/disc_patch/potential.h \ + potential/sine_wave/potential.h \ cooling/none/cooling.h cooling/none/cooling_struct.h \ cooling/const_du/cooling.h cooling/const_du/cooling_struct.h \ cooling/const_lambda/cooling.h cooling/const_lambda/cooling_struct.h \ @@ -106,15 +111,18 @@ version_string.h: version_string.h.in $(AM_SOURCES) $(include_HEADERS) $(noinst_ if test "X$(GIT_CMD)" != "X"; then \ GIT_REVISION=`$(GIT_CMD) describe --abbrev=8 --always --tags --dirty`; \ GIT_BRANCH=`$(GIT_CMD) branch | sed -n 's/^\* \(.*\)/\1/p'`; \ + GIT_DATE=`$(GIT_CMD) log -1 --format=%ci`; \ sed -e "s,@PACKAGE_VERSION\@,$(PACKAGE_VERSION)," \ -e "s,@GIT_REVISION\@,$${GIT_REVISION}," \ -e "s|@GIT_BRANCH\@|$${GIT_BRANCH}|" \ + -e "s|@GIT_DATE\@|$${GIT_DATE}|" \ -e "s|@SWIFT_CFLAGS\@|$(CFLAGS)|" $< > version_string.h; \ else \ if test ! -f version_string.h; then \ sed -e "s,@PACKAGE_VERSION\@,$(PACKAGE_VERSION)," \ -e "s,@GIT_REVISION\@,unknown," \ -e "s,@GIT_BRANCH\@,unknown," \ + -e "s,@GIT_DATE\@,unknown," \ -e "s|@SWIFT_CFLAGS\@|$(CFLAGS)|" $< > version_string.h; \ fi; \ fi diff --git a/src/active.h b/src/active.h index 8dfd063fda9189ff715107c599ef4851cc5a0387..02e504f762735994e6c57f7e155071fede016713 100644 --- a/src/active.h +++ b/src/active.h @@ -50,8 +50,10 @@ __attribute__((always_inline)) INLINE static int cell_is_drifted( return (c->ti_old == e->ti_current); } +/* Are cells / particles active for regular tasks ? */ + /** - * @brief Does a cell contain any active particle ? + * @brief Does a cell contain any particle finishing their time-step now ? * * @param c The #cell. * @param e The #engine containing information about the current time. @@ -73,7 +75,7 @@ __attribute__((always_inline)) INLINE static int cell_is_active( } /** - * @brief Are *all* particles in a cell active ? + * @brief Are *all* particles in a cell finishing their time-step now ? * * @param c The #cell. * @param e The #engine containing information about the current time. @@ -94,7 +96,7 @@ __attribute__((always_inline)) INLINE static int cell_is_all_active( } /** - * @brief Is this particle active ? + * @brief Is this particle finishing its time-step now ? * * @param p The #part. * @param e The #engine containing information about the current time. @@ -103,10 +105,12 @@ __attribute__((always_inline)) INLINE static int cell_is_all_active( __attribute__((always_inline)) INLINE static int part_is_active( const struct part *p, const struct engine *e) { - const integertime_t ti_current = e->ti_current; - const integertime_t ti_end = get_integer_time_end(ti_current, p->time_bin); + const timebin_t max_active_bin = e->max_active_bin; + const timebin_t part_bin = p->time_bin; #ifdef SWIFT_DEBUG_CHECKS + const integertime_t ti_current = e->ti_current; + const integertime_t ti_end = get_integer_time_end(ti_current, p->time_bin); if (ti_end < ti_current) error( "particle in an impossible time-zone! p->ti_end=%lld " @@ -114,11 +118,11 @@ __attribute__((always_inline)) INLINE static int part_is_active( ti_end, ti_current); #endif - return (ti_end == ti_current); + return (part_bin <= max_active_bin); } /** - * @brief Is this g-particle active ? + * @brief Is this g-particle finishing its time-step now ? * * @param gp The #gpart. * @param e The #engine containing information about the current time. @@ -127,10 +131,13 @@ __attribute__((always_inline)) INLINE static int part_is_active( __attribute__((always_inline)) INLINE static int gpart_is_active( const struct gpart *gp, const struct engine *e) { + const timebin_t max_active_bin = e->max_active_bin; + const timebin_t gpart_bin = gp->time_bin; + +#ifdef SWIFT_DEBUG_CHECKS const integertime_t ti_current = e->ti_current; const integertime_t ti_end = get_integer_time_end(ti_current, gp->time_bin); -#ifdef SWIFT_DEBUG_CHECKS if (ti_end < ti_current) error( "g-particle in an impossible time-zone! gp->ti_end=%lld " @@ -138,7 +145,141 @@ __attribute__((always_inline)) INLINE static int gpart_is_active( ti_end, ti_current); #endif - return (ti_end == ti_current); + return (gpart_bin <= max_active_bin); +} + +/** + * @brief Is this s-particle finishing its time-step now ? + * + * @param sp The #spart. + * @param e The #engine containing information about the current time. + * @return 1 if the #spart is active, 0 otherwise. + */ +__attribute__((always_inline)) INLINE static int spart_is_active( + const struct spart *sp, const struct engine *e) { + + const timebin_t max_active_bin = e->max_active_bin; + const timebin_t spart_bin = sp->time_bin; + +#ifdef SWIFT_DEBUG_CHECKS + const integertime_t ti_current = e->ti_current; + const integertime_t ti_end = get_integer_time_end(ti_current, sp->time_bin); + + if (ti_end < ti_current) + error( + "s-particle in an impossible time-zone! sp->ti_end=%lld " + "e->ti_current=%lld", + ti_end, ti_current); +#endif + + return (spart_bin <= max_active_bin); +} + +/* Are cells / particles active for kick1 tasks ? */ + +/** + * @brief Does a cell contain any particle starting their time-step now ? + * + * @param c The #cell. + * @param e The #engine containing information about the current time. + * @return 1 if the #cell contains at least an active particle, 0 otherwise. + */ +__attribute__((always_inline)) INLINE static int cell_is_starting( + const struct cell *c, const struct engine *e) { + +#ifdef SWIFT_DEBUG_CHECKS + if (c->ti_beg_max > e->ti_current) + error( + "cell in an impossible time-zone! c->ti_beg_max=%lld (t=%e) and " + "e->ti_current=%lld (t=%e)", + c->ti_beg_max, c->ti_beg_max * e->timeBase, e->ti_current, + e->ti_current * e->timeBase); +#endif + + return (c->ti_beg_max == e->ti_current); +} + +/** + * @brief Is this particle starting its time-step now ? + * + * @param p The #part. + * @param e The #engine containing information about the current time. + * @return 1 if the #part is active, 0 otherwise. + */ +__attribute__((always_inline)) INLINE static int part_is_starting( + const struct part *p, const struct engine *e) { + + const timebin_t max_active_bin = e->max_active_bin; + const timebin_t part_bin = p->time_bin; + +#ifdef SWIFT_DEBUG_CHECKS + const integertime_t ti_current = e->ti_current; + const integertime_t ti_beg = + get_integer_time_begin(ti_current + 1, p->time_bin); + + if (ti_beg > ti_current) + error( + "particle in an impossible time-zone! p->ti_beg=%lld " + "e->ti_current=%lld", + ti_beg, ti_current); +#endif + + return (part_bin <= max_active_bin); +} + +/** + * @brief Is this g-particle starting its time-step now ? + * + * @param gp The #gpart. + * @param e The #engine containing information about the current time. + * @return 1 if the #gpart is active, 0 otherwise. + */ +__attribute__((always_inline)) INLINE static int gpart_is_starting( + const struct gpart *gp, const struct engine *e) { + + const timebin_t max_active_bin = e->max_active_bin; + const timebin_t gpart_bin = gp->time_bin; + +#ifdef SWIFT_DEBUG_CHECKS + const integertime_t ti_current = e->ti_current; + const integertime_t ti_beg = + get_integer_time_begin(ti_current + 1, gp->time_bin); + + if (ti_beg > ti_current) + error( + "g-particle in an impossible time-zone! gp->ti_beg=%lld " + "e->ti_current=%lld", + ti_beg, ti_current); +#endif + + return (gpart_bin <= max_active_bin); } +/** + * @brief Is this s-particle starting its time-step now ? + * + * @param sp The #spart. + * @param e The #engine containing information about the current time. + * @return 1 if the #spart is active, 0 otherwise. + */ +__attribute__((always_inline)) INLINE static int spart_is_starting( + const struct spart *sp, const struct engine *e) { + + const timebin_t max_active_bin = e->max_active_bin; + const timebin_t spart_bin = sp->time_bin; + +#ifdef SWIFT_DEBUG_CHECKS + const integertime_t ti_current = e->ti_current; + const integertime_t ti_beg = + get_integer_time_begin(ti_current + 1, sp->time_bin); + + if (ti_beg > ti_current) + error( + "s-particle in an impossible time-zone! sp->ti_beg=%lld " + "e->ti_current=%lld", + ti_beg, ti_current); +#endif + + return (spart_bin <= max_active_bin); +} #endif /* SWIFT_ACTIVE_H */ diff --git a/src/cache.h b/src/cache.h index 6332f1470564a3a0869b851bd7cd9853d922c143..19d61b657b3aa1fe8675ee413fcde146071381e9 100644 --- a/src/cache.h +++ b/src/cache.h @@ -162,4 +162,22 @@ __attribute__((always_inline)) INLINE void cache_read_particles( #endif } +/** + * @brief Clean the memory allocated by a #cache object. + * + * @param c The #cache to clean. + */ +static INLINE void cache_clean(struct cache *c) { + if (c->count > 0) { + free(c->x); + free(c->y); + free(c->z); + free(c->m); + free(c->vx); + free(c->vy); + free(c->vz); + free(c->h); + } +} + #endif /* SWIFT_CACHE_H */ diff --git a/src/cell.c b/src/cell.c index bbab79d73f494a288112dd0900fa502033cfa674..b3bb2470e4b6e09400a5bdd624b9de78d082091d 100644 --- a/src/cell.c +++ b/src/cell.c @@ -102,6 +102,7 @@ int cell_unpack(struct pcell *pc, struct cell *c, struct space *s) { c->ti_old = pc->ti_old; c->count = pc->count; c->gcount = pc->gcount; + c->scount = pc->scount; c->tag = pc->tag; /* Number of new cells created. */ @@ -114,6 +115,7 @@ int cell_unpack(struct pcell *pc, struct cell *c, struct space *s) { space_getcells(s, 1, &temp); temp->count = 0; temp->gcount = 0; + temp->scount = 0; temp->loc[0] = c->loc[0]; temp->loc[1] = c->loc[1]; temp->loc[2] = c->loc[2]; @@ -195,6 +197,31 @@ int cell_link_gparts(struct cell *c, struct gpart *gparts) { return c->gcount; } +/** + * @brief Link the cells recursively to the given #spart array. + * + * @param c The #cell. + * @param sparts The #spart array. + * + * @return The number of particles linked. + */ +int cell_link_sparts(struct cell *c, struct spart *sparts) { + + c->sparts = sparts; + + /* Fill the progeny recursively, depth-first. */ + if (c->split) { + int offset = 0; + for (int k = 0; k < 8; k++) { + if (c->progeny[k] != NULL) + offset += cell_link_sparts(c->progeny[k], &sparts[offset]); + } + } + + /* Return the total number of linked particles. */ + return c->scount; +} + /** * @brief Pack the data of the given cell and all it's sub-cells. * @@ -215,6 +242,7 @@ int cell_pack(struct cell *c, struct pcell *pc) { pc->ti_old = c->ti_old; pc->count = c->count; pc->gcount = c->gcount; + pc->scount = c->scount; c->tag = pc->tag = atomic_inc(&cell_next_tag) % cell_max_tag; /* Fill in the progeny, depth-first recursion. */ @@ -426,6 +454,134 @@ int cell_glocktree(struct cell *c) { } } +/** + * @brief Lock a cell for access to its #multipole and hold its parents. + * + * @param c The #cell. + * @return 0 on success, 1 on failure + */ +int cell_mlocktree(struct cell *c) { + + TIMER_TIC + + /* First of all, try to lock this cell. */ + if (c->mhold || lock_trylock(&c->mlock) != 0) { + TIMER_TOC(timer_locktree); + return 1; + } + + /* Did somebody hold this cell in the meantime? */ + if (c->mhold) { + + /* Unlock this cell. */ + if (lock_unlock(&c->mlock) != 0) error("Failed to unlock cell."); + + /* Admit defeat. */ + TIMER_TOC(timer_locktree); + return 1; + } + + /* Climb up the tree and lock/hold/unlock. */ + struct cell *finger; + for (finger = c->parent; finger != NULL; finger = finger->parent) { + + /* Lock this cell. */ + if (lock_trylock(&finger->mlock) != 0) break; + + /* Increment the hold. */ + atomic_inc(&finger->mhold); + + /* Unlock the cell. */ + if (lock_unlock(&finger->mlock) != 0) error("Failed to unlock cell."); + } + + /* If we reached the top of the tree, we're done. */ + if (finger == NULL) { + TIMER_TOC(timer_locktree); + return 0; + } + + /* Otherwise, we hit a snag. */ + else { + + /* Undo the holds up to finger. */ + for (struct cell *finger2 = c->parent; finger2 != finger; + finger2 = finger2->parent) + atomic_dec(&finger2->mhold); + + /* Unlock this cell. */ + if (lock_unlock(&c->mlock) != 0) error("Failed to unlock cell."); + + /* Admit defeat. */ + TIMER_TOC(timer_locktree); + return 1; + } +} + +/** + * @brief Lock a cell for access to its array of #spart and hold its parents. + * + * @param c The #cell. + * @return 0 on success, 1 on failure + */ +int cell_slocktree(struct cell *c) { + + TIMER_TIC + + /* First of all, try to lock this cell. */ + if (c->shold || lock_trylock(&c->slock) != 0) { + TIMER_TOC(timer_locktree); + return 1; + } + + /* Did somebody hold this cell in the meantime? */ + if (c->shold) { + + /* Unlock this cell. */ + if (lock_unlock(&c->slock) != 0) error("Failed to unlock cell."); + + /* Admit defeat. */ + TIMER_TOC(timer_locktree); + return 1; + } + + /* Climb up the tree and lock/hold/unlock. */ + struct cell *finger; + for (finger = c->parent; finger != NULL; finger = finger->parent) { + + /* Lock this cell. */ + if (lock_trylock(&finger->slock) != 0) break; + + /* Increment the hold. */ + atomic_inc(&finger->shold); + + /* Unlock the cell. */ + if (lock_unlock(&finger->slock) != 0) error("Failed to unlock cell."); + } + + /* If we reached the top of the tree, we're done. */ + if (finger == NULL) { + TIMER_TOC(timer_locktree); + return 0; + } + + /* Otherwise, we hit a snag. */ + else { + + /* Undo the holds up to finger. */ + for (struct cell *finger2 = c->parent; finger2 != finger; + finger2 = finger2->parent) + atomic_dec(&finger2->shold); + + /* Unlock this cell. */ + if (lock_unlock(&c->slock) != 0) error("Failed to unlock cell."); + + /* Admit defeat. */ + TIMER_TOC(timer_locktree); + return 1; + } +} + /** * @brief Unlock a cell's parents for access to #part array. * @@ -464,24 +620,68 @@ void cell_gunlocktree(struct cell *c) { TIMER_TOC(timer_locktree); } +/** + * @brief Unlock a cell's parents for access to its #multipole. + * + * @param c The #cell. + */ +void cell_munlocktree(struct cell *c) { + + TIMER_TIC + + /* First of all, try to unlock this cell. */ + if (lock_unlock(&c->mlock) != 0) error("Failed to unlock cell."); + + /* Climb up the tree and unhold the parents. */ + for (struct cell *finger = c->parent; finger != NULL; finger = finger->parent) + atomic_dec(&finger->mhold); + + TIMER_TOC(timer_locktree); +} + +/** + * @brief Unlock a cell's parents for access to #spart array. + * + * @param c The #cell. + */ +void cell_sunlocktree(struct cell *c) { + + TIMER_TIC + + /* First of all, try to unlock this cell. */ + if (lock_unlock(&c->slock) != 0) error("Failed to unlock cell."); + + /* Climb up the tree and unhold the parents. */ + for (struct cell *finger = c->parent; finger != NULL; finger = finger->parent) + atomic_dec(&finger->shold); + + TIMER_TOC(timer_locktree); +} + /** * @brief Sort the parts into eight bins along the given pivots. * * @param c The #cell array to be sorted. * @param parts_offset Offset of the cell parts array relative to the * space's parts array, i.e. c->parts - s->parts. + * @param sparts_offset Offset of the cell sparts array relative to the + * space's sparts array, i.e. c->sparts - s->sparts. * @param buff A buffer with at least max(c->count, c->gcount) entries, * used for sorting indices. + * @param sbuff A buffer with at least max(c->scount, c->gcount) entries, + * used for sorting indices for the sparts. * @param gbuff A buffer with at least max(c->count, c->gcount) entries, * used for sorting indices for the gparts. */ -void cell_split(struct cell *c, ptrdiff_t parts_offset, struct cell_buff *buff, +void cell_split(struct cell *c, ptrdiff_t parts_offset, ptrdiff_t sparts_offset, + struct cell_buff *buff, struct cell_buff *sbuff, struct cell_buff *gbuff) { - const int count = c->count, gcount = c->gcount; + const int count = c->count, gcount = c->gcount, scount = c->scount; struct part *parts = c->parts; struct xpart *xparts = c->xparts; struct gpart *gparts = c->gparts; + struct spart *sparts = c->sparts; const double pivot[3] = {c->loc[0] + c->width[0] / 2, c->loc[1] + c->width[1] / 2, c->loc[2] + c->width[2] / 2}; @@ -495,6 +695,16 @@ void cell_split(struct cell *c, ptrdiff_t parts_offset, struct cell_buff *buff, buff[k].x[2] != parts[k].x[2]) error("Inconsistent buff contents."); } + for (int k = 0; k < gcount; k++) { + if (gbuff[k].x[0] != gparts[k].x[0] || gbuff[k].x[1] != gparts[k].x[1] || + gbuff[k].x[2] != gparts[k].x[2]) + error("Inconsistent gbuff contents."); + } + for (int k = 0; k < scount; k++) { + if (sbuff[k].x[0] != sparts[k].x[0] || sbuff[k].x[1] != sparts[k].x[1] || + sbuff[k].x[2] != sparts[k].x[2]) + error("Inconsistent sbuff contents."); + } #endif /* SWIFT_DEBUG_CHECKS */ /* Fill the buffer with the indices. */ @@ -548,7 +758,8 @@ void cell_split(struct cell *c, ptrdiff_t parts_offset, struct cell_buff *buff, } /* Re-link the gparts. */ - if (count > 0 && gcount > 0) part_relink_gparts(parts, count, parts_offset); + if (count > 0 && gcount > 0) + part_relink_gparts_to_parts(parts, count, parts_offset); #ifdef SWIFT_DEBUG_CHECKS /* Check that the buffs are OK. */ @@ -612,7 +823,60 @@ void cell_split(struct cell *c, ptrdiff_t parts_offset, struct cell_buff *buff, error("Sorting failed (progeny=7)."); #endif - /* Now do the same song and dance for the gparts. */ + /* Now do the same song and dance for the sparts. */ + for (int k = 0; k < 8; k++) bucket_count[k] = 0; + + /* Fill the buffer with the indices. */ + for (int k = 0; k < scount; k++) { + const int bid = (sbuff[k].x[0] > pivot[0]) * 4 + + (sbuff[k].x[1] > pivot[1]) * 2 + (sbuff[k].x[2] > pivot[2]); + bucket_count[bid]++; + sbuff[k].ind = bid; + } + + /* Set the buffer offsets. */ + bucket_offset[0] = 0; + for (int k = 1; k <= 8; k++) { + bucket_offset[k] = bucket_offset[k - 1] + bucket_count[k - 1]; + bucket_count[k - 1] = 0; + } + + /* Run through the buckets, and swap particles to their correct spot. */ + for (int bucket = 0; bucket < 8; bucket++) { + for (int k = bucket_offset[bucket] + bucket_count[bucket]; + k < bucket_offset[bucket + 1]; k++) { + int bid = sbuff[k].ind; + if (bid != bucket) { + struct spart spart = sparts[k]; + struct cell_buff temp_buff = sbuff[k]; + while (bid != bucket) { + int j = bucket_offset[bid] + bucket_count[bid]++; + while (sbuff[j].ind == bid) { + j++; + bucket_count[bid]++; + } + memswap(&sparts[j], &spart, sizeof(struct spart)); + memswap(&sbuff[j], &temp_buff, sizeof(struct cell_buff)); + bid = temp_buff.ind; + } + sparts[k] = spart; + sbuff[k] = temp_buff; + } + bucket_count[bid]++; + } + } + + /* Store the counts and offsets. */ + for (int k = 0; k < 8; k++) { + c->progeny[k]->scount = bucket_count[k]; + c->progeny[k]->sparts = &c->sparts[bucket_offset[k]]; + } + + /* Re-link the gparts. */ + if (scount > 0 && gcount > 0) + part_relink_gparts_to_sparts(sparts, scount, sparts_offset); + + /* Finally, do the same song and dance for the gparts. */ for (int k = 0; k < 8; k++) bucket_count[k] = 0; /* Fill the buffer with the indices. */ @@ -663,7 +927,11 @@ void cell_split(struct cell *c, ptrdiff_t parts_offset, struct cell_buff *buff, /* Re-link the parts. */ if (count > 0 && gcount > 0) - part_relink_parts(gparts, gcount, parts - parts_offset); + part_relink_parts_to_gparts(gparts, gcount, parts - parts_offset); + + /* Re-link the sparts. */ + if (scount > 0 && gcount > 0) + part_relink_sparts_to_gparts(gparts, gcount, sparts - sparts_offset); } /** @@ -759,11 +1027,51 @@ void cell_clean_links(struct cell *c, void *data) { */ void cell_check_drift_point(struct cell *c, void *data) { - integertime_t ti_current = *(integertime_t *)data; +#ifdef SWIFT_DEBUG_CHECKS + + const integertime_t ti_drift = *(integertime_t *)data; - if (c->ti_old != ti_current && c->nodeID == engine_rank) - error("Cell in an incorrect time-zone! c->ti_old=%lld ti_current=%lld", - c->ti_old, ti_current); + /* Only check local cells */ + if (c->nodeID != engine_rank) return; + + if (c->ti_old != ti_drift) + error("Cell in an incorrect time-zone! c->ti_old=%lld ti_drift=%lld", + c->ti_old, ti_drift); + + for (int i = 0; i < c->count; ++i) + if (c->parts[i].ti_drift != ti_drift) + error("part in an incorrect time-zone! p->ti_drift=%lld ti_drift=%lld", + c->parts[i].ti_drift, ti_drift); + + for (int i = 0; i < c->gcount; ++i) + if (c->gparts[i].ti_drift != ti_drift) + error("g-part in an incorrect time-zone! gp->ti_drift=%lld ti_drift=%lld", + c->gparts[i].ti_drift, ti_drift); + + for (int i = 0; i < c->scount; ++i) + if (c->sparts[i].ti_drift != ti_drift) + error("s-part in an incorrect time-zone! sp->ti_drift=%lld ti_drift=%lld", + c->sparts[i].ti_drift, ti_drift); +#else + error("Calling debugging code without debugging flag activated."); +#endif +} + +/** + * @brief Resets all the individual cell task counters to 0. + * + * Should only be used for debugging purposes. + * + * @param c The #cell to reset. + */ +void cell_reset_task_counters(struct cell *c) { + +#ifdef SWIFT_DEBUG_CHECKS + for (int t = 0; t < task_type_count; ++t) c->tasks_executed[t] = 0; + for (int t = 0; t < task_subtype_count; ++t) c->subtasks_executed[t] = 0; +#else + error("Calling debugging code without debugging flag activated."); +#endif } /** @@ -805,52 +1113,34 @@ int cell_are_neighbours(const struct cell *restrict ci, */ void cell_check_multipole(struct cell *c, void *data) { - struct multipole ma; +#ifdef SWIFT_DEBUG_CHECKS + struct gravity_tensors ma; + const double tolerance = 1e-5; /* Relative */ + + /* First recurse */ + if (c->split) + for (int k = 0; k < 8; k++) + if (c->progeny[k] != NULL) cell_check_multipole(c->progeny[k], NULL); if (c->gcount > 0) { /* Brute-force calculation */ - multipole_init(&ma, c->gparts, c->gcount); - - /* Compare with recursive one */ - struct multipole mb = c->multipole; - - if (fabsf(ma.mass - mb.mass) / fabsf(ma.mass + mb.mass) > 1e-5) - error("Multipole masses are different (%12.15e vs. %12.15e)", ma.mass, - mb.mass); - - for (int k = 0; k < 3; ++k) - if (fabs(ma.CoM[k] - mb.CoM[k]) / fabs(ma.CoM[k] + mb.CoM[k]) > 1e-5) - error("Multipole CoM are different (%12.15e vs. %12.15e", ma.CoM[k], - mb.CoM[k]); - -#if const_gravity_multipole_order >= 2 - if (fabsf(ma.I_xx - mb.I_xx) / fabsf(ma.I_xx + mb.I_xx) > 1e-5 && - ma.I_xx > 1e-9) - error("Multipole I_xx are different (%12.15e vs. %12.15e)", ma.I_xx, - mb.I_xx); - if (fabsf(ma.I_yy - mb.I_yy) / fabsf(ma.I_yy + mb.I_yy) > 1e-5 && - ma.I_yy > 1e-9) - error("Multipole I_yy are different (%12.15e vs. %12.15e)", ma.I_yy, - mb.I_yy); - if (fabsf(ma.I_zz - mb.I_zz) / fabsf(ma.I_zz + mb.I_zz) > 1e-5 && - ma.I_zz > 1e-9) - error("Multipole I_zz are different (%12.15e vs. %12.15e)", ma.I_zz, - mb.I_zz); - if (fabsf(ma.I_xy - mb.I_xy) / fabsf(ma.I_xy + mb.I_xy) > 1e-5 && - ma.I_xy > 1e-9) - error("Multipole I_xy are different (%12.15e vs. %12.15e)", ma.I_xy, - mb.I_xy); - if (fabsf(ma.I_xz - mb.I_xz) / fabsf(ma.I_xz + mb.I_xz) > 1e-5 && - ma.I_xz > 1e-9) - error("Multipole I_xz are different (%12.15e vs. %12.15e)", ma.I_xz, - mb.I_xz); - if (fabsf(ma.I_yz - mb.I_yz) / fabsf(ma.I_yz + mb.I_yz) > 1e-5 && - ma.I_yz > 1e-9) - error("Multipole I_yz are different (%12.15e vs. %12.15e)", ma.I_yz, - mb.I_yz); -#endif + gravity_P2M(&ma, c->gparts, c->gcount); + + /* Now compare the multipole expansion */ + if (!gravity_multipole_equal(&ma.m_pole, &c->multipole->m_pole, + tolerance)) { + message("Multipoles are not equal at depth=%d!", c->depth); + message("Correct answer:"); + gravity_multipole_print(&ma.m_pole); + message("Recursive multipole:"); + gravity_multipole_print(&c->multipole->m_pole); + error("Aborting"); + } } +#else + error("Calling debugging code without debugging flag activated."); +#endif } /** @@ -911,6 +1201,8 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) { struct engine *e = s->space->e; #endif + int rebuild = 0; + /* Un-skip the density tasks involved with this cell. */ for (struct link *l = c->density; l != NULL; l = l->next) { struct task *t = l->t; @@ -1053,10 +1345,13 @@ int cell_unskip_tasks(struct cell *c, struct scheduler *s) { if (c->kick1 != NULL) scheduler_activate(s, c->kick1); if (c->kick2 != NULL) scheduler_activate(s, c->kick2); if (c->timestep != NULL) scheduler_activate(s, c->timestep); + if (c->grav_down != NULL) scheduler_activate(s, c->grav_down); + if (c->grav_long_range != NULL) scheduler_activate(s, c->grav_long_range); + if (c->grav_top_level != NULL) scheduler_activate(s, c->grav_top_level); if (c->cooling != NULL) scheduler_activate(s, c->cooling); if (c->sourceterms != NULL) scheduler_activate(s, c->sourceterms); - return 0; + return rebuild; } /** @@ -1080,25 +1375,27 @@ void cell_set_super(struct cell *c, struct cell *super) { } /** - * @brief Recursively drifts all particles and g-particles in a cell hierarchy. + * @brief Recursively drifts particles of all kinds in a cell hierarchy. * * @param c The #cell. * @param e The #engine (to get ti_current). */ -void cell_drift(struct cell *c, const struct engine *e) { +void cell_drift_particles(struct cell *c, const struct engine *e) { + const float hydro_h_max = e->hydro_properties->h_max; const double timeBase = e->timeBase; const integertime_t ti_old = c->ti_old; const integertime_t ti_current = e->ti_current; struct part *const parts = c->parts; struct xpart *const xparts = c->xparts; struct gpart *const gparts = c->gparts; + struct spart *const sparts = c->sparts; /* Drift from the last time the cell was drifted to the current time */ const double dt = (ti_current - ti_old) * timeBase; float dx_max = 0.f, dx2_max = 0.f; float dx_max_sort = 0.0f, dx2_max_sort = 0.f; - float h_max = 0.f; + float cell_h_max = 0.f; /* Check that we are actually going to move forward. */ if (ti_current < ti_old) error("Attempt to drift to the past"); @@ -1110,10 +1407,10 @@ void cell_drift(struct cell *c, const struct engine *e) { for (int k = 0; k < 8; k++) if (c->progeny[k] != NULL) { struct cell *cp = c->progeny[k]; - cell_drift(cp, e); + cell_drift_particles(cp, e); dx_max = max(dx_max, cp->dx_max); dx_max_sort = max(dx_max_sort, cp->dx_max_sort); - h_max = max(h_max, cp->h_max); + cell_h_max = max(cell_h_max, cp->h_max); if (cp->ti_sort > c->ti_sort) c->sorted = 0; else @@ -1139,7 +1436,7 @@ void cell_drift(struct cell *c, const struct engine *e) { dx2_max = max(dx2_max, dx2); } - /* Loop over all the particles in the cell */ + /* Loop over all the gas particles in the cell */ const size_t nr_parts = c->count; for (size_t k = 0; k < nr_parts; k++) { @@ -1150,6 +1447,9 @@ void cell_drift(struct cell *c, const struct engine *e) { /* Drift... */ drift_part(p, xp, dt, timeBase, ti_old, ti_current); + /* Limit h to within the allowed range */ + p->h = min(p->h, hydro_h_max); + /* Compute (square of) motion since last cell construction */ const float dx2 = xp->x_diff[0] * xp->x_diff[0] + xp->x_diff[1] * xp->x_diff[1] + @@ -1161,7 +1461,20 @@ void cell_drift(struct cell *c, const struct engine *e) { dx2_max_sort = max(dx2_max_sort, dx2_sort); /* Maximal smoothing length */ - h_max = max(h_max, p->h); + cell_h_max = max(cell_h_max, p->h); + } + + /* Loop over all the star particles in the cell */ + const size_t nr_sparts = c->scount; + for (size_t k = 0; k < nr_sparts; k++) { + + /* Get a handle on the spart. */ + struct spart *const sp = &sparts[k]; + + /* Drift... */ + drift_spart(sp, dt, timeBase, ti_old, ti_current); + + /* Note: no need to compute dx_max as all spart have a gpart */ } /* Now, get the maximal particle motion from its square */ @@ -1170,13 +1483,13 @@ void cell_drift(struct cell *c, const struct engine *e) { } else { - h_max = c->h_max; + cell_h_max = c->h_max; dx_max = c->dx_max; dx_max_sort = c->dx_max_sort; } /* Store the values */ - c->h_max = h_max; + c->h_max = cell_h_max; c->dx_max = dx_max; c->dx_max_sort = dx_max_sort; @@ -1184,6 +1497,54 @@ void cell_drift(struct cell *c, const struct engine *e) { c->ti_old = ti_current; } +/** + * @brief Recursively drifts all multipoles in a cell hierarchy. + * + * @param c The #cell. + * @param e The #engine (to get ti_current). + */ +void cell_drift_all_multipoles(struct cell *c, const struct engine *e) { + + const double timeBase = e->timeBase; + const integertime_t ti_old_multipole = c->ti_old_multipole; + const integertime_t ti_current = e->ti_current; + + /* Drift from the last time the cell was drifted to the current time */ + const double dt = (ti_current - ti_old_multipole) * timeBase; + + /* Check that we are actually going to move forward. */ + if (ti_current < ti_old_multipole) error("Attempt to drift to the past"); + + /* Are we not in a leaf ? */ + if (c->split) { + + /* Loop over the progeny and drift the multipoles. */ + for (int k = 0; k < 8; k++) + if (c->progeny[k] != NULL) cell_drift_particles(c->progeny[k], e); + + } else if (ti_current > ti_old_multipole) { + + /* Drift the multipole */ + gravity_multipole_drift(c->multipole, dt); + } + + /* Update the time of the last drift */ + c->ti_old_multipole = ti_current; +} + +/** + * @brief Drifts the multipole of a cell to the current time. + * + * Only drifts the multipole at this level. Multipoles deeper in the + * tree are not updated. + * + * @param c The #cell. + * @param e The #engine (to get ti_current). + */ +void cell_drift_multipole(struct cell *c, const struct engine *e) { + error("To be implemented"); +} + /** * @brief Recursively checks that all particles in a cell have a time-step */ @@ -1203,5 +1564,7 @@ void cell_check_timesteps(struct cell *c) { if (c->parts[i].time_bin == 0) error("Particle without assigned time-bin"); } +#else + error("Calling debugging code without debugging flag activated."); #endif } diff --git a/src/cell.h b/src/cell.h index c9a3513af2202961192ddbeccb9ad57c8c6d9723..2227c9b3ba2998d2aa09b0461771bb998182f4c4 100644 --- a/src/cell.h +++ b/src/cell.h @@ -74,10 +74,10 @@ struct pcell { /* Stats on this cell's particles. */ double h_max; - integertime_t ti_end_min, ti_end_max, ti_old; + integertime_t ti_end_min, ti_end_max, ti_beg_max, ti_old; /* Number of particles in this cell. */ - int count, gcount; + int count, gcount, scount; /* tag used for MPI communication. */ int tag; @@ -94,9 +94,6 @@ struct pcell { */ struct cell { - /*! This cell's multipole. */ - struct multipole multipole; - /*! The cell location on the grid. */ double loc[3]; @@ -106,6 +103,9 @@ struct cell { /*! Max smoothing length in this cell. */ double h_max; + /*! This cell's multipole. */ + struct gravity_tensors *multipole; + /*! Linking pointer for "memory management". */ struct cell *next; @@ -118,6 +118,9 @@ struct cell { /*! Pointer to the #gpart data. */ struct gpart *gparts; + /*! Pointer to the #spart data. */ + struct spart *sparts; + /*! Pointer for the sorted indices. */ struct entry *sort; @@ -167,7 +170,10 @@ struct cell { struct task *timestep; /*! Task constructing the multipole from the particles */ - struct task *grav_up; + struct task *grav_top_level; + + /*! Task constructing the multipole from the particles */ + struct task *grav_long_range; /*! Task propagating the multipole to the particles */ struct task *grav_down; @@ -224,12 +230,18 @@ struct cell { /*! Maximum end of (integer) time step in this cell. */ integertime_t ti_end_max; - /*! Last (integer) time the cell's content was drifted forward in time. */ + /*! Maximum beginning of (integer) time step in this cell. */ + integertime_t ti_beg_max; + + /*! Last (integer) time the cell's particle was drifted forward in time. */ integertime_t ti_old; /*! Last (integer) time the cell's sort arrays were updated. */ integertime_t ti_sort; + /*! Last (integer) time the cell's multipole was drifted forward in time. */ + integertime_t ti_old_multipole; + /*! Minimum dimension, i.e. smallest edge of this cell (min(width)). */ float dmin; @@ -245,6 +257,9 @@ struct cell { /*! Nr of #gpart in this cell. */ int gcount; + /*! Nr of #spart in this cell. */ + int scount; + /*! The size of the sort array */ int sortsize; @@ -257,6 +272,12 @@ struct cell { /*! Spin lock for various uses (#gpart case). */ swift_lock_type glock; + /*! Spin lock for various uses (#multipole case). */ + swift_lock_type mlock; + + /*! Spin lock for various uses (#spart case). */ + swift_lock_type slock; + /*! ID of the previous owner, e.g. runner. */ int owner; @@ -266,6 +287,9 @@ struct cell { /*! Number of #gpart updated in this cell. */ int g_updated; + /*! Number of #spart updated in this cell. */ + int s_updated; + /*! ID of the node this cell lives on. */ int nodeID; @@ -275,6 +299,12 @@ struct cell { /*! Is the #gpart data of this cell being used in a sub-cell? */ int ghold; + /*! Is the #multipole data of this cell being used in a sub-cell? */ + int mhold; + + /*! Is the #spart data of this cell being used in a sub-cell? */ + int shold; + /*! Number of tasks that are associated with this cell. */ short int nr_tasks; @@ -287,6 +317,14 @@ struct cell { /*! The maximal depth of this cell and its progenies */ char maxdepth; +#ifdef SWIFT_DEBUG_CHECKS + /*! The list of tasks that have been executed on this cell */ + char tasks_executed[64]; + + /*! The list of sub-tasks that have been executed on this cell */ + char subtasks_executed[64]; +#endif + } SWIFT_STRUCT_ALIGN; /* Convert cell location to ID. */ @@ -294,13 +332,18 @@ struct cell { ((int)(k) + (cdim)[2] * ((int)(j) + (cdim)[1] * (int)(i))) /* Function prototypes. */ -void cell_split(struct cell *c, ptrdiff_t parts_offset, struct cell_buff *buff, +void cell_split(struct cell *c, ptrdiff_t parts_offset, ptrdiff_t sparts_offset, + struct cell_buff *buff, struct cell_buff *sbuff, struct cell_buff *gbuff); void cell_sanitize(struct cell *c); int cell_locktree(struct cell *c); void cell_unlocktree(struct cell *c); int cell_glocktree(struct cell *c); void cell_gunlocktree(struct cell *c); +int cell_mlocktree(struct cell *c); +void cell_munlocktree(struct cell *c); +int cell_slocktree(struct cell *c); +void cell_sunlocktree(struct cell *c); int cell_pack(struct cell *c, struct pcell *pc); int cell_unpack(struct pcell *pc, struct cell *c, struct space *s); int cell_pack_ti_ends(struct cell *c, integertime_t *ti_ends); @@ -308,6 +351,7 @@ int cell_unpack_ti_ends(struct cell *c, integertime_t *ti_ends); int cell_getsize(struct cell *c); int cell_link_parts(struct cell *c, struct part *parts); int cell_link_gparts(struct cell *c, struct gpart *gparts); +int cell_link_sparts(struct cell *c, struct spart *sparts); void cell_convert_hydro(struct cell *c, void *data); void cell_clean_links(struct cell *c, void *data); int cell_are_neighbours(const struct cell *restrict ci, @@ -315,10 +359,13 @@ int cell_are_neighbours(const struct cell *restrict ci, void cell_check_multipole(struct cell *c, void *data); void cell_clean(struct cell *c); void cell_check_drift_point(struct cell *c, void *data); +void cell_reset_task_counters(struct cell *c); int cell_is_drift_needed(struct cell *c, const struct engine *e); int cell_unskip_tasks(struct cell *c, struct scheduler *s); void cell_set_super(struct cell *c, struct cell *super); -void cell_drift(struct cell *c, const struct engine *e); +void cell_drift_particles(struct cell *c, const struct engine *e); +void cell_drift_multipole(struct cell *c, const struct engine *e); +void cell_drift_all_multipoles(struct cell *c, const struct engine *e); void cell_check_timesteps(struct cell *c); #endif /* SWIFT_CELL_H */ diff --git a/src/common_io.c b/src/common_io.c index 1f1ec401547c81e137b4e7d836ab58cb87280d8b..df0bbdc29ec357da3ba14410c0f9c56e0d69160a 100644 --- a/src/common_io.c +++ b/src/common_io.c @@ -47,9 +47,6 @@ #include "units.h" #include "version.h" -const char* particle_type_names[NUM_PARTICLE_TYPES] = { - "Gas", "DM", "Boundary", "Dummy", "Star", "BH"}; - /** * @brief Converts a C data type to the HDF5 equivalent. * @@ -57,7 +54,7 @@ const char* particle_type_names[NUM_PARTICLE_TYPES] = { * to change the exact storage types matching the code types in a transparent *way. */ -hid_t hdf5Type(enum DATA_TYPE type) { +hid_t io_hdf5_type(enum IO_DATA_TYPE type) { switch (type) { case INT: @@ -87,7 +84,7 @@ hid_t hdf5Type(enum DATA_TYPE type) { /** * @brief Returns the memory size of the data type */ -size_t sizeOfType(enum DATA_TYPE type) { +size_t io_sizeof_type(enum IO_DATA_TYPE type) { switch (type) { case INT: @@ -119,7 +116,7 @@ size_t sizeOfType(enum DATA_TYPE type) { * * Returns an error if the type is not FLOAT or DOUBLE */ -int isDoublePrecision(enum DATA_TYPE type) { +int io_is_double_precision(enum IO_DATA_TYPE type) { switch (type) { case FLOAT: @@ -137,12 +134,13 @@ int isDoublePrecision(enum DATA_TYPE type) { * * @param grp The group from which to read. * @param name The name of the attribute to read. - * @param type The #DATA_TYPE of the attribute. + * @param type The #IO_DATA_TYPE of the attribute. * @param data (output) The attribute read from the HDF5 group. * * Calls #error() if an error occurs. */ -void readAttribute(hid_t grp, char* name, enum DATA_TYPE type, void* data) { +void io_read_attribute(hid_t grp, char* name, enum IO_DATA_TYPE type, + void* data) { hid_t h_attr = 0, h_err = 0; h_attr = H5Aopen(grp, name, H5P_DEFAULT); @@ -150,7 +148,7 @@ void readAttribute(hid_t grp, char* name, enum DATA_TYPE type, void* data) { error("Error while opening attribute '%s'", name); } - h_err = H5Aread(h_attr, hdf5Type(type), data); + h_err = H5Aread(h_attr, io_hdf5_type(type), data); if (h_err < 0) { error("Error while reading attribute '%s'", name); } @@ -163,14 +161,14 @@ void readAttribute(hid_t grp, char* name, enum DATA_TYPE type, void* data) { * * @param grp The group in which to write. * @param name The name of the attribute to write. - * @param type The #DATA_TYPE of the attribute. + * @param type The #IO_DATA_TYPE of the attribute. * @param data The attribute to write. * @param num The number of elements to write * * Calls #error() if an error occurs. */ -void writeAttribute(hid_t grp, const char* name, enum DATA_TYPE type, - void* data, int num) { +void io_write_attribute(hid_t grp, const char* name, enum IO_DATA_TYPE type, + void* data, int num) { hid_t h_space = 0, h_attr = 0, h_err = 0; hsize_t dim[1] = {num}; @@ -184,12 +182,12 @@ void writeAttribute(hid_t grp, const char* name, enum DATA_TYPE type, error("Error while changing dataspace shape for attribute '%s'.", name); } - h_attr = H5Acreate1(grp, name, hdf5Type(type), h_space, H5P_DEFAULT); + h_attr = H5Acreate1(grp, name, io_hdf5_type(type), h_space, H5P_DEFAULT); if (h_attr < 0) { error("Error while creating attribute '%s'.", name); } - h_err = H5Awrite(h_attr, hdf5Type(type), data); + h_err = H5Awrite(h_attr, io_hdf5_type(type), data); if (h_err < 0) { error("Error while reading attribute '%s'.", name); } @@ -208,8 +206,8 @@ void writeAttribute(hid_t grp, const char* name, enum DATA_TYPE type, * * Calls #error() if an error occurs. */ -void writeStringAttribute(hid_t grp, const char* name, const char* str, - int length) { +void io_writeStringAttribute(hid_t grp, const char* name, const char* str, + int length) { hid_t h_space = 0, h_attr = 0, h_err = 0, h_type = 0; h_space = H5Screate(H5S_SCALAR); @@ -248,8 +246,8 @@ void writeStringAttribute(hid_t grp, const char* name, const char* str, * @param name The name of the attribute * @param data The value to write */ -void writeAttribute_d(hid_t grp, const char* name, double data) { - writeAttribute(grp, name, DOUBLE, &data, 1); +void io_write_attribute_d(hid_t grp, const char* name, double data) { + io_write_attribute(grp, name, DOUBLE, &data, 1); } /** @@ -258,8 +256,8 @@ void writeAttribute_d(hid_t grp, const char* name, double data) { * @param name The name of the attribute * @param data The value to write */ -void writeAttribute_f(hid_t grp, const char* name, float data) { - writeAttribute(grp, name, FLOAT, &data, 1); +void io_write_attribute_f(hid_t grp, const char* name, float data) { + io_write_attribute(grp, name, FLOAT, &data, 1); } /** @@ -269,8 +267,8 @@ void writeAttribute_f(hid_t grp, const char* name, float data) { * @param data The value to write */ -void writeAttribute_i(hid_t grp, const char* name, int data) { - writeAttribute(grp, name, INT, &data, 1); +void io_write_attribute_i(hid_t grp, const char* name, int data) { + io_write_attribute(grp, name, INT, &data, 1); } /** @@ -279,8 +277,8 @@ void writeAttribute_i(hid_t grp, const char* name, int data) { * @param name The name of the attribute * @param data The value to write */ -void writeAttribute_l(hid_t grp, const char* name, long data) { - writeAttribute(grp, name, LONG, &data, 1); +void io_write_attribute_l(hid_t grp, const char* name, long data) { + io_write_attribute(grp, name, LONG, &data, 1); } /** @@ -289,18 +287,18 @@ void writeAttribute_l(hid_t grp, const char* name, long data) { * @param name The name of the attribute * @param str The string to write */ -void writeAttribute_s(hid_t grp, const char* name, const char* str) { - writeStringAttribute(grp, name, str, strlen(str)); +void io_write_attribute_s(hid_t grp, const char* name, const char* str) { + io_writeStringAttribute(grp, name, str, strlen(str)); } /** * @brief Reads the Unit System from an IC file. * @param h_file The (opened) HDF5 file from which to read. - * @param us The UnitSystem to fill. + * @param us The unit_system to fill. * * If the 'Units' group does not exist in the ICs, cgs units will be assumed */ -void readUnitSystem(hid_t h_file, struct UnitSystem* us) { +void io_read_unit_system(hid_t h_file, struct unit_system* us) { hid_t h_grp = H5Gopen(h_file, "/Units", H5P_DEFAULT); @@ -318,14 +316,16 @@ void readUnitSystem(hid_t h_file, struct UnitSystem* us) { } /* Ok, Read the damn thing */ - readAttribute(h_grp, "Unit length in cgs (U_L)", DOUBLE, - &us->UnitLength_in_cgs); - readAttribute(h_grp, "Unit mass in cgs (U_M)", DOUBLE, &us->UnitMass_in_cgs); - readAttribute(h_grp, "Unit time in cgs (U_t)", DOUBLE, &us->UnitTime_in_cgs); - readAttribute(h_grp, "Unit current in cgs (U_I)", DOUBLE, - &us->UnitCurrent_in_cgs); - readAttribute(h_grp, "Unit temperature in cgs (U_T)", DOUBLE, - &us->UnitTemperature_in_cgs); + io_read_attribute(h_grp, "Unit length in cgs (U_L)", DOUBLE, + &us->UnitLength_in_cgs); + io_read_attribute(h_grp, "Unit mass in cgs (U_M)", DOUBLE, + &us->UnitMass_in_cgs); + io_read_attribute(h_grp, "Unit time in cgs (U_t)", DOUBLE, + &us->UnitTime_in_cgs); + io_read_attribute(h_grp, "Unit current in cgs (U_I)", DOUBLE, + &us->UnitCurrent_in_cgs); + io_read_attribute(h_grp, "Unit temperature in cgs (U_T)", DOUBLE, + &us->UnitTemperature_in_cgs); /* Clean up */ H5Gclose(h_grp); @@ -334,26 +334,26 @@ void readUnitSystem(hid_t h_file, struct UnitSystem* us) { /** * @brief Writes the current Unit System * @param h_file The (opened) HDF5 file in which to write - * @param us The UnitSystem to dump + * @param us The unit_system to dump * @param groupName The name of the HDF5 group to write to */ -void writeUnitSystem(hid_t h_file, const struct UnitSystem* us, - const char* groupName) { +void io_write_unit_system(hid_t h_file, const struct unit_system* us, + const char* groupName) { hid_t h_grpunit = 0; h_grpunit = H5Gcreate1(h_file, groupName, 0); if (h_grpunit < 0) error("Error while creating Unit System group"); - writeAttribute_d(h_grpunit, "Unit mass in cgs (U_M)", - units_get_base_unit(us, UNIT_MASS)); - writeAttribute_d(h_grpunit, "Unit length in cgs (U_L)", - units_get_base_unit(us, UNIT_LENGTH)); - writeAttribute_d(h_grpunit, "Unit time in cgs (U_t)", - units_get_base_unit(us, UNIT_TIME)); - writeAttribute_d(h_grpunit, "Unit current in cgs (U_I)", - units_get_base_unit(us, UNIT_CURRENT)); - writeAttribute_d(h_grpunit, "Unit temperature in cgs (U_T)", - units_get_base_unit(us, UNIT_TEMPERATURE)); + io_write_attribute_d(h_grpunit, "Unit mass in cgs (U_M)", + units_get_base_unit(us, UNIT_MASS)); + io_write_attribute_d(h_grpunit, "Unit length in cgs (U_L)", + units_get_base_unit(us, UNIT_LENGTH)); + io_write_attribute_d(h_grpunit, "Unit time in cgs (U_t)", + units_get_base_unit(us, UNIT_TIME)); + io_write_attribute_d(h_grpunit, "Unit current in cgs (U_I)", + units_get_base_unit(us, UNIT_CURRENT)); + io_write_attribute_d(h_grpunit, "Unit temperature in cgs (U_T)", + units_get_base_unit(us, UNIT_TEMPERATURE)); H5Gclose(h_grpunit); } @@ -362,34 +362,38 @@ void writeUnitSystem(hid_t h_file, const struct UnitSystem* us, * @brief Writes the code version to the file * @param h_file The (opened) HDF5 file in which to write */ -void writeCodeDescription(hid_t h_file) { +void io_write_code_description(hid_t h_file) { hid_t h_grpcode = 0; h_grpcode = H5Gcreate1(h_file, "/Code", 0); if (h_grpcode < 0) error("Error while creating code group"); - writeAttribute_s(h_grpcode, "Code Version", package_version()); - writeAttribute_s(h_grpcode, "Compiler Name", compiler_name()); - writeAttribute_s(h_grpcode, "Compiler Version", compiler_version()); - writeAttribute_s(h_grpcode, "Git Branch", git_branch()); - writeAttribute_s(h_grpcode, "Git Revision", git_revision()); - writeAttribute_s(h_grpcode, "Configuration options", configuration_options()); - writeAttribute_s(h_grpcode, "CFLAGS", compilation_cflags()); - writeAttribute_s(h_grpcode, "HDF5 library version", hdf5_version()); + io_write_attribute_s(h_grpcode, "Code Version", package_version()); + io_write_attribute_s(h_grpcode, "Compiler Name", compiler_name()); + io_write_attribute_s(h_grpcode, "Compiler Version", compiler_version()); + io_write_attribute_s(h_grpcode, "Git Branch", git_branch()); + io_write_attribute_s(h_grpcode, "Git Revision", git_revision()); + io_write_attribute_s(h_grpcode, "Git Date", git_date()); + io_write_attribute_s(h_grpcode, "Configuration options", + configuration_options()); + io_write_attribute_s(h_grpcode, "CFLAGS", compilation_cflags()); + io_write_attribute_s(h_grpcode, "HDF5 library version", hdf5_version()); #ifdef HAVE_FFTW - writeAttribute_s(h_grpcode, "FFTW library version", fftw3_version()); + io_write_attribute_s(h_grpcode, "FFTW library version", fftw3_version()); #endif #ifdef WITH_MPI - writeAttribute_s(h_grpcode, "MPI library", mpi_version()); + io_write_attribute_s(h_grpcode, "MPI library", mpi_version()); #ifdef HAVE_METIS - writeAttribute_s(h_grpcode, "METIS library version", metis_version()); + io_write_attribute_s(h_grpcode, "METIS library version", metis_version()); #endif #else - writeAttribute_s(h_grpcode, "MPI library", "Non-MPI version of SWIFT"); + io_write_attribute_s(h_grpcode, "MPI library", "Non-MPI version of SWIFT"); #endif H5Gclose(h_grpcode); } +#endif /* HAVE_HDF5 */ + /* ------------------------------------------------------------------------------------------------ * This part writes the XMF file descriptor enabling a visualisation through * ParaView @@ -403,170 +407,6 @@ void writeCodeDescription(hid_t h_file) { * * @todo Use a proper XML library to avoid stupid copies. */ -FILE* prepareXMFfile(const char* baseName) { - char buffer[1024]; - - char fileName[FILENAME_BUFFER_SIZE]; - char tempFileName[FILENAME_BUFFER_SIZE]; - snprintf(fileName, FILENAME_BUFFER_SIZE, "%s.xmf", baseName); - snprintf(tempFileName, FILENAME_BUFFER_SIZE, "%s_temp.xmf", baseName); - FILE* xmfFile = fopen(fileName, "r"); - FILE* tempFile = fopen(tempFileName, "w"); - - if (xmfFile == NULL) error("Unable to open current XMF file."); - - if (tempFile == NULL) error("Unable to open temporary file."); - - /* First we make a temporary copy of the XMF file and count the lines */ - int counter = 0; - while (fgets(buffer, 1024, xmfFile) != NULL) { - counter++; - fprintf(tempFile, "%s", buffer); - } - fclose(tempFile); - fclose(xmfFile); - - /* We then copy the XMF file back up to the closing lines */ - xmfFile = fopen(fileName, "w"); - tempFile = fopen(tempFileName, "r"); - - if (xmfFile == NULL) error("Unable to open current XMF file."); - - if (tempFile == NULL) error("Unable to open temporary file."); - - int i = 0; - while (fgets(buffer, 1024, tempFile) != NULL && i < counter - 3) { - i++; - fprintf(xmfFile, "%s", buffer); - } - fprintf(xmfFile, "\n"); - fclose(tempFile); - remove(tempFileName); - - return xmfFile; -} - -/** - * @brief Writes the begin of the XMF file - * - * @todo Exploit the XML nature of the XMF format to write a proper XML writer - *and simplify all the XMF-related stuff. - */ -void createXMFfile(const char* baseName) { - - char fileName[FILENAME_BUFFER_SIZE]; - snprintf(fileName, FILENAME_BUFFER_SIZE, "%s.xmf", baseName); - FILE* xmfFile = fopen(fileName, "w"); - - fprintf(xmfFile, "<?xml version=\"1.0\" ?> \n"); - fprintf(xmfFile, "<!DOCTYPE Xdmf SYSTEM \"Xdmf.dtd\" []> \n"); - fprintf( - xmfFile, - "<Xdmf xmlns:xi=\"http://www.w3.org/2003/XInclude\" Version=\"2.1\">\n"); - fprintf(xmfFile, "<Domain>\n"); - fprintf(xmfFile, - "<Grid Name=\"TimeSeries\" GridType=\"Collection\" " - "CollectionType=\"Temporal\">\n\n"); - - fprintf(xmfFile, "</Grid>\n"); - fprintf(xmfFile, "</Domain>\n"); - fprintf(xmfFile, "</Xdmf>\n"); - - fclose(xmfFile); -} - -/** - * @brief Writes the part of the XMF entry presenting the geometry of the - *snapshot - * - * @param xmfFile The file to write in. - * @param hdfFileName The name of the HDF5 file corresponding to this output. - * @param time The current simulation time. - */ -void writeXMFoutputheader(FILE* xmfFile, char* hdfFileName, float time) { - /* Write end of file */ - - fprintf(xmfFile, "<!-- XMF description for file: %s -->\n", hdfFileName); - fprintf(xmfFile, - "<Grid GridType=\"Collection\" CollectionType=\"Spatial\">\n"); - fprintf(xmfFile, "<Time Type=\"Single\" Value=\"%f\"/>\n", time); -} - -/** - * @brief Writes the end of the XMF file (closes all open markups) - * - * @param xmfFile The file to write in. - * @param output The number of this output. - * @param time The current simulation time. - */ -void writeXMFoutputfooter(FILE* xmfFile, int output, float time) { - /* Write end of the section of this time step */ - - fprintf(xmfFile, - "\n</Grid> <!-- End of meta-data for output=%03i, time=%f -->\n", - output, time); - fprintf(xmfFile, "\n</Grid> <!-- timeSeries -->\n"); - fprintf(xmfFile, "</Domain>\n"); - fprintf(xmfFile, "</Xdmf>\n"); - - fclose(xmfFile); -} - -void writeXMFgroupheader(FILE* xmfFile, char* hdfFileName, size_t N, - enum PARTICLE_TYPE ptype) { - fprintf(xmfFile, "\n<Grid Name=\"%s\" GridType=\"Uniform\">\n", - particle_type_names[ptype]); - fprintf(xmfFile, - "<Topology TopologyType=\"Polyvertex\" Dimensions=\"%zu\"/>\n", N); - fprintf(xmfFile, "<Geometry GeometryType=\"XYZ\">\n"); - fprintf(xmfFile, - "<DataItem Dimensions=\"%zu 3\" NumberType=\"Double\" " - "Precision=\"8\" " - "Format=\"HDF\">%s:/PartType%d/Coordinates</DataItem>\n", - N, hdfFileName, (int)ptype); - fprintf(xmfFile, - "</Geometry>\n <!-- Done geometry for %s, start of particle fields " - "list -->\n", - particle_type_names[ptype]); -} - -void writeXMFgroupfooter(FILE* xmfFile, enum PARTICLE_TYPE ptype) { - fprintf(xmfFile, "</Grid> <!-- End of meta-data for parttype=%s -->\n", - particle_type_names[ptype]); -} - -/** - * @brief Writes the lines corresponding to an array of the HDF5 output - * - * @param xmfFile The file in which to write - * @param fileName The name of the HDF5 file associated to this XMF descriptor. - * @param partTypeGroupName The name of the group containing the particles in - *the HDF5 file. - * @param name The name of the array in the HDF5 file. - * @param N The number of particles. - * @param dim The dimension of the quantity (1 for scalars, 3 for vectors). - * @param type The type of the data to write. - * - * @todo Treat the types in a better way. - */ -void writeXMFline(FILE* xmfFile, const char* fileName, - const char* partTypeGroupName, const char* name, size_t N, - int dim, enum DATA_TYPE type) { - fprintf(xmfFile, - "<Attribute Name=\"%s\" AttributeType=\"%s\" Center=\"Node\">\n", - name, dim == 1 ? "Scalar" : "Vector"); - if (dim == 1) - fprintf(xmfFile, - "<DataItem Dimensions=\"%zu\" NumberType=\"Double\" " - "Precision=\"%d\" Format=\"HDF\">%s:%s/%s</DataItem>\n", - N, type == FLOAT ? 4 : 8, fileName, partTypeGroupName, name); - else - fprintf(xmfFile, - "<DataItem Dimensions=\"%zu %d\" NumberType=\"Double\" " - "Precision=\"%d\" Format=\"HDF\">%s:%s/%s</DataItem>\n", - N, dim, type == FLOAT ? 4 : 8, fileName, partTypeGroupName, name); - fprintf(xmfFile, "</Attribute>\n"); -} /** * @brief Prepare the DM particles (in gparts) read in for the addition of the @@ -578,7 +418,7 @@ void writeXMFline(FILE* xmfFile, const char* fileName, * @param gparts The array of #gpart freshly read in. * @param Ndm The number of DM particles read in. */ -void prepare_dm_gparts(struct gpart* const gparts, size_t Ndm) { +void io_prepare_dm_gparts(struct gpart* const gparts, size_t Ndm) { /* Let's give all these gparts a negative id */ for (size_t i = 0; i < Ndm; ++i) { @@ -586,6 +426,9 @@ void prepare_dm_gparts(struct gpart* const gparts, size_t Ndm) { if (gparts[i].id_or_neg_offset <= 0) error("0 or negative ID for DM particle %zu: ID=%lld", i, gparts[i].id_or_neg_offset); + + /* Set gpart type */ + gparts[i].type = swift_type_dark_matter; } } @@ -597,13 +440,13 @@ void prepare_dm_gparts(struct gpart* const gparts, size_t Ndm) { * * @param parts The array of #part freshly read in. * @param gparts The array of #gpart freshly read in with all the DM particles - *at the start + * at the start * @param Ngas The number of gas particles read in. * @param Ndm The number of DM particles read in. */ -void duplicate_hydro_gparts(struct part* const parts, - struct gpart* const gparts, size_t Ngas, - size_t Ndm) { +void io_duplicate_hydro_gparts(struct part* const parts, + struct gpart* const gparts, size_t Ngas, + size_t Ndm) { for (size_t i = 0; i < Ngas; ++i) { @@ -618,12 +461,53 @@ void duplicate_hydro_gparts(struct part* const parts, gparts[i + Ndm].mass = hydro_get_mass(&parts[i]); + /* Set gpart type */ + gparts[i + Ndm].type = swift_type_gas; + /* Link the particles */ gparts[i + Ndm].id_or_neg_offset = -i; parts[i].gpart = &gparts[i + Ndm]; } } +/** + * @brief Copy every #spart into the corresponding #gpart and link them. + * + * This function assumes that the DM particles and gas particles are all at + * the start of the gparts array and adds the star particles afterwards + * + * @param sparts The array of #spart freshly read in. + * @param gparts The array of #gpart freshly read in with all the DM and gas + * particles at the start. + * @param Nstars The number of stars particles read in. + * @param Ndm The number of DM and gas particles read in. + */ +void io_duplicate_star_gparts(struct spart* const sparts, + struct gpart* const gparts, size_t Nstars, + size_t Ndm) { + + for (size_t i = 0; i < Nstars; ++i) { + + /* Duplicate the crucial information */ + gparts[i + Ndm].x[0] = sparts[i].x[0]; + gparts[i + Ndm].x[1] = sparts[i].x[1]; + gparts[i + Ndm].x[2] = sparts[i].x[2]; + + gparts[i + Ndm].v_full[0] = sparts[i].v[0]; + gparts[i + Ndm].v_full[1] = sparts[i].v[1]; + gparts[i + Ndm].v_full[2] = sparts[i].v[2]; + + gparts[i + Ndm].mass = sparts[i].mass; + + /* Set gpart type */ + gparts[i + Ndm].type = swift_type_star; + + /* Link the particles */ + gparts[i + Ndm].id_or_neg_offset = -i; + sparts[i].gpart = &gparts[i + Ndm]; + } +} + /** * @brief Copy every DM #gpart into the dmparts array. * @@ -632,8 +516,8 @@ void duplicate_hydro_gparts(struct part* const parts, * @param dmparts The array of #gpart containg DM particles to be filled. * @param Ndm The number of DM particles. */ -void collect_dm_gparts(const struct gpart* const gparts, size_t Ntot, - struct gpart* const dmparts, size_t Ndm) { +void io_collect_dm_gparts(const struct gpart* const gparts, size_t Ntot, + struct gpart* const dmparts, size_t Ndm) { size_t count = 0; @@ -644,7 +528,7 @@ void collect_dm_gparts(const struct gpart* const gparts, size_t Ntot, * gparts[i].part); */ /* And collect the DM ones */ - if (gparts[i].id_or_neg_offset > 0) { + if (gparts[i].type == swift_type_dark_matter) { dmparts[count] = gparts[i]; count++; } @@ -655,5 +539,3 @@ void collect_dm_gparts(const struct gpart* const gparts, size_t Ntot, error("Collected the wrong number of dm particles (%zu vs. %zu expected)", count, Ndm); } - -#endif diff --git a/src/common_io.h b/src/common_io.h index 7aedee0f2624dcff916a8398e244009a87109915..577d1664db20196c8d0c48ef5b5e4e960b0e195e 100644 --- a/src/common_io.h +++ b/src/common_io.h @@ -23,17 +23,22 @@ /* Config parameters. */ #include "../config.h" -#if defined(HAVE_HDF5) - +/* Local includes. */ #include "part.h" #include "units.h" +#define FIELD_BUFFER_SIZE 200 +#define PARTICLE_GROUP_BUFFER_SIZE 50 +#define FILENAME_BUFFER_SIZE 150 + +#if defined(HAVE_HDF5) + /** * @brief The different types of data used in the GADGET IC files. * * (This is admittedly a poor substitute to C++ templates...) */ -enum DATA_TYPE { +enum IO_DATA_TYPE { INT, LONG, LONGLONG, @@ -45,65 +50,38 @@ enum DATA_TYPE { CHAR }; -/** - * @brief The different particle types present in a GADGET IC file - * - */ -enum PARTICLE_TYPE { - GAS = 0, - DM = 1, - BOUNDARY = 2, - DUMMY = 3, - STAR = 4, - BH = 5, - NUM_PARTICLE_TYPES -}; - -extern const char* particle_type_names[]; - -#define FILENAME_BUFFER_SIZE 150 -#define FIELD_BUFFER_SIZE 200 -#define PARTICLE_GROUP_BUFFER_SIZE 50 - -hid_t hdf5Type(enum DATA_TYPE type); -size_t sizeOfType(enum DATA_TYPE type); -int isDoublePrecision(enum DATA_TYPE type); - -void collect_dm_gparts(const struct gpart* const gparts, size_t Ntot, - struct gpart* const dmparts, size_t Ndm); -void prepare_dm_gparts(struct gpart* const gparts, size_t Ndm); -void duplicate_hydro_gparts(struct part* const parts, - struct gpart* const gparts, size_t Ngas, - size_t Ndm); +hid_t io_hdf5_type(enum IO_DATA_TYPE type); +size_t io_sizeof_type(enum IO_DATA_TYPE type); +int io_is_double_precision(enum IO_DATA_TYPE type); -void readAttribute(hid_t grp, char* name, enum DATA_TYPE type, void* data); +void io_read_attribute(hid_t grp, char* name, enum IO_DATA_TYPE type, + void* data); -void writeAttribute(hid_t grp, const char* name, enum DATA_TYPE type, - void* data, int num); +void io_write_attribute(hid_t grp, const char* name, enum IO_DATA_TYPE type, + void* data, int num); -void writeAttribute_d(hid_t grp, const char* name, double data); -void writeAttribute_f(hid_t grp, const char* name, float data); -void writeAttribute_i(hid_t grp, const char* name, int data); -void writeAttribute_l(hid_t grp, const char* name, long data); -void writeAttribute_s(hid_t grp, const char* name, const char* str); +void io_write_attribute_d(hid_t grp, const char* name, double data); +void io_write_attribute_f(hid_t grp, const char* name, float data); +void io_write_attribute_i(hid_t grp, const char* name, int data); +void io_write_attribute_l(hid_t grp, const char* name, long data); +void io_write_attribute_s(hid_t grp, const char* name, const char* str); -void createXMFfile(const char* baseName); -FILE* prepareXMFfile(const char* baseName); -void writeXMFoutputheader(FILE* xmfFile, char* hdfFileName, float time); -void writeXMFoutputfooter(FILE* xmfFile, int outputCount, float time); -void writeXMFgroupheader(FILE* xmfFile, char* hdfFileName, size_t N, - enum PARTICLE_TYPE ptype); -void writeXMFgroupfooter(FILE* xmfFile, enum PARTICLE_TYPE ptype); -void writeXMFline(FILE* xmfFile, const char* fileName, - const char* partTypeGroupName, const char* name, size_t N, - int dim, enum DATA_TYPE type); +void io_write_code_description(hid_t h_file); -void writeCodeDescription(hid_t h_file); - -void readUnitSystem(hid_t h_file, struct UnitSystem* us); -void writeUnitSystem(hid_t h_grp, const struct UnitSystem* us, - const char* groupName); +void io_read_unit_system(hid_t h_file, struct unit_system* us); +void io_write_unit_system(hid_t h_grp, const struct unit_system* us, + const char* groupName); #endif /* defined HDF5 */ +void io_collect_dm_gparts(const struct gpart* const gparts, size_t Ntot, + struct gpart* const dmparts, size_t Ndm); +void io_prepare_dm_gparts(struct gpart* const gparts, size_t Ndm); +void io_duplicate_hydro_gparts(struct part* const parts, + struct gpart* const gparts, size_t Ngas, + size_t Ndm); +void io_duplicate_star_gparts(struct spart* const sparts, + struct gpart* const gparts, size_t Nstars, + size_t Ndm); + #endif /* SWIFT_COMMON_IO_H */ diff --git a/src/const.h b/src/const.h index c290a3e73a524e9003cadb63f8bd45e8b3c51dac..88c1a1af1cfc36cc401fdfea0b077f79fcd13bc0 100644 --- a/src/const.h +++ b/src/const.h @@ -41,9 +41,6 @@ /* Self gravity stuff. */ #define const_gravity_multipole_order 1 -#define const_gravity_a_smooth 1.25f -#define const_gravity_r_cut 4.5f -#define const_gravity_eta 0.025f /* Type of gradients to use (GIZMO_SPH only) */ /* If no option is chosen, no gradients are used (first order scheme) */ @@ -55,6 +52,11 @@ #define SLOPE_LIMITER_PER_FACE #define SLOPE_LIMITER_CELL_WIDE +/* Options to control the movement of particles for GIZMO_SPH. */ +/* This option disables particle movement */ +//#define GIZMO_FIX_PARTICLES +//#define GIZMO_TOTAL_ENERGY + /* Source terms */ #define SOURCETERMS_NONE //#define SOURCETERMS_SN_FEEDBACK diff --git a/src/cooling.c b/src/cooling.c index e0208dbb591445d0877ef1e703d6e8cf349ddfd6..9452857117367cfc1d4d3eab262c73eba555d4f5 100644 --- a/src/cooling.c +++ b/src/cooling.c @@ -34,7 +34,7 @@ * @param cooling The cooling properties to initialize */ void cooling_init(const struct swift_params* parameter_file, - const struct UnitSystem* us, + const struct unit_system* us, const struct phys_const* phys_const, struct cooling_function_data* cooling) { diff --git a/src/cooling.h b/src/cooling.h index f0f227a619182af90d48c9416182f7a2132fd08c..3d50658b9b2cf03cadb6138315b3936d3c4ea4ad 100644 --- a/src/cooling.h +++ b/src/cooling.h @@ -42,7 +42,7 @@ /* Common functions */ void cooling_init(const struct swift_params* parameter_file, - const struct UnitSystem* us, + const struct unit_system* us, const struct phys_const* phys_const, struct cooling_function_data* cooling); diff --git a/src/cooling/const_du/cooling.h b/src/cooling/const_du/cooling.h index 30ae644bdecbe795794505f64ba1ed767419d82b..fef44e51d78eecbc55a353e809989318b208db50 100644 --- a/src/cooling/const_du/cooling.h +++ b/src/cooling/const_du/cooling.h @@ -58,7 +58,7 @@ */ __attribute__((always_inline)) INLINE static void cooling_cool_part( const struct phys_const* restrict phys_const, - const struct UnitSystem* restrict us, + const struct unit_system* restrict us, const struct cooling_function_data* restrict cooling, struct part* restrict p, struct xpart* restrict xp, float dt) { @@ -103,7 +103,7 @@ __attribute__((always_inline)) INLINE static void cooling_cool_part( __attribute__((always_inline)) INLINE static float cooling_timestep( const struct cooling_function_data* restrict cooling, const struct phys_const* restrict phys_const, - const struct UnitSystem* restrict us, const struct part* restrict p) { + const struct unit_system* restrict us, const struct part* restrict p) { const float cooling_rate = cooling->cooling_rate; const float internal_energy = hydro_get_internal_energy(p); @@ -153,7 +153,7 @@ __attribute__((always_inline)) INLINE static float cooling_get_radiated_energy( * @param cooling The cooling properties to initialize */ static INLINE void cooling_init_backend( - const struct swift_params* parameter_file, const struct UnitSystem* us, + const struct swift_params* parameter_file, const struct unit_system* us, const struct phys_const* phys_const, struct cooling_function_data* cooling) { diff --git a/src/cooling/const_lambda/cooling.h b/src/cooling/const_lambda/cooling.h index 9fadd51e3c2a3c5462c8476e0aac893e3a2d530d..370d318f255b5c9d6527d288f2e341f37c1e6f40 100644 --- a/src/cooling/const_lambda/cooling.h +++ b/src/cooling/const_lambda/cooling.h @@ -45,7 +45,7 @@ * @param p Pointer to the particle data.. */ __attribute__((always_inline)) INLINE static float cooling_rate( - const struct phys_const* const phys_const, const struct UnitSystem* us, + const struct phys_const* const phys_const, const struct unit_system* us, const struct cooling_function_data* cooling, const struct part* p) { /* Get particle density */ @@ -72,7 +72,7 @@ __attribute__((always_inline)) INLINE static float cooling_rate( */ __attribute__((always_inline)) INLINE static void cooling_cool_part( const struct phys_const* restrict phys_const, - const struct UnitSystem* restrict us, + const struct unit_system* restrict us, const struct cooling_function_data* restrict cooling, struct part* restrict p, struct xpart* restrict xp, float dt) { @@ -112,7 +112,7 @@ __attribute__((always_inline)) INLINE static void cooling_cool_part( __attribute__((always_inline)) INLINE static float cooling_timestep( const struct cooling_function_data* restrict cooling, const struct phys_const* restrict phys_const, - const struct UnitSystem* restrict us, const struct part* restrict p) { + const struct unit_system* restrict us, const struct part* restrict p) { /* Get current internal energy */ const float u = hydro_get_internal_energy(p); @@ -158,7 +158,7 @@ __attribute__((always_inline)) INLINE static float cooling_get_radiated_energy( * @param cooling The cooling properties to initialize */ static INLINE void cooling_init_backend( - const struct swift_params* parameter_file, const struct UnitSystem* us, + const struct swift_params* parameter_file, const struct unit_system* us, const struct phys_const* phys_const, struct cooling_function_data* cooling) { diff --git a/src/cooling/none/cooling.h b/src/cooling/none/cooling.h index 0461100dc11e7ffbb4616766923142442b4ac943..63caf27fc6494c6fdd5a52f84eaa371eb824fd16 100644 --- a/src/cooling/none/cooling.h +++ b/src/cooling/none/cooling.h @@ -49,7 +49,7 @@ */ __attribute__((always_inline)) INLINE static void cooling_cool_part( const struct phys_const* restrict phys_const, - const struct UnitSystem* restrict us, + const struct unit_system* restrict us, const struct cooling_function_data* restrict cooling, struct part* restrict p, struct xpart* restrict xp, float dt) {} @@ -66,7 +66,7 @@ __attribute__((always_inline)) INLINE static void cooling_cool_part( __attribute__((always_inline)) INLINE static float cooling_timestep( const struct cooling_function_data* restrict cooling, const struct phys_const* restrict phys_const, - const struct UnitSystem* restrict us, const struct part* restrict p) { + const struct unit_system* restrict us, const struct part* restrict p) { return FLT_MAX; } @@ -107,7 +107,7 @@ __attribute__((always_inline)) INLINE static float cooling_get_radiated_energy( * @param cooling The cooling properties to initialize */ static INLINE void cooling_init_backend( - const struct swift_params* parameter_file, const struct UnitSystem* us, + const struct swift_params* parameter_file, const struct unit_system* us, const struct phys_const* phys_const, struct cooling_function_data* cooling) {} diff --git a/src/drift.h b/src/drift.h index 86ea4bfdc81157863615a4ca329dfb2705a26cee..d9b79f7f0549d85b6f05e8ce4a394aaa5b2a4d8d 100644 --- a/src/drift.h +++ b/src/drift.h @@ -41,6 +41,18 @@ __attribute__((always_inline)) INLINE static void drift_gpart( struct gpart *restrict gp, float dt, double timeBase, integertime_t ti_old, integertime_t ti_current) { + +#ifdef SWIFT_DEBUG_CHECKS + if (gp->ti_drift != ti_old) + error( + "g-particle has not been drifted to the current time " + "gp->ti_drift=%lld, " + "c->ti_old=%lld, ti_current=%lld", + gp->ti_drift, ti_old, ti_current); + + gp->ti_drift = ti_current; +#endif + /* Drift... */ gp->x[0] += gp->v_full[0] * dt; gp->x[1] += gp->v_full[1] * dt; @@ -69,7 +81,7 @@ __attribute__((always_inline)) INLINE static void drift_part( #ifdef SWIFT_DEBUG_CHECKS if (p->ti_drift != ti_old) error( - "Particle has not been drifted to the current time p->ti_drift=%lld, " + "particle has not been drifted to the current time p->ti_drift=%lld, " "c->ti_old=%lld, ti_current=%lld", p->ti_drift, ti_old, ti_current); @@ -97,4 +109,34 @@ __attribute__((always_inline)) INLINE static void drift_part( } } +/** + * @brief Perform the 'drift' operation on a #spart + * + * @param sp The #spart to drift. + * @param dt The drift time-step + * @param timeBase The minimal allowed time-step size. + * @param ti_old Integer start of time-step + * @param ti_current Integer end of time-step + */ +__attribute__((always_inline)) INLINE static void drift_spart( + struct spart *restrict sp, float dt, double timeBase, integertime_t ti_old, + integertime_t ti_current) { + +#ifdef SWIFT_DEBUG_CHECKS + if (sp->ti_drift != ti_old) + error( + "s-particle has not been drifted to the current time " + "sp->ti_drift=%lld, " + "c->ti_old=%lld, ti_current=%lld", + sp->ti_drift, ti_old, ti_current); + + sp->ti_drift = ti_current; +#endif + + /* Drift... */ + sp->x[0] += sp->v[0] * dt; + sp->x[1] += sp->v[1] * dt; + sp->x[2] += sp->v[2] * dt; +} + #endif /* SWIFT_DRIFT_H */ diff --git a/src/engine.c b/src/engine.c index 631f2c692d59347476e1d3721d99797a437998b1..f4dd5612812fe597101ec623ee16e8ae982a2e30 100644 --- a/src/engine.c +++ b/src/engine.c @@ -55,6 +55,7 @@ #include "cycle.h" #include "debug.h" #include "error.h" +#include "gravity.h" #include "hydro.h" #include "minmax.h" #include "parallel_io.h" @@ -88,7 +89,8 @@ const char *engine_policy_names[16] = {"none", "cosmology_integration", "drift_all", "cooling", - "sourceterms"}; + "sourceterms", + "stars"}; /** The rank of the engine as a global variable (for messages). */ int engine_rank; @@ -131,6 +133,7 @@ void engine_make_hierarchical_tasks(struct engine *e, struct cell *c) { struct scheduler *s = &e->sched; const int is_hydro = (e->policy & engine_policy_hydro); + const int is_self_gravity = (e->policy & engine_policy_self_gravity); const int is_with_cooling = (e->policy & engine_policy_cooling); const int is_with_sourceterms = (e->policy & engine_policy_sourceterms); @@ -156,14 +159,35 @@ void engine_make_hierarchical_tasks(struct engine *e, struct cell *c) { 0, 0, c, NULL, 0); scheduler_addunlock(s, c->kick2, c->timestep); + scheduler_addunlock(s, c->timestep, c->kick1); /* Add the drift task and its dependencies. */ c->drift = scheduler_addtask(s, task_type_drift, task_subtype_none, 0, 0, c, NULL, 0); - scheduler_addunlock(s, c->kick1, c->drift); scheduler_addunlock(s, c->drift, c->init); + if (is_self_gravity) { + + /* Gravity non-neighbouring pm calculations */ + c->grav_long_range = scheduler_addtask( + s, task_type_grav_long_range, task_subtype_none, 0, 0, c, NULL, 0); + + /* Gravity top-level periodic calculation */ + c->grav_top_level = scheduler_addtask( + s, task_type_grav_top_level, task_subtype_none, 0, 0, c, NULL, 0); + + /* Gravity recursive down-pass */ + c->grav_down = scheduler_addtask(s, task_type_grav_down, + task_subtype_none, 0, 0, c, NULL, 0); + + scheduler_addunlock(s, c->init, c->grav_long_range); + scheduler_addunlock(s, c->init, c->grav_top_level); + scheduler_addunlock(s, c->grav_long_range, c->grav_down); + scheduler_addunlock(s, c->grav_top_level, c->grav_down); + scheduler_addunlock(s, c->grav_down, c->kick2); + } + /* Generate the ghost task. */ if (is_hydro) c->ghost = scheduler_addtask(s, task_type_ghost, task_subtype_none, 0, @@ -236,24 +260,30 @@ void engine_redistribute(struct engine *e) { struct part *parts = s->parts; struct xpart *xparts = s->xparts; struct gpart *gparts = s->gparts; + struct spart *sparts = s->sparts; ticks tic = getticks(); /* Allocate temporary arrays to store the counts of particles to be sent and the destination of each particle */ - int *counts, *g_counts; + int *counts, *g_counts, *s_counts; if ((counts = (int *)malloc(sizeof(int) * nr_nodes * nr_nodes)) == NULL) - error("Failed to allocate count temporary buffer."); + error("Failed to allocate counts temporary buffer."); if ((g_counts = (int *)malloc(sizeof(int) * nr_nodes * nr_nodes)) == NULL) - error("Failed to allocate gcount temporary buffer."); + error("Failed to allocate g_gcount temporary buffer."); + if ((s_counts = (int *)malloc(sizeof(int) * nr_nodes * nr_nodes)) == NULL) + error("Failed to allocate s_counts temporary buffer."); bzero(counts, sizeof(int) * nr_nodes * nr_nodes); bzero(g_counts, sizeof(int) * nr_nodes * nr_nodes); + bzero(s_counts, sizeof(int) * nr_nodes * nr_nodes); /* Allocate the destination index arrays. */ - int *dest, *g_dest; + int *dest, *g_dest, *s_dest; if ((dest = (int *)malloc(sizeof(int) * s->nr_parts)) == NULL) error("Failed to allocate dest temporary buffer."); if ((g_dest = (int *)malloc(sizeof(int) * s->nr_gparts)) == NULL) error("Failed to allocate g_dest temporary buffer."); + if ((s_dest = (int *)malloc(sizeof(int) * s->nr_sparts)) == NULL) + error("Failed to allocate s_dest temporary buffer."); /* Get destination of each particle */ for (size_t k = 0; k < s->nr_parts; k++) { @@ -281,7 +311,32 @@ void engine_redistribute(struct engine *e) { } /* Sort the particles according to their cell index. */ - space_parts_sort(s, dest, s->nr_parts, 0, nr_nodes - 1, e->verbose); + if (s->nr_parts > 0) + space_parts_sort(s, dest, s->nr_parts, 0, nr_nodes - 1, e->verbose); + +#ifdef SWIFT_DEBUG_CHECKS + /* Verify that the part have been sorted correctly. */ + for (size_t k = 0; k < s->nr_parts; k++) { + const struct part *p = &s->parts[k]; + + /* New cell index */ + const int new_cid = + cell_getid(s->cdim, p->x[0] * s->iwidth[0], p->x[1] * s->iwidth[1], + p->x[2] * s->iwidth[2]); + + /* New cell of this part */ + const struct cell *c = &s->cells_top[new_cid]; + const int new_node = c->nodeID; + + if (dest[k] != new_node) + error("part's new node index not matching sorted index."); + + if (p->x[0] < c->loc[0] || p->x[0] > c->loc[0] + c->width[0] || + p->x[1] < c->loc[1] || p->x[1] > c->loc[1] + c->width[1] || + p->x[2] < c->loc[2] || p->x[2] > c->loc[2] + c->width[2]) + error("part not sorted into the right top-level cell!"); + } +#endif /* We need to re-link the gpart partners of parts. */ if (s->nr_parts > 0) { @@ -293,7 +348,7 @@ void engine_redistribute(struct engine *e) { /* As the addresses will be invalidated by the communications, we will * instead store the absolute index from the start of the sub-array of * particles to be sent to a given node. - * Recall that gparts without partners have a negative id. + * Recall that gparts without partners have a positive id. * We will restore the pointers on the receiving node later on. */ if (dest[k] != current_dest) { current_dest = dest[k]; @@ -301,7 +356,7 @@ void engine_redistribute(struct engine *e) { } #ifdef SWIFT_DEBUG_CHECKS - if (s->parts[k].gpart->id_or_neg_offset >= 0) + if (s->parts[k].gpart->id_or_neg_offset > 0) error("Trying to link a partnerless gpart !"); #endif @@ -311,6 +366,87 @@ void engine_redistribute(struct engine *e) { } } + /* Get destination of each s-particle */ + for (size_t k = 0; k < s->nr_sparts; k++) { + + /* Periodic boundary conditions */ + for (int j = 0; j < 3; j++) { + if (sparts[k].x[j] < 0.0) + sparts[k].x[j] += dim[j]; + else if (sparts[k].x[j] >= dim[j]) + sparts[k].x[j] -= dim[j]; + } + const int cid = + cell_getid(cdim, sparts[k].x[0] * iwidth[0], sparts[k].x[1] * iwidth[1], + sparts[k].x[2] * iwidth[2]); +#ifdef SWIFT_DEBUG_CHECKS + if (cid < 0 || cid >= s->nr_cells) + error("Bad cell id %i for part %zu at [%.3e,%.3e,%.3e].", cid, k, + sparts[k].x[0], sparts[k].x[1], sparts[k].x[2]); +#endif + + s_dest[k] = cells[cid].nodeID; + + /* The counts array is indexed as count[from * nr_nodes + to]. */ + s_counts[nodeID * nr_nodes + s_dest[k]] += 1; + } + + /* Sort the particles according to their cell index. */ + if (s->nr_sparts > 0) + space_sparts_sort(s, s_dest, s->nr_sparts, 0, nr_nodes - 1, e->verbose); + +#ifdef SWIFT_DEBUG_CHECKS + /* Verify that the spart have been sorted correctly. */ + for (size_t k = 0; k < s->nr_sparts; k++) { + const struct spart *sp = &s->sparts[k]; + + /* New cell index */ + const int new_cid = + cell_getid(s->cdim, sp->x[0] * s->iwidth[0], sp->x[1] * s->iwidth[1], + sp->x[2] * s->iwidth[2]); + + /* New cell of this spart */ + const struct cell *c = &s->cells_top[new_cid]; + const int new_node = c->nodeID; + + if (s_dest[k] != new_node) + error("spart's new node index not matching sorted index."); + + if (sp->x[0] < c->loc[0] || sp->x[0] > c->loc[0] + c->width[0] || + sp->x[1] < c->loc[1] || sp->x[1] > c->loc[1] + c->width[1] || + sp->x[2] < c->loc[2] || sp->x[2] > c->loc[2] + c->width[2]) + error("spart not sorted into the right top-level cell!"); + } +#endif + + /* We need to re-link the gpart partners of sparts. */ + if (s->nr_sparts > 0) { + int current_dest = s_dest[0]; + size_t count_this_dest = 0; + for (size_t k = 0; k < s->nr_sparts; ++k) { + if (s->sparts[k].gpart != NULL) { + + /* As the addresses will be invalidated by the communications, we will + * instead store the absolute index from the start of the sub-array of + * particles to be sent to a given node. + * Recall that gparts without partners have a positive id. + * We will restore the pointers on the receiving node later on. */ + if (s_dest[k] != current_dest) { + current_dest = s_dest[k]; + count_this_dest = 0; + } + +#ifdef SWIFT_DEBUG_CHECKS + if (s->sparts[k].gpart->id_or_neg_offset > 0) + error("Trying to link a partnerless gpart !"); +#endif + + s->sparts[k].gpart->id_or_neg_offset = -count_this_dest; + count_this_dest++; + } + } + } + /* Get destination of each g-particle */ for (size_t k = 0; k < s->nr_gparts; k++) { @@ -337,48 +473,96 @@ void engine_redistribute(struct engine *e) { } /* Sort the gparticles according to their cell index. */ - space_gparts_sort(s, g_dest, s->nr_gparts, 0, nr_nodes - 1, e->verbose); + if (s->nr_gparts > 0) + space_gparts_sort(s, g_dest, s->nr_gparts, 0, nr_nodes - 1, e->verbose); + +#ifdef SWIFT_DEBUG_CHECKS + /* Verify that the gpart have been sorted correctly. */ + for (size_t k = 0; k < s->nr_gparts; k++) { + const struct gpart *gp = &s->gparts[k]; + + /* New cell index */ + const int new_cid = + cell_getid(s->cdim, gp->x[0] * s->iwidth[0], gp->x[1] * s->iwidth[1], + gp->x[2] * s->iwidth[2]); + + /* New cell of this gpart */ + const struct cell *c = &s->cells_top[new_cid]; + const int new_node = c->nodeID; + + if (g_dest[k] != new_node) + error("gpart's new node index not matching sorted index."); + + if (gp->x[0] < c->loc[0] || gp->x[0] > c->loc[0] + c->width[0] || + gp->x[1] < c->loc[1] || gp->x[1] > c->loc[1] + c->width[1] || + gp->x[2] < c->loc[2] || gp->x[2] > c->loc[2] + c->width[2]) + error("gpart not sorted into the right top-level cell!"); + } +#endif /* Get all the counts from all the nodes. */ if (MPI_Allreduce(MPI_IN_PLACE, counts, nr_nodes * nr_nodes, MPI_INT, MPI_SUM, MPI_COMM_WORLD) != MPI_SUCCESS) error("Failed to allreduce particle transfer counts."); + /* Get all the s_counts from all the nodes. */ + if (MPI_Allreduce(MPI_IN_PLACE, g_counts, nr_nodes * nr_nodes, MPI_INT, + MPI_SUM, MPI_COMM_WORLD) != MPI_SUCCESS) + error("Failed to allreduce gparticle transfer counts."); + + /* Get all the g_counts from all the nodes. */ + if (MPI_Allreduce(MPI_IN_PLACE, s_counts, nr_nodes * nr_nodes, MPI_INT, + MPI_SUM, MPI_COMM_WORLD) != MPI_SUCCESS) + error("Failed to allreduce sparticle transfer counts."); + /* Report how many particles will be moved. */ if (e->verbose) { if (e->nodeID == 0) { - size_t total = 0; - size_t unmoved = 0; + size_t total = 0, g_total = 0, s_total = 0; + size_t unmoved = 0, g_unmoved = 0, s_unmoved = 0; for (int p = 0, r = 0; p < nr_nodes; p++) { for (int s = 0; s < nr_nodes; s++) { total += counts[r]; - if (p == s) unmoved += counts[r]; + g_total += g_counts[r]; + s_total += s_counts[r]; + if (p == s) { + unmoved += counts[r]; + g_unmoved += g_counts[r]; + s_unmoved += s_counts[r]; + } r++; } } - message("%ld of %ld (%.2f%%) of particles moved", total - unmoved, total, - 100.0 * (double)(total - unmoved) / (double)total); + if (total > 0) + message("%ld of %ld (%.2f%%) of particles moved", total - unmoved, + total, 100.0 * (double)(total - unmoved) / (double)total); + if (g_total > 0) + message("%ld of %ld (%.2f%%) of g-particles moved", g_total - g_unmoved, + g_total, + 100.0 * (double)(g_total - g_unmoved) / (double)g_total); + if (s_total > 0) + message("%ld of %ld (%.2f%%) of s-particles moved", s_total - s_unmoved, + s_total, + 100.0 * (double)(s_total - s_unmoved) / (double)s_total); } } - /* Get all the g_counts from all the nodes. */ - if (MPI_Allreduce(MPI_IN_PLACE, g_counts, nr_nodes * nr_nodes, MPI_INT, - MPI_SUM, MPI_COMM_WORLD) != MPI_SUCCESS) - error("Failed to allreduce gparticle transfer counts."); - - /* Each node knows how many parts and gparts will be transferred to every - other node. We can start preparing to receive data */ + /* Each node knows how many parts, sparts and gparts will be transferred + to every other node. We can start preparing to receive data */ /* Get the new number of parts and gparts for this node */ - size_t nr_parts = 0, nr_gparts = 0; + size_t nr_parts = 0, nr_gparts = 0, nr_sparts = 0; for (int k = 0; k < nr_nodes; k++) nr_parts += counts[k * nr_nodes + nodeID]; for (int k = 0; k < nr_nodes; k++) nr_gparts += g_counts[k * nr_nodes + nodeID]; + for (int k = 0; k < nr_nodes; k++) + nr_sparts += s_counts[k * nr_nodes + nodeID]; /* Allocate the new arrays with some extra margin */ struct part *parts_new = NULL; struct xpart *xparts_new = NULL; struct gpart *gparts_new = NULL; + struct spart *sparts_new = NULL; if (posix_memalign((void **)&parts_new, part_align, sizeof(struct part) * nr_parts * engine_redistribute_alloc_margin) != 0) @@ -391,17 +575,22 @@ void engine_redistribute(struct engine *e) { sizeof(struct gpart) * nr_gparts * engine_redistribute_alloc_margin) != 0) error("Failed to allocate new gpart data."); + if (posix_memalign((void **)&sparts_new, spart_align, + sizeof(struct spart) * nr_sparts * + engine_redistribute_alloc_margin) != 0) + error("Failed to allocate new spart data."); /* Prepare MPI requests for the asynchronous communications */ MPI_Request *reqs; - if ((reqs = (MPI_Request *)malloc(sizeof(MPI_Request) * 6 * nr_nodes)) == + if ((reqs = (MPI_Request *)malloc(sizeof(MPI_Request) * 8 * nr_nodes)) == NULL) error("Failed to allocate MPI request list."); - for (int k = 0; k < 6 * nr_nodes; k++) reqs[k] = MPI_REQUEST_NULL; + for (int k = 0; k < 8 * nr_nodes; k++) reqs[k] = MPI_REQUEST_NULL; /* Emit the sends and recvs for the particle and gparticle data. */ size_t offset_send = 0, offset_recv = 0; size_t g_offset_send = 0, g_offset_recv = 0; + size_t s_offset_send = 0, s_offset_recv = 0; for (int k = 0; k < nr_nodes; k++) { /* Indices in the count arrays of the node of interest */ @@ -425,12 +614,12 @@ void engine_redistribute(struct engine *e) { /* Else, emit some communications */ } else { if (MPI_Isend(&s->parts[offset_send], counts[ind_send], part_mpi_type, - k, 3 * ind_send + 0, MPI_COMM_WORLD, - &reqs[6 * k]) != MPI_SUCCESS) + k, 4 * ind_send + 0, MPI_COMM_WORLD, + &reqs[8 * k + 0]) != MPI_SUCCESS) error("Failed to isend parts to node %i.", k); if (MPI_Isend(&s->xparts[offset_send], counts[ind_send], xpart_mpi_type, - k, 3 * ind_send + 1, MPI_COMM_WORLD, - &reqs[6 * k + 1]) != MPI_SUCCESS) + k, 4 * ind_send + 1, MPI_COMM_WORLD, + &reqs[8 * k + 1]) != MPI_SUCCESS) error("Failed to isend xparts to node %i.", k); offset_send += counts[ind_send]; } @@ -451,24 +640,46 @@ void engine_redistribute(struct engine *e) { /* Else, emit some communications */ } else { if (MPI_Isend(&s->gparts[g_offset_send], g_counts[ind_send], - gpart_mpi_type, k, 3 * ind_send + 2, MPI_COMM_WORLD, - &reqs[6 * k + 2]) != MPI_SUCCESS) + gpart_mpi_type, k, 4 * ind_send + 2, MPI_COMM_WORLD, + &reqs[8 * k + 2]) != MPI_SUCCESS) error("Failed to isend gparts to node %i.", k); g_offset_send += g_counts[ind_send]; } } + /* Are we sending any spart ? */ + if (s_counts[ind_send] > 0) { + + /* message("Sending %d spart to node %d", s_counts[ind_send], k); */ + + /* If the send is to the same node, just copy */ + if (k == nodeID) { + memcpy(&sparts_new[s_offset_recv], &s->sparts[s_offset_send], + sizeof(struct spart) * s_counts[ind_recv]); + s_offset_send += s_counts[ind_send]; + s_offset_recv += s_counts[ind_recv]; + + /* Else, emit some communications */ + } else { + if (MPI_Isend(&s->sparts[s_offset_send], s_counts[ind_send], + spart_mpi_type, k, 4 * ind_send + 3, MPI_COMM_WORLD, + &reqs[8 * k + 3]) != MPI_SUCCESS) + error("Failed to isend gparts to node %i.", k); + s_offset_send += s_counts[ind_send]; + } + } + /* Now emit the corresponding Irecv() */ /* Are we receiving any part/xpart from this node ? */ if (k != nodeID && counts[ind_recv] > 0) { if (MPI_Irecv(&parts_new[offset_recv], counts[ind_recv], part_mpi_type, k, - 3 * ind_recv + 0, MPI_COMM_WORLD, - &reqs[6 * k + 3]) != MPI_SUCCESS) + 4 * ind_recv + 0, MPI_COMM_WORLD, + &reqs[8 * k + 4]) != MPI_SUCCESS) error("Failed to emit irecv of parts from node %i.", k); if (MPI_Irecv(&xparts_new[offset_recv], counts[ind_recv], xpart_mpi_type, - k, 3 * ind_recv + 1, MPI_COMM_WORLD, - &reqs[6 * k + 4]) != MPI_SUCCESS) + k, 4 * ind_recv + 1, MPI_COMM_WORLD, + &reqs[8 * k + 5]) != MPI_SUCCESS) error("Failed to emit irecv of xparts from node %i.", k); offset_recv += counts[ind_recv]; } @@ -476,18 +687,27 @@ void engine_redistribute(struct engine *e) { /* Are we receiving any gpart from this node ? */ if (k != nodeID && g_counts[ind_recv] > 0) { if (MPI_Irecv(&gparts_new[g_offset_recv], g_counts[ind_recv], - gpart_mpi_type, k, 3 * ind_recv + 2, MPI_COMM_WORLD, - &reqs[6 * k + 5]) != MPI_SUCCESS) + gpart_mpi_type, k, 4 * ind_recv + 2, MPI_COMM_WORLD, + &reqs[8 * k + 6]) != MPI_SUCCESS) error("Failed to emit irecv of gparts from node %i.", k); g_offset_recv += g_counts[ind_recv]; } + + /* Are we receiving any spart from this node ? */ + if (k != nodeID && s_counts[ind_recv] > 0) { + if (MPI_Irecv(&sparts_new[s_offset_recv], s_counts[ind_recv], + spart_mpi_type, k, 4 * ind_recv + 3, MPI_COMM_WORLD, + &reqs[8 * k + 7]) != MPI_SUCCESS) + error("Failed to emit irecv of sparts from node %i.", k); + s_offset_recv += s_counts[ind_recv]; + } } /* Wait for all the sends and recvs to tumble in. */ - MPI_Status stats[6 * nr_nodes]; + MPI_Status stats[8 * nr_nodes]; int res; - if ((res = MPI_Waitall(6 * nr_nodes, reqs, stats)) != MPI_SUCCESS) { - for (int k = 0; k < 6 * nr_nodes; k++) { + if ((res = MPI_Waitall(8 * nr_nodes, reqs, stats)) != MPI_SUCCESS) { + for (int k = 0; k < 8 * nr_nodes; k++) { char buff[MPI_MAX_ERROR_STRING]; MPI_Error_string(stats[k].MPI_ERROR, buff, &res); message("request %i has error '%s'.", k, buff); @@ -495,19 +715,23 @@ void engine_redistribute(struct engine *e) { error("Failed during waitall for part data."); } - /* We now need to restore the part<->gpart links */ - size_t offset_parts = 0, offset_gparts = 0; + /* All particles have now arrived. Time for some final operations on the + stuff we just received */ + + /* Restore the part<->gpart and spart<->gpart links */ + size_t offset_parts = 0, offset_sparts = 0, offset_gparts = 0; for (int node = 0; node < nr_nodes; ++node) { const int ind_recv = node * nr_nodes + nodeID; const size_t count_parts = counts[ind_recv]; const size_t count_gparts = g_counts[ind_recv]; + const size_t count_sparts = s_counts[ind_recv]; /* Loop over the gparts received from that node */ for (size_t k = offset_gparts; k < offset_gparts + count_gparts; ++k) { - /* Does this gpart have a partner ? */ - if (gparts_new[k].id_or_neg_offset <= 0) { + /* Does this gpart have a gas partner ? */ + if (gparts_new[k].type == swift_type_gas) { const ptrdiff_t partner_index = offset_parts - gparts_new[k].id_or_neg_offset; @@ -516,10 +740,22 @@ void engine_redistribute(struct engine *e) { gparts_new[k].id_or_neg_offset = -partner_index; parts_new[partner_index].gpart = &gparts_new[k]; } + + /* Does this gpart have a star partner ? */ + if (gparts_new[k].type == swift_type_star) { + + const ptrdiff_t partner_index = + offset_sparts - gparts_new[k].id_or_neg_offset; + + /* Re-link */ + gparts_new[k].id_or_neg_offset = -partner_index; + sparts_new[partner_index].gpart = &gparts_new[k]; + } } offset_parts += count_parts; offset_gparts += count_gparts; + offset_sparts += count_sparts; } #ifdef SWIFT_DEBUG_CHECKS @@ -532,41 +768,43 @@ void engine_redistribute(struct engine *e) { error("Received particle (%zu) that does not belong here (nodeID=%i).", k, cells[cid].nodeID); } - - /* Verify that the links are correct */ - for (size_t k = 0; k < nr_gparts; ++k) { - - if (gparts_new[k].id_or_neg_offset <= 0) { - - struct part *part = &parts_new[-gparts_new[k].id_or_neg_offset]; - - if (part->gpart != &gparts_new[k]) error("Linking problem !"); - - if (gparts_new[k].x[0] != part->x[0] || - gparts_new[k].x[1] != part->x[1] || gparts_new[k].x[2] != part->x[2]) - error("Linked particles are not at the same position !"); - } + for (size_t k = 0; k < nr_gparts; k++) { + const int cid = cell_getid(cdim, gparts_new[k].x[0] * iwidth[0], + gparts_new[k].x[1] * iwidth[1], + gparts_new[k].x[2] * iwidth[2]); + if (cells[cid].nodeID != nodeID) + error("Received g-particle (%zu) that does not belong here (nodeID=%i).", + k, cells[cid].nodeID); } - for (size_t k = 0; k < nr_parts; ++k) { - - if (parts_new[k].gpart != NULL && - parts_new[k].gpart->id_or_neg_offset != -(ptrdiff_t)k) { - error("Linking problem !"); - } + for (size_t k = 0; k < nr_sparts; k++) { + const int cid = cell_getid(cdim, sparts_new[k].x[0] * iwidth[0], + sparts_new[k].x[1] * iwidth[1], + sparts_new[k].x[2] * iwidth[2]); + if (cells[cid].nodeID != nodeID) + error("Received s-particle (%zu) that does not belong here (nodeID=%i).", + k, cells[cid].nodeID); } + + /* Verify that the links are correct */ + part_verify_links(parts_new, gparts_new, sparts_new, nr_parts, nr_gparts, + nr_sparts, e->verbose); #endif /* Set the new part data, free the old. */ free(parts); free(xparts); free(gparts); + free(sparts); s->parts = parts_new; s->xparts = xparts_new; s->gparts = gparts_new; + s->sparts = sparts_new; s->nr_parts = nr_parts; s->nr_gparts = nr_gparts; + s->nr_sparts = nr_sparts; s->size_parts = engine_redistribute_alloc_margin * nr_parts; s->size_gparts = engine_redistribute_alloc_margin * nr_gparts; + s->size_sparts = engine_redistribute_alloc_margin * nr_sparts; /* Clean up the temporary stuff. */ free(reqs); @@ -578,8 +816,8 @@ void engine_redistribute(struct engine *e) { int my_cells = 0; for (int k = 0; k < nr_cells; k++) if (cells[k].nodeID == nodeID) my_cells += 1; - message("node %i now has %zu parts and %zu gparts in %i cells.", nodeID, - nr_parts, nr_gparts, my_cells); + message("node %i now has %zu parts, %zu sparts and %zu gparts in %i cells.", + nodeID, nr_parts, nr_sparts, nr_gparts, my_cells); } if (e->verbose) @@ -602,20 +840,23 @@ void engine_repartition(struct engine *e) { ticks tic = getticks(); #ifdef SWIFT_DEBUG_CHECKS + /* Be verbose about this. */ + if (e->nodeID == 0 || e->verbose) message("repartitioning space"); + fflush(stdout); + /* Check that all cells have been drifted to the current time */ - space_check_drift_point(e->s, e->ti_current); + space_check_drift_point(e->s, e->ti_old); #endif /* Clear the repartition flag. */ - enum repartition_type reparttype = e->forcerepart; - e->forcerepart = REPART_NONE; + e->forcerepart = 0; /* Nothing to do if only using a single node. Also avoids METIS * bug that doesn't handle this case well. */ if (e->nr_nodes == 1) return; /* Do the repartitioning. */ - partition_repartition(reparttype, e->nodeID, e->nr_nodes, e->s, + partition_repartition(e->reparttype, e->nodeID, e->nr_nodes, e->s, e->sched.tasks, e->sched.nr_tasks); /* Now comes the tricky part: Exchange particles between all nodes. @@ -653,7 +894,7 @@ void engine_addtasks_grav(struct engine *e, struct cell *c, struct task *up, struct task *down) { /* Link the tasks to this cell. */ - c->grav_up = up; + // c->grav_up = up; c->grav_down = down; /* Recurse? */ @@ -929,11 +1170,12 @@ void engine_exchange_cells(struct engine *e) { /* Count the number of particles we need to import and re-allocate the buffer if needed. */ - size_t count_parts_in = 0, count_gparts_in = 0; + size_t count_parts_in = 0, count_gparts_in = 0, count_sparts_in = 0; for (int k = 0; k < nr_proxies; k++) for (int j = 0; j < e->proxies[k].nr_cells_in; j++) { count_parts_in += e->proxies[k].cells_in[j]->count; count_gparts_in += e->proxies[k].cells_in[j]->gcount; + count_sparts_in += e->proxies[k].cells_in[j]->scount; } if (count_parts_in > s->size_parts_foreign) { if (s->parts_foreign != NULL) free(s->parts_foreign); @@ -949,20 +1191,31 @@ void engine_exchange_cells(struct engine *e) { sizeof(struct gpart) * s->size_gparts_foreign) != 0) error("Failed to allocate foreign gpart data."); } + if (count_sparts_in > s->size_sparts_foreign) { + if (s->sparts_foreign != NULL) free(s->sparts_foreign); + s->size_sparts_foreign = 1.1 * count_sparts_in; + if (posix_memalign((void **)&s->sparts_foreign, spart_align, + sizeof(struct spart) * s->size_sparts_foreign) != 0) + error("Failed to allocate foreign spart data."); + } /* Unpack the cells and link to the particle data. */ struct part *parts = s->parts_foreign; struct gpart *gparts = s->gparts_foreign; + struct spart *sparts = s->sparts_foreign; for (int k = 0; k < nr_proxies; k++) { for (int j = 0; j < e->proxies[k].nr_cells_in; j++) { cell_link_parts(e->proxies[k].cells_in[j], parts); cell_link_gparts(e->proxies[k].cells_in[j], gparts); + cell_link_sparts(e->proxies[k].cells_in[j], sparts); parts = &parts[e->proxies[k].cells_in[j]->count]; gparts = &gparts[e->proxies[k].cells_in[j]->gcount]; + sparts = &sparts[e->proxies[k].cells_in[j]->scount]; } } s->nr_parts_foreign = parts - s->parts_foreign; s->nr_gparts_foreign = gparts - s->gparts_foreign; + s->nr_sparts_foreign = sparts - s->sparts_foreign; /* Free the pcell buffer. */ free(pcells); @@ -977,7 +1230,7 @@ void engine_exchange_cells(struct engine *e) { } /** - * @brief Exchange straying parts with other nodes. + * @brief Exchange straying particles with other nodes. * * @param e The #engine. * @param offset_parts The index in the parts array as of which the foreign @@ -990,13 +1243,20 @@ void engine_exchange_cells(struct engine *e) { * @param ind_gpart The foreign #cell ID of each gpart. * @param Ngpart The number of stray gparts, contains the number of gparts * received on return. + * @param offset_sparts The index in the sparts array as of which the foreign + * parts reside. + * @param ind_spart The foreign #cell ID of each spart. + * @param Nspart The number of stray sparts, contains the number of sparts + * received on return. * * Note that this function does not mess-up the linkage between parts and * gparts, i.e. the received particles have correct linkeage. */ void engine_exchange_strays(struct engine *e, size_t offset_parts, int *ind_part, size_t *Npart, size_t offset_gparts, - int *ind_gpart, size_t *Ngpart) { + int *ind_gpart, size_t *Ngpart, + size_t offset_sparts, int *ind_spart, + size_t *Nspart) { #ifdef WITH_MPI @@ -1007,9 +1267,10 @@ void engine_exchange_strays(struct engine *e, size_t offset_parts, for (int k = 0; k < e->nr_proxies; k++) { e->proxies[k].nr_parts_out = 0; e->proxies[k].nr_gparts_out = 0; + e->proxies[k].nr_sparts_out = 0; } - /* Put the parts and gparts into the corresponding proxies. */ + /* Put the parts into the corresponding proxies. */ for (size_t k = 0; k < *Npart; k++) { /* Get the target node and proxy ID. */ const int node_id = e->s->cells_top[ind_part[k]].nodeID; @@ -1035,6 +1296,32 @@ void engine_exchange_strays(struct engine *e, size_t offset_parts, proxy_parts_load(&e->proxies[pid], &s->parts[offset_parts + k], &s->xparts[offset_parts + k], 1); } + + /* Put the sparts into the corresponding proxies. */ + for (size_t k = 0; k < *Nspart; k++) { + const int node_id = e->s->cells_top[ind_spart[k]].nodeID; + if (node_id < 0 || node_id >= e->nr_nodes) + error("Bad node ID %i.", node_id); + const int pid = e->proxy_ind[node_id]; + if (pid < 0) + error( + "Do not have a proxy for the requested nodeID %i for part with " + "id=%lld, x=[%e,%e,%e].", + node_id, s->sparts[offset_sparts + k].id, + s->sparts[offset_sparts + k].x[0], s->sparts[offset_sparts + k].x[1], + s->sparts[offset_sparts + k].x[2]); + + /* Re-link the associated gpart with the buffer offset of the spart. */ + if (s->sparts[offset_sparts + k].gpart != NULL) { + s->sparts[offset_sparts + k].gpart->id_or_neg_offset = + -e->proxies[pid].nr_sparts_out; + } + + /* Load the spart into the proxy */ + proxy_sparts_load(&e->proxies[pid], &s->sparts[offset_sparts + k], 1); + } + + /* Put the gparts into the corresponding proxies. */ for (size_t k = 0; k < *Ngpart; k++) { const int node_id = e->s->cells_top[ind_gpart[k]].nodeID; if (node_id < 0 || node_id >= e->nr_nodes) @@ -1044,15 +1331,17 @@ void engine_exchange_strays(struct engine *e, size_t offset_parts, error( "Do not have a proxy for the requested nodeID %i for part with " "id=%lli, x=[%e,%e,%e].", - node_id, s->gparts[offset_parts + k].id_or_neg_offset, - s->gparts[offset_gparts + k].x[0], s->gparts[offset_parts + k].x[1], + node_id, s->gparts[offset_gparts + k].id_or_neg_offset, + s->gparts[offset_gparts + k].x[0], s->gparts[offset_gparts + k].x[1], s->gparts[offset_gparts + k].x[2]); + + /* Load the gpart into the proxy */ proxy_gparts_load(&e->proxies[pid], &s->gparts[offset_gparts + k], 1); } /* Launch the proxies. */ - MPI_Request reqs_in[3 * engine_maxproxies]; - MPI_Request reqs_out[3 * engine_maxproxies]; + MPI_Request reqs_in[4 * engine_maxproxies]; + MPI_Request reqs_out[4 * engine_maxproxies]; for (int k = 0; k < e->nr_proxies; k++) { proxy_parts_exch1(&e->proxies[k]); reqs_in[k] = e->proxies[k].req_parts_count_in; @@ -1078,14 +1367,19 @@ void engine_exchange_strays(struct engine *e, size_t offset_parts, enough space to accommodate them. */ int count_parts_in = 0; int count_gparts_in = 0; + int count_sparts_in = 0; for (int k = 0; k < e->nr_proxies; k++) { count_parts_in += e->proxies[k].nr_parts_in; count_gparts_in += e->proxies[k].nr_gparts_in; + count_sparts_in += e->proxies[k].nr_sparts_in; } if (e->verbose) { - message("sent out %zu/%zu parts/gparts, got %i/%i back.", *Npart, *Ngpart, - count_parts_in, count_gparts_in); + message("sent out %zu/%zu/%zu parts/gparts/sparts, got %i/%i/%i back.", + *Npart, *Ngpart, *Nspart, count_parts_in, count_gparts_in, + count_sparts_in); } + + /* Reallocate the particle arrays if necessary */ if (offset_parts + count_parts_in > s->size_parts) { message("re-allocating parts array."); s->size_parts = (offset_parts + count_parts_in) * engine_parts_size_grow; @@ -1108,6 +1402,22 @@ void engine_exchange_strays(struct engine *e, size_t offset_parts, } } } + if (offset_sparts + count_sparts_in > s->size_sparts) { + message("re-allocating sparts array."); + s->size_sparts = (offset_sparts + count_sparts_in) * engine_parts_size_grow; + struct spart *sparts_new = NULL; + if (posix_memalign((void **)&sparts_new, spart_align, + sizeof(struct spart) * s->size_sparts) != 0) + error("Failed to allocate new spart data."); + memcpy(sparts_new, s->sparts, sizeof(struct spart) * offset_sparts); + free(s->sparts); + s->sparts = sparts_new; + for (size_t k = 0; k < offset_sparts; k++) { + if (s->sparts[k].gpart != NULL) { + s->sparts[k].gpart->id_or_neg_offset = -k; + } + } + } if (offset_gparts + count_gparts_in > s->size_gparts) { message("re-allocating gparts array."); s->size_gparts = (offset_gparts + count_gparts_in) * engine_parts_size_grow; @@ -1118,9 +1428,12 @@ void engine_exchange_strays(struct engine *e, size_t offset_parts, memcpy(gparts_new, s->gparts, sizeof(struct gpart) * offset_gparts); free(s->gparts); s->gparts = gparts_new; + for (size_t k = 0; k < offset_gparts; k++) { - if (s->gparts[k].id_or_neg_offset < 0) { + if (s->gparts[k].type == swift_type_gas) { s->parts[-s->gparts[k].id_or_neg_offset].gpart = &s->gparts[k]; + } else if (s->gparts[k].type == swift_type_star) { + s->sparts[-s->gparts[k].id_or_neg_offset].gpart = &s->gparts[k]; } } } @@ -1129,39 +1442,52 @@ void engine_exchange_strays(struct engine *e, size_t offset_parts, int nr_in = 0, nr_out = 0; for (int k = 0; k < e->nr_proxies; k++) { if (e->proxies[k].nr_parts_in > 0) { - reqs_in[3 * k] = e->proxies[k].req_parts_in; - reqs_in[3 * k + 1] = e->proxies[k].req_xparts_in; + reqs_in[4 * k] = e->proxies[k].req_parts_in; + reqs_in[4 * k + 1] = e->proxies[k].req_xparts_in; nr_in += 2; } else { - reqs_in[3 * k] = reqs_in[3 * k + 1] = MPI_REQUEST_NULL; + reqs_in[4 * k] = reqs_in[4 * k + 1] = MPI_REQUEST_NULL; } if (e->proxies[k].nr_gparts_in > 0) { - reqs_in[3 * k + 2] = e->proxies[k].req_gparts_in; + reqs_in[4 * k + 2] = e->proxies[k].req_gparts_in; nr_in += 1; } else { - reqs_in[3 * k + 2] = MPI_REQUEST_NULL; + reqs_in[4 * k + 2] = MPI_REQUEST_NULL; } + if (e->proxies[k].nr_sparts_in > 0) { + reqs_in[4 * k + 3] = e->proxies[k].req_sparts_in; + nr_in += 1; + } else { + reqs_in[4 * k + 3] = MPI_REQUEST_NULL; + } + if (e->proxies[k].nr_parts_out > 0) { - reqs_out[3 * k] = e->proxies[k].req_parts_out; - reqs_out[3 * k + 1] = e->proxies[k].req_xparts_out; + reqs_out[4 * k] = e->proxies[k].req_parts_out; + reqs_out[4 * k + 1] = e->proxies[k].req_xparts_out; nr_out += 2; } else { - reqs_out[3 * k] = reqs_out[3 * k + 1] = MPI_REQUEST_NULL; + reqs_out[4 * k] = reqs_out[4 * k + 1] = MPI_REQUEST_NULL; } if (e->proxies[k].nr_gparts_out > 0) { - reqs_out[3 * k + 2] = e->proxies[k].req_gparts_out; + reqs_out[4 * k + 2] = e->proxies[k].req_gparts_out; + nr_out += 1; + } else { + reqs_out[4 * k + 2] = MPI_REQUEST_NULL; + } + if (e->proxies[k].nr_sparts_out > 0) { + reqs_out[4 * k + 3] = e->proxies[k].req_sparts_out; nr_out += 1; } else { - reqs_out[3 * k + 2] = MPI_REQUEST_NULL; + reqs_out[4 * k + 3] = MPI_REQUEST_NULL; } } /* Wait for each part array to come in and collect the new parts from the proxies. */ - int count_parts = 0, count_gparts = 0; + int count_parts = 0, count_gparts = 0, count_sparts = 0; for (int k = 0; k < nr_in; k++) { int err, pid; - if ((err = MPI_Waitany(3 * e->nr_proxies, reqs_in, &pid, + if ((err = MPI_Waitany(4 * e->nr_proxies, reqs_in, &pid, MPI_STATUS_IGNORE)) != MPI_SUCCESS) { char buff[MPI_MAX_ERROR_STRING]; int res; @@ -1169,21 +1495,24 @@ void engine_exchange_strays(struct engine *e, size_t offset_parts, error("MPI_Waitany failed (%s).", buff); } if (pid == MPI_UNDEFINED) break; - // message( "request from proxy %i has arrived." , pid / 3 ); - pid = 3 * (pid / 3); + // message( "request from proxy %i has arrived." , pid / 4 ); + pid = 4 * (pid / 4); /* If all the requests for a given proxy have arrived... */ if (reqs_in[pid + 0] == MPI_REQUEST_NULL && reqs_in[pid + 1] == MPI_REQUEST_NULL && - reqs_in[pid + 2] == MPI_REQUEST_NULL) { + reqs_in[pid + 2] == MPI_REQUEST_NULL && + reqs_in[pid + 3] == MPI_REQUEST_NULL) { /* Copy the particle data to the part/xpart/gpart arrays. */ - struct proxy *prox = &e->proxies[pid / 3]; + struct proxy *prox = &e->proxies[pid / 4]; memcpy(&s->parts[offset_parts + count_parts], prox->parts_in, sizeof(struct part) * prox->nr_parts_in); memcpy(&s->xparts[offset_parts + count_parts], prox->xparts_in, sizeof(struct xpart) * prox->nr_parts_in); memcpy(&s->gparts[offset_gparts + count_gparts], prox->gparts_in, sizeof(struct gpart) * prox->nr_gparts_in); + memcpy(&s->sparts[offset_sparts + count_sparts], prox->sparts_in, + sizeof(struct spart) * prox->nr_sparts_in); /* for (int k = offset; k < offset + count; k++) message( "received particle %lli, x=[%.3e %.3e %.3e], h=%.3e, from node %i.", @@ -1193,23 +1522,30 @@ void engine_exchange_strays(struct engine *e, size_t offset_parts, /* Re-link the gparts. */ for (int kk = 0; kk < prox->nr_gparts_in; kk++) { struct gpart *gp = &s->gparts[offset_gparts + count_gparts + kk]; - if (gp->id_or_neg_offset <= 0) { + + if (gp->type == swift_type_gas) { struct part *p = - &s->parts[offset_gparts + count_parts - gp->id_or_neg_offset]; + &s->parts[offset_parts + count_parts - gp->id_or_neg_offset]; gp->id_or_neg_offset = s->parts - p; p->gpart = gp; + } else if (gp->type == swift_type_star) { + struct spart *sp = + &s->sparts[offset_sparts + count_sparts - gp->id_or_neg_offset]; + gp->id_or_neg_offset = s->sparts - sp; + sp->gpart = gp; } } /* Advance the counters. */ count_parts += prox->nr_parts_in; count_gparts += prox->nr_gparts_in; + count_sparts += prox->nr_sparts_in; } } /* Wait for all the sends to have finished too. */ if (nr_out > 0) - if (MPI_Waitall(3 * e->nr_proxies, reqs_out, MPI_STATUSES_IGNORE) != + if (MPI_Waitall(4 * e->nr_proxies, reqs_out, MPI_STATUSES_IGNORE) != MPI_SUCCESS) error("MPI_Waitall on sends failed."); @@ -1220,6 +1556,7 @@ void engine_exchange_strays(struct engine *e, size_t offset_parts, /* Return the number of harvested parts. */ *Npart = count_parts; *Ngpart = count_gparts; + *Nspart = count_sparts; #else error("SWIFT was not compiled with MPI support."); @@ -1236,7 +1573,7 @@ void engine_exchange_strays(struct engine *e, size_t offset_parts, * * @param e The #engine. */ -void engine_make_gravity_tasks(struct engine *e) { +void engine_make_self_gravity_tasks(struct engine *e) { struct space *s = e->s; struct scheduler *sched = &e->sched; @@ -1258,10 +1595,6 @@ void engine_make_gravity_tasks(struct engine *e) { scheduler_addtask(sched, task_type_self, task_subtype_grav, 0, 0, ci, NULL, 0); - /* Let's also build a task for all the non-neighbouring pm calculations */ - scheduler_addtask(sched, task_type_grav_mm, task_subtype_none, 0, 0, ci, - NULL, 0); - for (int cjd = cid + 1; cjd < nr_cells; ++cjd) { struct cell *cj = &cells[cjd]; @@ -1470,16 +1803,12 @@ void engine_count_and_link_tasks(struct engine *e) { * @param gravity The gravity task to link. * @param c The cell. */ -static inline void engine_make_gravity_dependencies(struct scheduler *sched, - struct task *gravity, - struct cell *c) { +static inline void engine_make_self_gravity_dependencies( + struct scheduler *sched, struct task *gravity, struct cell *c) { - /* init --> gravity --> kick */ + /* init --> gravity --> grav_down --> kick */ scheduler_addunlock(sched, c->super->init, gravity); - scheduler_addunlock(sched, gravity, c->super->kick2); - - /* grav_up --> gravity ( --> kick) */ - scheduler_addunlock(sched, c->super->grav_up, gravity); + scheduler_addunlock(sched, gravity, c->super->grav_down); } /** @@ -1509,47 +1838,19 @@ void engine_link_gravity_tasks(struct engine *e) { const int nodeID = e->nodeID; const int nr_tasks = sched->nr_tasks; - /* Add one task gathering all the multipoles */ - struct task *gather = scheduler_addtask( - sched, task_type_grav_gather_m, task_subtype_none, 0, 0, NULL, NULL, 0); - - /* And one task performing the FFT */ - struct task *fft = scheduler_addtask(sched, task_type_grav_fft, - task_subtype_none, 0, 0, NULL, NULL, 0); - - scheduler_addunlock(sched, gather, fft); - for (int k = 0; k < nr_tasks; k++) { /* Get a pointer to the task. */ struct task *t = &sched->tasks[k]; - /* Multipole construction */ - if (t->type == task_type_grav_up) { - scheduler_addunlock(sched, t, gather); - } - - /* Long-range interaction */ - if (t->type == task_type_grav_mm) { - - /* Gather the multipoles --> mm interaction --> kick */ - scheduler_addunlock(sched, gather, t); - scheduler_addunlock(sched, t, t->ci->super->kick2); - - /* init --> mm interaction */ - scheduler_addunlock(sched, t->ci->super->init, t); - } - /* Self-interaction for self-gravity? */ if (t->type == task_type_self && t->subtype == task_subtype_grav) { - engine_make_gravity_dependencies(sched, t, t->ci); - + engine_make_self_gravity_dependencies(sched, t, t->ci); } /* Self-interaction for external gravity ? */ - else if (t->type == task_type_self && - t->subtype == task_subtype_external_grav) { + if (t->type == task_type_self && t->subtype == task_subtype_external_grav) { engine_make_external_gravity_dependencies(sched, t, t->ci); @@ -1560,12 +1861,12 @@ void engine_link_gravity_tasks(struct engine *e) { if (t->ci->nodeID == nodeID) { - engine_make_gravity_dependencies(sched, t, t->ci); + engine_make_self_gravity_dependencies(sched, t, t->ci); } if (t->cj->nodeID == nodeID && t->ci->super != t->cj->super) { - engine_make_gravity_dependencies(sched, t, t->cj); + engine_make_self_gravity_dependencies(sched, t, t->cj); } } @@ -1574,7 +1875,7 @@ void engine_link_gravity_tasks(struct engine *e) { else if (t->type == task_type_sub_self && t->subtype == task_subtype_grav) { if (t->ci->nodeID == nodeID) { - engine_make_gravity_dependencies(sched, t, t->ci); + engine_make_self_gravity_dependencies(sched, t, t->ci); } } @@ -1592,11 +1893,11 @@ void engine_link_gravity_tasks(struct engine *e) { if (t->ci->nodeID == nodeID) { - engine_make_gravity_dependencies(sched, t, t->ci); + engine_make_self_gravity_dependencies(sched, t, t->ci); } if (t->cj->nodeID == nodeID && t->ci->super != t->cj->super) { - engine_make_gravity_dependencies(sched, t, t->cj); + engine_make_self_gravity_dependencies(sched, t, t->cj); } } } @@ -1612,10 +1913,12 @@ void engine_link_gravity_tasks(struct engine *e) { * @param gradient The gradient task to link. * @param force The force task to link. * @param c The cell. + * @param with_cooling Do we have a cooling task ? */ static inline void engine_make_hydro_loops_dependencies( struct scheduler *sched, struct task *density, struct task *gradient, struct task *force, struct cell *c, int with_cooling) { + /* init --> density loop --> ghost --> gradient loop --> extra_ghost */ /* extra_ghost --> force loop */ scheduler_addunlock(sched, c->super->init, density); @@ -1895,32 +2198,28 @@ void engine_make_extra_hydroloop_tasks(struct engine *e) { */ void engine_make_gravityrecursive_tasks(struct engine *e) { - struct space *s = e->s; - struct scheduler *sched = &e->sched; - const int nodeID = e->nodeID; - const int nr_cells = s->nr_cells; - struct cell *cells = s->cells_top; - - for (int k = 0; k < nr_cells; k++) { + /* struct space *s = e->s; */ + /* struct scheduler *sched = &e->sched; */ + /* const int nodeID = e->nodeID; */ + /* const int nr_cells = s->nr_cells; */ + /* struct cell *cells = s->cells_top; */ - /* Only do this for local cells containing gravity particles */ - if (cells[k].nodeID == nodeID && cells[k].gcount > 0) { + /* for (int k = 0; k < nr_cells; k++) { */ - /* Create tasks at top level. */ - struct task *up = - scheduler_addtask(sched, task_type_grav_up, task_subtype_none, 0, 0, - &cells[k], NULL, 0); + /* /\* Only do this for local cells containing gravity particles *\/ */ + /* if (cells[k].nodeID == nodeID && cells[k].gcount > 0) { */ - struct task *down = NULL; - /* struct task *down = */ - /* scheduler_addtask(sched, task_type_grav_down, task_subtype_none, 0, - * 0, */ - /* &cells[k], NULL, 0); */ + /* /\* Create tasks at top level. *\/ */ + /* struct task *up = NULL; */ + /* struct task *down = NULL; */ + /* /\* scheduler_addtask(sched, task_type_grav_down, + * task_subtype_none, 0, 0, *\/ */ + /* /\* &cells[k], NULL, 0); *\/ */ - /* Push tasks down the cell hierarchy. */ - engine_addtasks_grav(e, &cells[k], up, down); - } - } + /* /\* Push tasks down the cell hierarchy. *\/ */ + /* engine_addtasks_grav(e, &cells[k], up, down); */ + /* } */ + /* } */ } /** @@ -1942,8 +2241,8 @@ void engine_maketasks(struct engine *e) { /* Construct the firt hydro loop over neighbours */ if (e->policy & engine_policy_hydro) engine_make_hydroloop_tasks(e); - /* Add the gravity mm tasks. */ - if (e->policy & engine_policy_self_gravity) engine_make_gravity_tasks(e); + /* Add the self gravity tasks. */ + if (e->policy & engine_policy_self_gravity) engine_make_self_gravity_tasks(e); /* Add the external gravity tasks. */ if (e->policy & engine_policy_external_gravity) @@ -2207,18 +2506,20 @@ void engine_marktasks_mapper(void *map_data, int num_elements, if (cell_is_active(t->ci, e)) scheduler_activate(s, t); } + /* Gravity ? */ + else if (t->type == task_type_grav_down || + t->type == task_type_grav_long_range || + t->type == task_type_grav_top_level) { + if (cell_is_active(t->ci, e)) scheduler_activate(s, t); + } + /* Time-step? */ else if (t->type == task_type_timestep) { t->ci->updated = 0; t->ci->g_updated = 0; + t->ci->s_updated = 0; if (cell_is_active(t->ci, e)) scheduler_activate(s, t); } - - /* Tasks with no cells should not be skipped? */ - else if (t->type == task_type_grav_gather_m || - t->type == task_type_grav_fft) { - scheduler_activate(s, t); - } } } @@ -2282,6 +2583,7 @@ void engine_print_task_counts(struct engine *e) { fflush(stdout); message("nr_parts = %zu.", e->s->nr_parts); message("nr_gparts = %zu.", e->s->nr_gparts); + message("nr_sparts = %zu.", e->s->nr_sparts); if (e->verbose) message("took %.3f %s.", clocks_from_ticks(getticks() - tic), @@ -2318,8 +2620,15 @@ void engine_rebuild(struct engine *e) { if (engine_marktasks(e)) error("engine_marktasks failed after space_rebuild."); - /* Print the status of the system */ - if (e->verbose) engine_print_task_counts(e); +/* Print the status of the system */ +// if (e->verbose) engine_print_task_counts(e); + +#ifdef SWIFT_DEBUG_CHECKS + /* Check that all cells have been drifted to the current time. + * That can include cells that have not + * previously been active on this rank. */ + space_check_drift_point(e->s, e->ti_old); +#endif if (e->verbose) message("took %.3f %s.", clocks_from_ticks(getticks() - tic), @@ -2330,48 +2639,28 @@ void engine_rebuild(struct engine *e) { * @brief Prepare the #engine by re-building the cells and tasks. * * @param e The #engine to prepare. - * @param drift_all Whether to drift particles before rebuilding or not. Will - * not be necessary if all particles have already been - * drifted (before repartitioning for instance). - * @param postrepart If we have just repartitioned, if so we need to defer the - * skip until after the rebuild and not check the if all - * cells have been drifted. */ -void engine_prepare(struct engine *e, int drift_all, int postrepart) { +void engine_prepare(struct engine *e) { TIMER_TIC; - /* Unskip active tasks and check for rebuild */ - if (!postrepart) engine_unskip(e); - - /* Run through the tasks and mark as skip or not. */ - int rebuild = e->forcerebuild; - -/* Collect the values of rebuild from all nodes. */ -#ifdef WITH_MPI - int buff = 0; - if (MPI_Allreduce(&rebuild, &buff, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD) != - MPI_SUCCESS) - error("Failed to aggregate the rebuild flag across nodes."); - rebuild = buff; -#endif - - /* And rebuild if necessary. */ - if (rebuild) { - - /* Drift all particles to the current time if needed. */ - if (drift_all) engine_drift_all(e); - #ifdef SWIFT_DEBUG_CHECKS - /* Check that all cells have been drifted to the current time, unless - * we have just repartitioned, that can include cells that have not + if (e->forcerepart || e->forcerebuild) { + /* Check that all cells have been drifted to the current time. + * That can include cells that have not * previously been active on this rank. */ - if (!postrepart) space_check_drift_point(e->s, e->ti_current); + space_check_drift_point(e->s, e->ti_old); + } #endif - engine_rebuild(e); - } - if (postrepart) engine_unskip(e); + /* Do we need repartitioning ? */ + if (e->forcerepart) engine_repartition(e); + + /* Do we need rebuilding ? */ + if (e->forcerebuild) engine_rebuild(e); + + /* Unskip active tasks and check for rebuild */ + engine_unskip(e); /* Re-rank the tasks every now and then. */ if (e->tasks_age % engine_tasksreweight == 1) { @@ -2382,7 +2671,7 @@ void engine_prepare(struct engine *e, int drift_all, int postrepart) { TIMER_TOC(timer_prepare); if (e->verbose) - message("took %.3f %s (including drift all, rebuild and reweight).", + message("took %.3f %s (including unskip and reweight).", clocks_from_ticks(getticks() - tic), clocks_getunit()); } @@ -2440,32 +2729,39 @@ void engine_collect_kick(struct cell *c) { #endif /* WITH_MPI */ /* Counters for the different quantities. */ - int updated = 0, g_updated = 0; - integertime_t ti_end_min = max_nr_timesteps; + int updated = 0, g_updated = 0, s_updated = 0; + integertime_t ti_end_min = max_nr_timesteps, ti_end_max = 0, ti_beg_max = 0; /* Collect the values from the progeny. */ for (int k = 0; k < 8; k++) { struct cell *cp = c->progeny[k]; - if (cp != NULL && (cp->count > 0 || cp->gcount > 0)) { + if (cp != NULL && (cp->count > 0 || cp->gcount > 0 || cp->scount > 0)) { /* Recurse */ engine_collect_kick(cp); /* And update */ ti_end_min = min(ti_end_min, cp->ti_end_min); + ti_end_max = max(ti_end_max, cp->ti_end_max); + ti_beg_max = max(ti_beg_max, cp->ti_beg_max); updated += cp->updated; g_updated += cp->g_updated; + s_updated += cp->s_updated; /* Collected, so clear for next time. */ cp->updated = 0; cp->g_updated = 0; + cp->s_updated = 0; } } /* Store the collected values in the cell. */ c->ti_end_min = ti_end_min; + c->ti_end_max = ti_end_max; + c->ti_beg_max = ti_beg_max; c->updated = updated; c->g_updated = g_updated; + c->s_updated = s_updated; } /** @@ -2477,26 +2773,30 @@ void engine_collect_kick(struct cell *c) { void engine_collect_timestep(struct engine *e) { const ticks tic = getticks(); - int updates = 0, g_updates = 0; - integertime_t ti_end_min = max_nr_timesteps; + int updates = 0, g_updates = 0, s_updates = 0; + integertime_t ti_end_min = max_nr_timesteps, ti_end_max = 0, ti_beg_max = 0; const struct space *s = e->s; /* Collect the cell data. */ for (int k = 0; k < s->nr_cells; k++) { struct cell *c = &s->cells_top[k]; - if (c->count > 0 || c->gcount > 0) { + if (c->count > 0 || c->gcount > 0 || c->scount > 0) { /* Make the top-cells recurse */ engine_collect_kick(c); /* And aggregate */ ti_end_min = min(ti_end_min, c->ti_end_min); + ti_end_max = max(ti_end_max, c->ti_end_max); + ti_beg_max = max(ti_beg_max, c->ti_beg_max); updates += c->updated; g_updates += c->g_updated; + s_updates += c->s_updated; /* Collected, so clear for next time. */ c->updated = 0; c->g_updated = 0; + c->s_updated = 0; } } @@ -2512,20 +2812,25 @@ void engine_collect_timestep(struct engine *e) { ti_end_min = in_i[0]; } { - long long in_ll[2], out_ll[2]; + long long in_ll[3], out_ll[3]; out_ll[0] = updates; out_ll[1] = g_updates; - if (MPI_Allreduce(out_ll, in_ll, 2, MPI_LONG_LONG_INT, MPI_SUM, + out_ll[2] = s_updates; + if (MPI_Allreduce(out_ll, in_ll, 3, MPI_LONG_LONG_INT, MPI_SUM, MPI_COMM_WORLD) != MPI_SUCCESS) error("Failed to aggregate energies."); updates = in_ll[0]; g_updates = in_ll[1]; + s_updates = in_ll[2]; } #endif e->ti_end_min = ti_end_min; + e->ti_end_max = ti_end_max; + e->ti_beg_max = ti_beg_max; e->updates = updates; e->g_updates = g_updates; + e->s_updates = s_updates; if (e->verbose) message("took %.3f %s.", clocks_from_ticks(getticks() - tic), @@ -2541,6 +2846,20 @@ void engine_print_stats(struct engine *e) { const ticks tic = getticks(); +#ifdef SWIFT_DEBUG_CHECKS + /* Check that all cells have been drifted to the current time. + * That can include cells that have not + * previously been active on this rank. */ + space_check_drift_point(e->s, e->ti_current); + + /* Be verbose about this */ + message("Saving statistics at t=%e.", e->time); +#else + if (e->verbose) message("Saving statistics at t=%e.", e->time); +#endif + + e->save_stats = 0; + struct statistics stats; stats_init(&stats); @@ -2585,8 +2904,10 @@ void engine_skip_force_and_kick(struct engine *e) { /* Skip everything that updates the particles */ if (t->type == task_type_drift || t->type == task_type_kick1 || t->type == task_type_kick2 || t->type == task_type_timestep || - t->subtype == task_subtype_force || t->type == task_type_cooling || - t->type == task_type_sourceterms) + t->subtype == task_subtype_force || t->subtype == task_subtype_grav || + t->type == task_type_grav_long_range || + t->type == task_type_grav_top_level || t->type == task_type_grav_down || + t->type == task_type_cooling || t->type == task_type_sourceterms) t->skip = 1; } } @@ -2596,7 +2917,7 @@ void engine_skip_force_and_kick(struct engine *e) { * * @param e The #engine to act on. */ -void engine_skip_drift_and_kick(struct engine *e) { +void engine_skip_drift(struct engine *e) { struct task *tasks = e->sched.tasks; const int nr_tasks = e->sched.nr_tasks; @@ -2606,7 +2927,7 @@ void engine_skip_drift_and_kick(struct engine *e) { struct task *t = &tasks[i]; /* Skip everything that updates the particles */ - if (t->type == task_type_drift || t->type == task_type_kick1) t->skip = 1; + if (t->type == task_type_drift) t->skip = 1; } } @@ -2620,6 +2941,11 @@ void engine_launch(struct engine *e, int nr_runners) { const ticks tic = getticks(); +#ifdef SWIFT_DEBUG_CHECKS + /* Re-set all the cell task counters to 0 */ + space_reset_task_counters(e->s); +#endif + /* Prepare the scheduler. */ atomic_inc(&e->sched.waiting); @@ -2667,13 +2993,15 @@ void engine_init_particles(struct engine *e, int flag_entropy_ICs) { if (e->nodeID == 0) message("Computing initial gas densities."); - engine_prepare(e, 0, 0); - - engine_marktasks(e); + /* Construct all cells and tasks to start everything */ + engine_rebuild(e); /* No time integration. We just want the density and ghosts */ engine_skip_force_and_kick(e); + /* Print the number of active tasks ? */ + if (e->verbose) engine_print_task_counts(e); + /* Now, launch the calculation */ TIMER_TIC; engine_launch(e, e->nr_threads); @@ -2685,7 +3013,6 @@ void engine_init_particles(struct engine *e, int flag_entropy_ICs) { if (e->nodeID == 0) message("Converting internal energy variable."); /* Apply the conversion */ - // space_map_cells_pre(s, 0, cell_convert_hydro, NULL); for (size_t i = 0; i < s->nr_parts; ++i) hydro_convert_quantities(&s->parts[i], &s->xparts[i]); @@ -2697,19 +3024,43 @@ void engine_init_particles(struct engine *e, int flag_entropy_ICs) { } } +#ifdef SWIFT_DEBUG_CHECKS + /* Let's store the total mass in the system for future checks */ + e->s->total_mass = 0.; + for (size_t i = 0; i < s->nr_gparts; ++i) + e->s->total_mass += s->gparts[i].mass; +#ifdef WITH_MPI + if (MPI_Allreduce(MPI_IN_PLACE, &e->s->total_mass, 1, MPI_DOUBLE, MPI_SUM, + MPI_COMM_WORLD) != MPI_SUCCESS) + error("Failed to all-reduce total mass in the system."); +#endif + message("Total mass in the system: %e", e->s->total_mass); +#endif + /* Now time to get ready for the first time-step */ if (e->nodeID == 0) message("Running initial fake time-step."); + /* Prepare all the tasks again for a new round */ engine_marktasks(e); - engine_skip_drift_and_kick(e); + /* No drift this time */ + engine_skip_drift(e); + /* Print the number of active tasks ? */ + if (e->verbose) engine_print_task_counts(e); + + /* Run the 0th time-step */ engine_launch(e, e->nr_threads); + /* Recover the (integer) end of the next time-step */ + engine_collect_timestep(e); + clocks_gettime(&time2); #ifdef SWIFT_DEBUG_CHECKS space_check_timesteps(e->s); + part_verify_links(e->s->parts, e->s->gparts, e->s->sparts, e->s->nr_parts, + e->s->nr_gparts, e->s->nr_sparts, e->verbose); #endif /* Ready to go */ @@ -2727,8 +3078,6 @@ void engine_init_particles(struct engine *e, int flag_entropy_ICs) { */ void engine_step(struct engine *e) { - double snapshot_drift_time = 0.; - TIMER_TIC2; struct clocks_time time1, time2; @@ -2736,73 +3085,103 @@ void engine_step(struct engine *e) { e->tic_step = getticks(); - /* Recover the (integer) end of the next time-step */ - engine_collect_timestep(e); - - /* Check for output */ - while (e->ti_end_min >= e->ti_nextSnapshot && e->ti_nextSnapshot > 0) { - - e->ti_old = e->ti_current; - e->ti_current = e->ti_nextSnapshot; - e->time = e->ti_current * e->timeBase + e->timeBegin; - e->timeOld = e->ti_old * e->timeBase + e->timeBegin; - e->timeStep = (e->ti_current - e->ti_old) * e->timeBase; - snapshot_drift_time = e->timeStep; - - /* Drift everybody to the snapshot position */ - engine_drift_all(e); + if (e->nodeID == 0) { - /* Dump... */ - engine_dump_snapshot(e); + /* Print some information to the screen */ + printf(" %6d %14e %14e %10zu %10zu %10zu %21.3f\n", e->step, e->time, + e->timeStep, e->updates, e->g_updates, e->s_updates, + e->wallclock_time); + fflush(stdout); - /* ... and find the next output time */ - engine_compute_next_snapshot_time(e); + fprintf(e->file_timesteps, " %6d %14e %14e %10zu %10zu %10zu %21.3f\n", + e->step, e->time, e->timeStep, e->updates, e->g_updates, + e->s_updates, e->wallclock_time); + fflush(e->file_timesteps); } /* Move forward in time */ e->ti_old = e->ti_current; e->ti_current = e->ti_end_min; + e->max_active_bin = get_max_active_bin(e->ti_end_min); e->step += 1; e->time = e->ti_current * e->timeBase + e->timeBegin; e->timeOld = e->ti_old * e->timeBase + e->timeBegin; - e->timeStep = (e->ti_current - e->ti_old) * e->timeBase + snapshot_drift_time; + e->timeStep = (e->ti_current - e->ti_old) * e->timeBase; - if (e->nodeID == 0) { + /* Prepare the tasks to be launched, rebuild or repartition if needed. */ + engine_prepare(e); - /* Print some information to the screen */ - printf(" %6d %14e %14e %10zu %10zu %21.3f\n", e->step, e->time, - e->timeStep, e->updates, e->g_updates, e->wallclock_time); - fflush(stdout); +/* Repartition the space amongst the nodes? */ +#if defined(WITH_MPI) && defined(HAVE_METIS) + if (e->step % 100 == 2) e->forcerepart = 1; +#endif - fprintf(e->file_timesteps, " %6d %14e %14e %10zu %10zu %21.3f\n", e->step, - e->time, e->timeStep, e->updates, e->g_updates, e->wallclock_time); - fflush(e->file_timesteps); - } + /* Are we drifting everything (a la Gadget/GIZMO) ? */ + if (e->policy & engine_policy_drift_all) engine_drift_all(e); - /* Drift only the necessary particles, that means all particles - * if we are about to repartition. */ - const int repart = (e->forcerepart != REPART_NONE); - const int drift_all = (e->policy & engine_policy_drift_all); - if (repart || drift_all) engine_drift_all(e); + /* Print the number of active tasks ? */ + if (e->verbose) engine_print_task_counts(e); - /* Re-distribute the particles amongst the nodes? */ - if (repart) engine_repartition(e); +#ifdef SWIFT_GRAVITY_FORCE_CHECKS + /* Run the brute-force gravity calculation for some gparts */ + gravity_exact_force_compute(e->s, e); +#endif - /* Prepare the space. */ - engine_prepare(e, !(drift_all || repart), repart); + /* Start all the tasks. */ + TIMER_TIC; + engine_launch(e, e->nr_threads); + TIMER_TOC(timer_runners); - if (e->verbose) engine_print_task_counts(e); +#ifdef SWIFT_GRAVITY_FORCE_CHECKS + /* Check the accuracy of the gravity calculation */ + gravity_exact_force_check(e->s, e, 1e-1); +#endif - /* Save some statistics */ +/* Collect the values of rebuild from all nodes. */ +#ifdef WITH_MPI + int buff = 0; + if (MPI_Allreduce(&e->forcerebuild, &buff, 1, MPI_INT, MPI_MAX, + MPI_COMM_WORLD) != MPI_SUCCESS) + error("Failed to aggregate the rebuild flag across nodes."); + e->forcerebuild = buff; +#endif + + /* Save some statistics ? */ if (e->time - e->timeLastStatistics >= e->deltaTimeStatistics) { + e->save_stats = 1; + } + + /* Do we want a snapshot? */ + if (e->ti_end_min >= e->ti_nextSnapshot && e->ti_nextSnapshot > 0) + e->dump_snapshot = 1; + + /* Drift everybody (i.e. what has not yet been drifted) */ + /* to the current time */ + if (e->dump_snapshot || e->forcerebuild || e->forcerepart || e->save_stats) + engine_drift_all(e); + + /* Write a snapshot ? */ + if (e->dump_snapshot) { + + /* Dump... */ + engine_dump_snapshot(e); + + /* ... and find the next output time */ + engine_compute_next_snapshot_time(e); + } + + /* Save some statistics */ + if (e->save_stats) { + + /* Dump */ engine_print_stats(e); + + /* and move on */ e->timeLastStatistics += e->deltaTimeStatistics; } - /* Send off the runners. */ - TIMER_TIC; - engine_launch(e, e->nr_threads); - TIMER_TOC(timer_runners); + /* Recover the (integer) end of the next time-step */ + engine_collect_timestep(e); TIMER_TOC2(timer_step); @@ -2835,6 +3214,33 @@ void engine_unskip(struct engine *e) { clocks_getunit()); } +/** + * @brief Mapper function to drift ALL particle types and multipoles forward in + * time. + * + * @param map_data An array of #cell%s. + * @param num_elements Chunk size. + * @param extra_data Pointer to an #engine. + */ +void engine_do_drift_all_mapper(void *map_data, int num_elements, + void *extra_data) { + + struct engine *e = (struct engine *)extra_data; + struct cell *cells = (struct cell *)map_data; + + for (int ind = 0; ind < num_elements; ind++) { + struct cell *c = &cells[ind]; + if (c != NULL && c->nodeID == e->nodeID) { + /* Drift all the particles */ + cell_drift_particles(c, e); + + /* Drift the multipole */ + if (e->policy & engine_policy_self_gravity) + cell_drift_all_multipoles(c, e); + } + } +} + /** * @brief Drift *all* particles forward to the current time. * @@ -2843,7 +3249,12 @@ void engine_unskip(struct engine *e) { void engine_drift_all(struct engine *e) { const ticks tic = getticks(); - threadpool_map(&e->threadpool, runner_do_drift_mapper, e->s->cells_top, + +#ifdef SWIFT_DEBUG_CHECKS + if (e->nodeID == 0) message("Drifting all"); +#endif + + threadpool_map(&e->threadpool, engine_do_drift_all_mapper, e->s->cells_top, e->s->nr_cells, sizeof(struct cell), 1, e); #ifdef SWIFT_DEBUG_CHECKS @@ -2996,9 +3407,26 @@ void engine_split(struct engine *e, struct partition *initial_partition) { s->parts = parts_new; s->xparts = xparts_new; - /* Re-link the gparts. */ + /* Re-link the gparts to their parts. */ if (s->nr_parts > 0 && s->nr_gparts > 0) - part_relink_gparts(s->parts, s->nr_parts, 0); + part_relink_gparts_to_parts(s->parts, s->nr_parts, 0); + + /* Re-allocate the local sparts. */ + if (e->verbose) + message("Re-allocating sparts array from %zu to %zu.", s->size_sparts, + (size_t)(s->nr_sparts * 1.2)); + s->size_sparts = s->nr_sparts * 1.2; + struct spart *sparts_new = NULL; + if (posix_memalign((void **)&sparts_new, spart_align, + sizeof(struct spart) * s->size_sparts) != 0) + error("Failed to allocate new spart data."); + memcpy(sparts_new, s->sparts, sizeof(struct spart) * s->nr_sparts); + free(s->sparts); + s->sparts = sparts_new; + + /* Re-link the gparts to their sparts. */ + if (s->nr_sparts > 0 && s->nr_gparts > 0) + part_relink_gparts_to_sparts(s->sparts, s->nr_sparts, 0); /* Re-allocate the local gparts. */ if (e->verbose) @@ -3015,31 +3443,17 @@ void engine_split(struct engine *e, struct partition *initial_partition) { /* Re-link the parts. */ if (s->nr_parts > 0 && s->nr_gparts > 0) - part_relink_parts(s->gparts, s->nr_gparts, s->parts); + part_relink_parts_to_gparts(s->gparts, s->nr_gparts, s->parts); + + /* Re-link the sparts. */ + if (s->nr_sparts > 0 && s->nr_gparts > 0) + part_relink_sparts_to_gparts(s->gparts, s->nr_gparts, s->sparts); #ifdef SWIFT_DEBUG_CHECKS /* Verify that the links are correct */ - for (size_t k = 0; k < s->nr_gparts; ++k) { - - if (s->gparts[k].id_or_neg_offset <= 0) { - - struct part *part = &s->parts[-s->gparts[k].id_or_neg_offset]; - - if (part->gpart != &s->gparts[k]) error("Linking problem !"); - - if (s->gparts[k].x[0] != part->x[0] || s->gparts[k].x[1] != part->x[1] || - s->gparts[k].x[2] != part->x[2]) - error("Linked particles are not at the same position !"); - } - } - for (size_t k = 0; k < s->nr_parts; ++k) { - - if (s->parts[k].gpart != NULL && - s->parts[k].gpart->id_or_neg_offset != -(ptrdiff_t)k) - error("Linking problem !"); - } - + part_verify_links(s->parts, s->gparts, s->sparts, s->nr_parts, s->nr_gparts, + s->nr_sparts, e->verbose); #endif #else @@ -3057,24 +3471,36 @@ void engine_dump_snapshot(struct engine *e) { struct clocks_time time1, time2; clocks_gettime(&time1); +#ifdef SWIFT_DEBUG_CHECKS + /* Check that all cells have been drifted to the current time. + * That can include cells that have not + * previously been active on this rank. */ + space_check_drift_point(e->s, e->ti_current); + + /* Be verbose about this */ + message("writing snapshot at t=%e.", e->time); +#else if (e->verbose) message("writing snapshot at t=%e.", e->time); +#endif /* Dump... */ #if defined(WITH_MPI) #if defined(HAVE_PARALLEL_HDF5) - write_output_parallel(e, e->snapshotBaseName, e->internalUnits, + write_output_parallel(e, e->snapshotBaseName, e->internal_units, e->snapshotUnits, e->nodeID, e->nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL); #else - write_output_serial(e, e->snapshotBaseName, e->internalUnits, + write_output_serial(e, e->snapshotBaseName, e->internal_units, e->snapshotUnits, e->nodeID, e->nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL); #endif #else - write_output_single(e, e->snapshotBaseName, e->internalUnits, + write_output_single(e, e->snapshotBaseName, e->internal_units, e->snapshotUnits); #endif + e->dump_snapshot = 0; + clocks_gettime(&time2); if (e->verbose) message("writing particle properties took %.3f %s.", @@ -3102,7 +3528,7 @@ static cpu_set_t *engine_entry_affinity() { /** * @brief Ensure the NUMA node on which we initialise (first touch) everything - * doesn't change before engine_init allocates NUMA-local workers. + * doesn't change before engine_init allocates NUMA-local workers. */ void engine_pin() { @@ -3149,9 +3575,11 @@ void engine_unpin() { * @param with_aff use processor affinity, if supported. * @param policy The queuing policy to use. * @param verbose Is this #engine talkative ? + * @param reparttype What type of repartition algorithm are we using ? * @param internal_units The system of units used internally. * @param physical_constants The #phys_const used for this run. * @param hydro The #hydro_props used for this run. + * @param gravity The #gravity_props used for this run. * @param potential The properties of the external potential. * @param cooling_func The properties of the cooling function. * @param sourceterms The properties of the source terms function. @@ -3159,9 +3587,11 @@ void engine_unpin() { void engine_init(struct engine *e, struct space *s, const struct swift_params *params, int nr_nodes, int nodeID, int nr_threads, int with_aff, int policy, int verbose, - const struct UnitSystem *internal_units, + enum repartition_type reparttype, + const struct unit_system *internal_units, const struct phys_const *physical_constants, const struct hydro_props *hydro, + const struct gravity_props *gravity, const struct external_potential *potential, const struct cooling_function_data *cooling_func, struct sourceterms *sourceterms) { @@ -3179,7 +3609,10 @@ void engine_init(struct engine *e, struct space *s, e->proxy_ind = NULL; e->nr_proxies = 0; e->forcerebuild = 1; - e->forcerepart = REPART_NONE; + e->forcerepart = 0; + e->reparttype = reparttype; + e->dump_snapshot = 0; + e->save_stats = 0; e->links = NULL; e->nr_links = 0; e->timeBegin = parser_get_param_double(params, "TimeIntegration:time_begin"); @@ -3188,10 +3621,11 @@ void engine_init(struct engine *e, struct space *s, e->time = e->timeBegin; e->ti_old = 0; e->ti_current = 0; + e->max_active_bin = num_time_bins; e->timeStep = 0.; e->timeBase = 0.; e->timeBase_inv = 0.; - e->internalUnits = internal_units; + e->internal_units = internal_units; e->timeFirstSnapshot = parser_get_param_double(params, "Snapshots:time_first"); e->deltaTimeSnapshot = @@ -3200,7 +3634,7 @@ void engine_init(struct engine *e, struct space *s, parser_get_param_string(params, "Snapshots:basename", e->snapshotBaseName); e->snapshotCompression = parser_get_opt_param_int(params, "Snapshots:compression", 0); - e->snapshotUnits = malloc(sizeof(struct UnitSystem)); + e->snapshotUnits = malloc(sizeof(struct unit_system)); units_init_default(e->snapshotUnits, params, "Snapshots", internal_units); e->dt_min = parser_get_param_double(params, "TimeIntegration:dt_min"); e->dt_max = parser_get_param_double(params, "TimeIntegration:dt_max"); @@ -3214,6 +3648,7 @@ void engine_init(struct engine *e, struct space *s, e->wallclock_time = 0.f; e->physical_constants = physical_constants; e->hydro_properties = hydro; + e->gravity_properties = gravity; e->external_potential = potential; e->cooling_func = cooling_func; e->sourceterms = sourceterms; @@ -3397,9 +3832,9 @@ void engine_init(struct engine *e, struct space *s, e->hydro_properties->delta_neighbours, e->hydro_properties->eta_neighbours); - fprintf(e->file_timesteps, "# %6s %14s %14s %10s %10s %16s [%s]\n", "Step", - "Time", "Time-step", "Updates", "g-Updates", "Wall-clock time", - clocks_getunit()); + fprintf(e->file_timesteps, "# %6s %14s %14s %10s %10s %10s %16s [%s]\n", + "Step", "Time", "Time-step", "Updates", "g-Updates", "s-Updates", + "Wall-clock time", clocks_getunit()); fflush(e->file_timesteps); } @@ -3410,6 +3845,10 @@ void engine_init(struct engine *e, struct space *s, if (e->policy & engine_policy_hydro) if (e->nodeID == 0) hydro_props_print(e->hydro_properties); + /* Print information about the hydro scheme */ + if (e->policy & engine_policy_self_gravity) + if (e->nodeID == 0) gravity_props_print(e->gravity_properties); + /* Check we have sensible time bounds */ if (e->timeBegin >= e->timeEnd) error( @@ -3624,6 +4063,8 @@ void engine_compute_next_snapshot_time(struct engine *e) { */ void engine_clean(struct engine *e) { + for (int i = 0; i < e->nr_threads; ++i) cache_clean(&e->runners[i].par_cache); + free(e->runners); free(e->snapshotUnits); free(e->links); scheduler_clean(&e->sched); diff --git a/src/engine.h b/src/engine.h index db5a7317d233a8e386fb03817cd42659ecd10816..80903ee78156c7e4efb2650e59e4fa832fecbfa3 100644 --- a/src/engine.h +++ b/src/engine.h @@ -39,6 +39,7 @@ /* Includes. */ #include "clocks.h" #include "cooling_struct.h" +#include "gravity_properties.h" #include "parser.h" #include "partition.h" #include "potential.h" @@ -65,7 +66,8 @@ enum engine_policy { engine_policy_cosmology = (1 << 10), engine_policy_drift_all = (1 << 11), engine_policy_cooling = (1 << 12), - engine_policy_sourceterms = (1 << 13) + engine_policy_sourceterms = (1 << 13), + engine_policy_stars = (1 << 14) }; extern const char *engine_policy_names[]; @@ -120,6 +122,9 @@ struct engine { double time; integertime_t ti_current; + /* The highest active bin at this time */ + timebin_t max_active_bin; + /* Time step */ double timeStep; @@ -130,11 +135,17 @@ struct engine { /* Minimal ti_end for the next time-step */ integertime_t ti_end_min; + /* Maximal ti_end for the next time-step */ + integertime_t ti_end_max; + + /* Maximal ti_beg for the next time-step */ + integertime_t ti_beg_max; + /* Number of particles updated */ - size_t updates, g_updates; + size_t updates, g_updates, s_updates; /* The internal system of units */ - const struct UnitSystem *internalUnits; + const struct unit_system *internal_units; /* Snapshot information */ double timeFirstSnapshot; @@ -142,7 +153,7 @@ struct engine { integertime_t ti_nextSnapshot; char snapshotBaseName[200]; int snapshotCompression; - struct UnitSystem *snapshotUnits; + struct unit_system *snapshotUnits; /* Statistics information */ FILE *file_stats; @@ -178,7 +189,16 @@ struct engine { /* Force the engine to rebuild? */ int forcerebuild; - enum repartition_type forcerepart; + + /* Force the engine to repartition ? */ + int forcerepart; + enum repartition_type reparttype; + + /* Need to dump some statistics ? */ + int save_stats; + + /* Need to dump a snapshot ? */ + int dump_snapshot; /* How many steps have we done with the same set of tasks? */ int tasks_age; @@ -196,6 +216,9 @@ struct engine { /* Properties of the hydro scheme */ const struct hydro_props *hydro_properties; + /* Properties of the self-gravity scheme */ + const struct gravity_props *gravity_properties; + /* Properties of external gravitational potential */ const struct external_potential *external_potential; @@ -218,22 +241,25 @@ void engine_dump_snapshot(struct engine *e); void engine_init(struct engine *e, struct space *s, const struct swift_params *params, int nr_nodes, int nodeID, int nr_threads, int with_aff, int policy, int verbose, - const struct UnitSystem *internal_units, + enum repartition_type reparttype, + const struct unit_system *internal_units, const struct phys_const *physical_constants, const struct hydro_props *hydro, + const struct gravity_props *gravity, const struct external_potential *potential, const struct cooling_function_data *cooling, struct sourceterms *sourceterms); void engine_launch(struct engine *e, int nr_runners); -void engine_prepare(struct engine *e, int drift_all, int postrepart); -void engine_print(struct engine *e); +void engine_prepare(struct engine *e); void engine_init_particles(struct engine *e, int flag_entropy_ICs); void engine_step(struct engine *e); void engine_maketasks(struct engine *e); void engine_split(struct engine *e, struct partition *initial_partition); void engine_exchange_strays(struct engine *e, size_t offset_parts, int *ind_part, size_t *Npart, size_t offset_gparts, - int *ind_gpart, size_t *Ngpart); + int *ind_gpart, size_t *Ngpart, + size_t offset_sparts, int *ind_spart, + size_t *Nspart); void engine_rebuild(struct engine *e); void engine_repartition(struct engine *e); void engine_makeproxies(struct engine *e); diff --git a/src/gravity.c b/src/gravity.c new file mode 100644 index 0000000000000000000000000000000000000000..86f9fa82e3eb693cc3c051420fb9c7bff277eb9f --- /dev/null +++ b/src/gravity.c @@ -0,0 +1,187 @@ +/******************************************************************************* + * This file is part of SWIFT. + * Copyright (c) 2017 Matthieu Schaller (matthieu.schaller@durham.ac.uk) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + ******************************************************************************/ + +/* Config parameters. */ +#include "../config.h" + +/* This object's header. */ +#include "gravity.h" + +/* Local headers. */ +#include "active.h" +#include "error.h" + +/** + * @brief Run a brute-force gravity calculation for a subset of particles. + * + * All gpart with ID modulo SWIFT_GRAVITY_FORCE_CHECKS will get their forces + * computed. + * + * @param s The #space to use. + * @param e The #engine (to access the current time). + */ +void gravity_exact_force_compute(struct space *s, const struct engine *e) { + +#ifdef SWIFT_GRAVITY_FORCE_CHECKS + + const ticks tic = getticks(); + const double const_G = e->physical_constants->const_newton_G; + int counter = 0; + + for (size_t i = 0; i < s->nr_gparts; ++i) { + + struct gpart *gpi = &s->gparts[i]; + + /* Is the particle active and part of the subset to be tested ? */ + if (gpi->id_or_neg_offset % SWIFT_GRAVITY_FORCE_CHECKS == 0 && + gpart_is_active(gpi, e)) { + + /* Be ready for the calculation */ + gpi->a_grav[0] = 0.f; + gpi->a_grav[1] = 0.f; + gpi->a_grav[2] = 0.f; + + /* Interact it with all other particles in the space.*/ + for (size_t j = 0; j < s->nr_gparts; ++j) { + + /* No self interaction */ + if (i == j) continue; + + struct gpart *gpj = &s->gparts[j]; + + /* Compute the pairwise distance. */ + float dx[3] = {gpi->x[0] - gpj->x[0], // x + gpi->x[1] - gpj->x[1], // y + gpi->x[2] - gpj->x[2]}; // z + const float r2 = dx[0] * dx[0] + dx[1] * dx[1] + dx[2] * dx[2]; + + runner_iact_grav_pp_nonsym(0.f, r2, dx, gpi, gpj); + } + + /* Finish the calculation */ + gravity_end_force(gpi, const_G); + + /* Store the exact answer */ + gpi->a_grav_exact[0] = gpi->a_grav[0]; + gpi->a_grav_exact[1] = gpi->a_grav[1]; + gpi->a_grav_exact[2] = gpi->a_grav[2]; + + /* Restore everything */ + gpi->a_grav[0] = 0.f; + gpi->a_grav[1] = 0.f; + gpi->a_grav[2] = 0.f; + + counter++; + } + } + + message("Computed exact gravity for %d gparts.", counter); + + if (e->verbose) + message("took %.3f %s.", clocks_from_ticks(getticks() - tic), + clocks_getunit()); +#else + error("Gravity checking function called without the corresponding flag."); +#endif +} + +/** + * @brief Check the accuracy of the gravity calculation by comparing the + * accelerations + * to the brute-force computed ones. + * + * All gpart with ID modulo SWIFT_GRAVITY_FORCE_CHECKS will be checked. + * + * @param s The #space to use. + * @param e The #engine (to access the current time). + * @param rel_tol The maximal relative error. Will call error() if one particle + * has a larger error. + */ +void gravity_exact_force_check(struct space *s, const struct engine *e, + float rel_tol) { + +#ifdef SWIFT_GRAVITY_FORCE_CHECKS + + const double const_G = e->physical_constants->const_newton_G; + + int counter = 0; + + /* Some accumulators */ + float err_rel[3]; + float err_rel_max[3] = {0.f, 0.f, 0.f}; + float err_rel_min[3] = {FLT_MAX, FLT_MAX, FLT_MAX}; + float err_rel_mean[3] = {0.f, 0.f, 0.f}; + float err_rel_mean2[3] = {0.f, 0.f, 0.f}; + float err_rel_std[3] = {0.f, 0.f, 0.f}; + + for (size_t i = 0; i < s->nr_gparts; ++i) { + + struct gpart *gpi = &s->gparts[i]; + + /* Is the particle was active and part of the subset to be tested ? */ + if (gpi->id_or_neg_offset % SWIFT_GRAVITY_FORCE_CHECKS == 0 && + gpart_is_starting(gpi, e)) { + + /* Compute relative error */ + for (int k = 0; k < 3; ++k) + if (fabsf(gpi->a_grav_exact[k]) > FLT_EPSILON * const_G) + err_rel[k] = (gpi->a_grav[k] - gpi->a_grav_exact[k]) / + fabsf(gpi->a_grav_exact[k]); + else + err_rel[k] = 0.f; + + /* Check that we are not above tolerance */ + if (fabsf(err_rel[0]) > rel_tol || fabsf(err_rel[1]) > rel_tol || + fabsf(err_rel[2]) > rel_tol) + error("Error too large ! gp->a_grav=[%e %e %e] gp->a_exact=[%e %e %e]", + gpi->a_grav[0], gpi->a_grav[1], gpi->a_grav[2], + gpi->a_grav_exact[0], gpi->a_grav_exact[1], gpi->a_grav_exact[2]); + + /* Construct some statistics */ + for (int k = 0; k < 3; ++k) { + err_rel_max[k] = max(err_rel_max[k], fabsf(err_rel[k])); + err_rel_min[k] = min(err_rel_min[k], fabsf(err_rel[k])); + err_rel_mean[k] += err_rel[k]; + err_rel_mean2[k] += err_rel[k] * err_rel[k]; + } + + counter++; + } + } + + /* Final operation on the stats */ + if (counter > 0) { + for (int k = 0; k < 3; ++k) { + err_rel_mean[k] /= counter; + err_rel_mean2[k] /= counter; + err_rel_std[k] = + sqrtf(err_rel_mean2[k] - err_rel_mean[k] * err_rel_mean[k]); + } + } + + /* Report on the findings */ + message("Checked gravity for %d gparts.", counter); + for (int k = 0; k < 3; ++k) + message("Error on a_grav[%d]: min=%e max=%e mean=%e std=%e", k, + err_rel_min[k], err_rel_max[k], err_rel_mean[k], err_rel_std[k]); + +#else + error("Gravity checking function called without the corresponding flag."); +#endif +} diff --git a/src/gravity.h b/src/gravity.h index 6edcd59d9955029217364d4fc67874216b0e24dc..00b930c00fb2558f274feb2991b78e96dc8b990b 100644 --- a/src/gravity.h +++ b/src/gravity.h @@ -22,9 +22,20 @@ /* Config parameters. */ #include "../config.h" +/* Local headers. */ +#include "const.h" +#include "engine.h" +#include "inline.h" +#include "part.h" +#include "space.h" + /* So far only one model here */ /* Straight-forward import */ #include "./gravity/Default/gravity.h" #include "./gravity/Default/gravity_iact.h" +void gravity_exact_force_compute(struct space *s, const struct engine *e); +void gravity_exact_force_check(struct space *s, const struct engine *e, + float rel_tol); + #endif diff --git a/src/gravity/Default/gravity.h b/src/gravity/Default/gravity.h index a0bfee05f8b7f93cce65e8b9a3e7e322e166569d..93a65e2f5a70e09ad14280cf9c334753359fb8b5 100644 --- a/src/gravity/Default/gravity.h +++ b/src/gravity/Default/gravity.h @@ -21,15 +21,18 @@ #define SWIFT_DEFAULT_GRAVITY_H #include <float.h> +#include "gravity_properties.h" #include "minmax.h" /** * @brief Computes the gravity time-step of a given particle due to self-gravity * * @param gp Pointer to the g-particle data. + * @param grav_props Constants used in the gravity scheme. */ __attribute__((always_inline)) INLINE static float -gravity_compute_timestep_self(const struct gpart* const gp) { +gravity_compute_timestep_self(const struct gpart* const gp, + const struct gravity_props* restrict grav_props) { const float ac2 = gp->a_grav[0] * gp->a_grav[0] + gp->a_grav[1] * gp->a_grav[1] + @@ -37,7 +40,7 @@ gravity_compute_timestep_self(const struct gpart* const gp) { const float ac = (ac2 > 0.f) ? sqrtf(ac2) : FLT_MIN; - const float dt = sqrtf(2.f * const_gravity_eta * gp->epsilon / ac); + const float dt = sqrtf(2.f * grav_props->eta * gp->epsilon / ac); return dt; } @@ -57,6 +60,10 @@ __attribute__((always_inline)) INLINE static void gravity_init_gpart( gp->a_grav[0] = 0.f; gp->a_grav[1] = 0.f; gp->a_grav[2] = 0.f; + +#ifdef SWIFT_DEBUG_CHECKS + gp->mass_interacted = 0.; +#endif } /** @@ -85,6 +92,15 @@ __attribute__((always_inline)) INLINE static void gravity_end_force( __attribute__((always_inline)) INLINE static void gravity_kick_extra( struct gpart* gp, float dt) {} +/** + * @brief Sets the values to be predicted in the drifts to their values at a + * kick time + * + * @param gp The particle. + */ +__attribute__((always_inline)) INLINE static void +gravity_reset_predicted_values(struct gpart* gp) {} + /** * @brief Initialises the g-particles for the first time * diff --git a/src/gravity/Default/gravity_part.h b/src/gravity/Default/gravity_part.h index aa309a5beb201c1aeec0cabe9d64a158d2071fa9..00ae0f5b05cd95750c34b60e2353a9fc1d0a5c32 100644 --- a/src/gravity/Default/gravity_part.h +++ b/src/gravity/Default/gravity_part.h @@ -47,8 +47,14 @@ struct gpart { /* Time-step length */ timebin_t time_bin; + /* Type of the #gpart (DM, gas, star, ...) */ + enum part_type type; + #ifdef SWIFT_DEBUG_CHECKS + /* Total mass this gpart interacted with */ + double mass_interacted; + /* Time of the last drift */ integertime_t ti_drift; @@ -57,6 +63,13 @@ struct gpart { #endif +#ifdef SWIFT_GRAVITY_FORCE_CHECKS + + /* Brute-force particle acceleration. */ + float a_grav_exact[3]; + +#endif + } SWIFT_STRUCT_ALIGN; #endif /* SWIFT_DEFAULT_GRAVITY_PART_H */ diff --git a/src/gravity_derivatives.h b/src/gravity_derivatives.h new file mode 100644 index 0000000000000000000000000000000000000000..4730d9df5dc573de74fde422e1b7dafc0ee0994a --- /dev/null +++ b/src/gravity_derivatives.h @@ -0,0 +1,72 @@ +/******************************************************************************* + * This file is part of SWIFT. + * Copyright (c) 2016 Matthieu Schaller (matthieu.schaller@durham.ac.uk) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + ******************************************************************************/ +#ifndef SWIFT_GRAVITY_DERIVATIVE_H +#define SWIFT_GRAVITY_DERIVATIVE_H + +/* Some standard headers. */ +#include <math.h> + +/* Local headers. */ +#include "inline.h" + +/** + * @brief \f$ \phi(r_x, r_y, r_z) \f$. + */ +__attribute__((always_inline)) INLINE static double D_000(double r_x, + double r_y, + double r_z, + double r_inv) { + + return r_inv; +} + +/** + * @brief \f$ \frac{\partial\phi(r_x, r_y, r_z)}{\partial r_x} \f$. + */ +__attribute__((always_inline)) INLINE static double D_100(double r_x, + double r_y, + double r_z, + double r_inv) { + + return -r_x * r_inv * r_inv * r_inv; +} + +/** + * @brief \f$ \frac{\partial\phi(r_x, r_y, r_z)}{\partial r_x} \f$. + */ +__attribute__((always_inline)) INLINE static double D_010(double r_x, + double r_y, + double r_z, + double r_inv) { + + return -r_y * r_inv * r_inv * r_inv; +} + +/** + * @brief \f$ \frac{\partial\phi(r_x, r_y, r_z)}{\partial r_x} \f$. + */ +__attribute__((always_inline)) INLINE static double D_001(double r_x, + double r_y, + double r_z, + double r_inv) { + + return -r_z * r_inv * r_inv * r_inv; +} + +#endif /* SWIFT_GRAVITY_DERIVATIVE_H */ diff --git a/src/gravity_properties.c b/src/gravity_properties.c new file mode 100644 index 0000000000000000000000000000000000000000..6c1ec1f6d9e80d46278892cc7a2081b2cadad923 --- /dev/null +++ b/src/gravity_properties.c @@ -0,0 +1,78 @@ +/******************************************************************************* + * This file is part of SWIFT. + * Copyright (c) 2016 Matthieu Schaller (matthieu.schaller@durham.ac.uk) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + ******************************************************************************/ + +/* This object's header. */ +#include "gravity_properties.h" + +/* Standard headers */ +#include <float.h> +#include <math.h> + +/* Local headers. */ +#include "adiabatic_index.h" +#include "common_io.h" +#include "dimension.h" +#include "error.h" +#include "gravity.h" +#include "kernel_gravity.h" + +#define gravity_props_default_a_smooth 1.25f +#define gravity_props_default_r_cut 4.5f + +void gravity_props_init(struct gravity_props *p, + const struct swift_params *params) { + + /* Tree-PM parameters */ + p->a_smooth = parser_get_opt_param_float(params, "Gravity:a_smooth", + gravity_props_default_a_smooth); + p->r_cut = parser_get_opt_param_float(params, "Gravity:r_cut", + gravity_props_default_r_cut); + + /* Time integration */ + p->eta = parser_get_param_float(params, "Gravity:eta"); + + /* Softening lengths */ + p->epsilon = parser_get_param_float(params, "Gravity:epsilon"); +} + +void gravity_props_print(const struct gravity_props *p) { + + message("Self-gravity scheme: FMM-MM"); + + message("Self-gravity time integration: eta=%.4f", p->eta); + + message("Self-gravity softening: epsilon=%.4f", p->epsilon); + + if (p->a_smooth != gravity_props_default_a_smooth) + message("Self-gravity smoothing-scale: a_smooth=%f", p->a_smooth); + + if (p->r_cut != gravity_props_default_r_cut) + message("Self-gravity MM cut-off: r_cut=%f", p->r_cut); +} + +#if defined(HAVE_HDF5) +void gravity_props_print_snapshot(hid_t h_grpgrav, + const struct gravity_props *p) { + + io_write_attribute_f(h_grpgrav, "Time integration eta", p->eta); + io_write_attribute_f(h_grpgrav, "Softening", p->epsilon); + io_write_attribute_f(h_grpgrav, "MM a_smooth", p->a_smooth); + io_write_attribute_f(h_grpgrav, "MM r_cut", p->r_cut); +} +#endif diff --git a/src/gravity_properties.h b/src/gravity_properties.h new file mode 100644 index 0000000000000000000000000000000000000000..6fde91b1d8f2d32a27ec278d5e42f91e5fe930a6 --- /dev/null +++ b/src/gravity_properties.h @@ -0,0 +1,57 @@ +/******************************************************************************* + * This file is part of SWIFT. + * Copyright (c) 2016 Matthieu Schaller (matthieu.schaller@durham.ac.uk) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + ******************************************************************************/ +#ifndef SWIFT_GRAVITY_PROPERTIES +#define SWIFT_GRAVITY_PROPERTIES + +/* Config parameters. */ +#include "../config.h" + +#if defined(HAVE_HDF5) +#include <hdf5.h> +#endif + +/* Local includes. */ +#include "parser.h" + +/** + * @brief Contains all the constants and parameters of the self-gravity scheme + */ +struct gravity_props { + + /* Tree-PM parameters */ + float a_smooth; + float r_cut; + + /* Time integration parameters */ + float eta; + + /* Softening lengths */ + float epsilon; +}; + +void gravity_props_print(const struct gravity_props *p); +void gravity_props_init(struct gravity_props *p, + const struct swift_params *params); + +#if defined(HAVE_HDF5) +void gravity_props_print_snapshot(hid_t h_grpsph, + const struct gravity_props *p); +#endif + +#endif /* SWIFT_GRAVITY_PROPERTIES */ diff --git a/src/hydro/Default/hydro.h b/src/hydro/Default/hydro.h index bfb5cd1ce39a9908573c66406f41b56561a870d6..a614d08c30b21f9e7d422bf6b6a09d10d2e89799 100644 --- a/src/hydro/Default/hydro.h +++ b/src/hydro/Default/hydro.h @@ -148,6 +148,16 @@ __attribute__((always_inline)) INLINE static float hydro_compute_timestep( return min(dt_cfl, dt_u_change); } +/** + * @brief Does some extra hydro operations once the actual physical time step + * for the particle is known. + * + * @param p The particle to act upon. + * @param dt Physical time step of the particle during the next step. + */ +__attribute__((always_inline)) INLINE static void hydro_timestep_extra( + struct part *p, float dt) {} + /** * @brief Prepares a particle for the density calculation. * diff --git a/src/hydro/Default/hydro_io.h b/src/hydro/Default/hydro_io.h index bb35c914bcab8787f609b4dd49acd0cc883b4263..62e94b05ffea259aac99d4b3714e0eea7e7c955f 100644 --- a/src/hydro/Default/hydro_io.h +++ b/src/hydro/Default/hydro_io.h @@ -91,21 +91,25 @@ void hydro_write_particles(struct part* parts, struct io_props* list, void writeSPHflavour(hid_t h_grpsph) { /* Viscosity and thermal conduction */ - writeAttribute_s(h_grpsph, "Thermal Conductivity Model", - "Price (2008) without switch"); - writeAttribute_f(h_grpsph, "Thermal Conductivity alpha", - const_conductivity_alpha); - writeAttribute_s(h_grpsph, "Viscosity Model", - "Morris & Monaghan (1997), Rosswog, Davies, Thielemann & " - "Piran (2000) with additional Balsara (1995) switch"); - writeAttribute_f(h_grpsph, "Viscosity alpha_min", const_viscosity_alpha_min); - writeAttribute_f(h_grpsph, "Viscosity alpha_max", const_viscosity_alpha_max); - writeAttribute_f(h_grpsph, "Viscosity beta", 2.f); - writeAttribute_f(h_grpsph, "Viscosity decay length", const_viscosity_length); + io_write_attribute_s(h_grpsph, "Thermal Conductivity Model", + "Price (2008) without switch"); + io_write_attribute_f(h_grpsph, "Thermal Conductivity alpha", + const_conductivity_alpha); + io_write_attribute_s( + h_grpsph, "Viscosity Model", + "Morris & Monaghan (1997), Rosswog, Davies, Thielemann & " + "Piran (2000) with additional Balsara (1995) switch"); + io_write_attribute_f(h_grpsph, "Viscosity alpha_min", + const_viscosity_alpha_min); + io_write_attribute_f(h_grpsph, "Viscosity alpha_max", + const_viscosity_alpha_max); + io_write_attribute_f(h_grpsph, "Viscosity beta", 2.f); + io_write_attribute_f(h_grpsph, "Viscosity decay length", + const_viscosity_length); /* Time integration properties */ - writeAttribute_f(h_grpsph, "Maximal Delta u change over dt", - const_max_u_change); + io_write_attribute_f(h_grpsph, "Maximal Delta u change over dt", + const_max_u_change); } /** diff --git a/src/hydro/Gadget2/hydro.h b/src/hydro/Gadget2/hydro.h index 160a2d8b5d25a97cefb2afd5e22d8e6bcea0006e..cc7b422ccbe7c678969df5779a4d4a054c65528e 100644 --- a/src/hydro/Gadget2/hydro.h +++ b/src/hydro/Gadget2/hydro.h @@ -152,6 +152,16 @@ __attribute__((always_inline)) INLINE static float hydro_compute_timestep( return dt_cfl; } +/** + * @brief Does some extra hydro operations once the actual physical time step + * for the particle is known. + * + * @param p The particle to act upon. + * @param dt Physical time step of the particle during the next step. + */ +__attribute__((always_inline)) INLINE static void hydro_timestep_extra( + struct part *p, float dt) {} + /** * @brief Prepares a particle for the density calculation. * diff --git a/src/hydro/Gadget2/hydro_io.h b/src/hydro/Gadget2/hydro_io.h index 162d368dd073be2fd0f06f4ecbc1431fb34e7798..3e46b2351eb6a3871946dd9c69c8d108b10da0df 100644 --- a/src/hydro/Gadget2/hydro_io.h +++ b/src/hydro/Gadget2/hydro_io.h @@ -108,13 +108,13 @@ void hydro_write_particles(struct part* parts, struct io_props* list, void writeSPHflavour(hid_t h_grpsph) { /* Viscosity and thermal conduction */ - writeAttribute_s(h_grpsph, "Thermal Conductivity Model", - "(No treatment) as in Springel (2005)"); - writeAttribute_s( + io_write_attribute_s(h_grpsph, "Thermal Conductivity Model", + "(No treatment) as in Springel (2005)"); + io_write_attribute_s( h_grpsph, "Viscosity Model", "as in Springel (2005), i.e. Monaghan (1992) with Balsara (1995) switch"); - writeAttribute_f(h_grpsph, "Viscosity alpha", const_viscosity_alpha); - writeAttribute_f(h_grpsph, "Viscosity beta", 3.f); + io_write_attribute_f(h_grpsph, "Viscosity alpha", const_viscosity_alpha); + io_write_attribute_f(h_grpsph, "Viscosity beta", 3.f); } /** diff --git a/src/hydro/Gizmo/hydro.h b/src/hydro/Gizmo/hydro.h index c59af05460157a756c15d8ca84af8a7834fde2d3..643489912a6c6b1db921e73b508910cc670d49ae 100644 --- a/src/hydro/Gizmo/hydro.h +++ b/src/hydro/Gizmo/hydro.h @@ -1,3 +1,4 @@ + /******************************************************************************* * This file is part of SWIFT. * Coypright (c) 2015 Matthieu Schaller (matthieu.schaller@durham.ac.uk) @@ -23,6 +24,7 @@ #include "equation_of_state.h" #include "hydro_gradients.h" #include "minmax.h" +#include "riemann.h" /** * @brief Computes the hydro time-step of a given particle @@ -37,7 +39,46 @@ __attribute__((always_inline)) INLINE static float hydro_compute_timestep( const float CFL_condition = hydro_properties->CFL_condition; - return CFL_condition * p->h / fabsf(p->timestepvars.vmax); + if (p->timestepvars.vmax == 0.) { + /* vmax can be zero in vacuum cells that only have vacuum neighbours */ + /* in this case, the time step should be limited by the maximally + allowed time step. Since we do not know what that value is here, we set + the time step to a very large value */ + return FLT_MAX; + } else { + return CFL_condition * p->h / fabsf(p->timestepvars.vmax); + } +} + +/** + * @brief Does some extra hydro operations once the actual physical time step + * for the particle is known. + * + * We use this to store the physical time step, since it is used for the flux + * exchange during the force loop. + * + * We also set the active flag of the particle to inactive. It will be set to + * active in hydro_init_part, which is called the next time the particle becomes + * active. + * + * @param p The particle to act upon. + * @param dt Physical time step of the particle during the next step. + */ +__attribute__((always_inline)) INLINE static void hydro_timestep_extra( + struct part* p, float dt) { + +#ifdef SWIFT_DEBUG_CHECKS + if (dt == 0.) { + error("Zero time step assigned to particle!"); + } + + if (dt != dt) { + error("NaN time step assigned to particle!"); + } +#endif + + p->force.dt = dt; + p->force.active = 0; } /** @@ -58,13 +99,44 @@ __attribute__((always_inline)) INLINE static float hydro_compute_timestep( __attribute__((always_inline)) INLINE static void hydro_first_init_part( struct part* p, struct xpart* xp) { - xp->v_full[0] = p->v[0]; - xp->v_full[1] = p->v[1]; - xp->v_full[2] = p->v[2]; + const float mass = p->conserved.mass; p->primitives.v[0] = p->v[0]; p->primitives.v[1] = p->v[1]; p->primitives.v[2] = p->v[2]; + + /* we can already initialize the momentum */ + p->conserved.momentum[0] = mass * p->primitives.v[0]; + p->conserved.momentum[1] = mass * p->primitives.v[1]; + p->conserved.momentum[2] = mass * p->primitives.v[2]; + +/* and the thermal energy */ +/* remember that we store the total thermal energy, not the specific thermal + energy (as in Gadget) */ +#if defined(EOS_ISOTHERMAL_GAS) + /* this overwrites the internal energy from the initial condition file */ + p->conserved.energy = mass * const_isothermal_internal_energy; +#else + p->conserved.energy *= mass; +#endif + +#ifdef GIZMO_TOTAL_ENERGY + /* add the total kinetic energy */ + p->conserved.energy += 0.5f * (p->conserved.momentum[0] * p->primitives.v[0] + + p->conserved.momentum[1] * p->primitives.v[1] + + p->conserved.momentum[2] * p->primitives.v[2]); +#endif + +#if defined(GIZMO_FIX_PARTICLES) + /* make sure the particles are initially at rest */ + p->v[0] = 0.; + p->v[1] = 0.; + p->v[2] = 0.; +#endif + + xp->v_full[0] = p->v[0]; + xp->v_full[1] = p->v[1]; + xp->v_full[2] = p->v[2]; } /** @@ -89,6 +161,9 @@ __attribute__((always_inline)) INLINE static void hydro_init_part( p->geometry.matrix_E[2][0] = 0.0f; p->geometry.matrix_E[2][1] = 0.0f; p->geometry.matrix_E[2][2] = 0.0f; + + /* Set the active flag to active. */ + p->force.active = 1; } /** @@ -149,6 +224,17 @@ __attribute__((always_inline)) INLINE static void hydro_end_density( /* compute primitive variables */ /* eqns (3)-(5) */ const float m = p->conserved.mass; + +#ifdef SWIFT_DEBUG_CHECKS + if (m == 0.) { + error("Mass is 0!"); + } + + if (volume == 0.) { + error("Volume is 0!"); + } +#endif + float momentum[3]; momentum[0] = p->conserved.momentum[0]; momentum[1] = p->conserved.momentum[1]; @@ -157,8 +243,35 @@ __attribute__((always_inline)) INLINE static void hydro_end_density( p->primitives.v[0] = momentum[0] / m; p->primitives.v[1] = momentum[1] / m; p->primitives.v[2] = momentum[2] / m; - const float energy = p->conserved.energy; + +#ifdef EOS_ISOTHERMAL_GAS + /* although the pressure is not formally used anywhere if an isothermal eos + has been selected, we still make sure it is set to the correct value */ + p->primitives.P = const_isothermal_soundspeed * const_isothermal_soundspeed * + p->primitives.rho; +#else + + float energy = p->conserved.energy; + +#ifdef GIZMO_TOTAL_ENERGY + /* subtract the kinetic energy; we want the thermal energy */ + energy -= 0.5f * (momentum[0] * p->primitives.v[0] + + momentum[1] * p->primitives.v[1] + + momentum[2] * p->primitives.v[2]); +#endif + + /* energy contains the total thermal energy, we want the specific energy. + this is why we divide by the volume, and not by the density */ p->primitives.P = hydro_gamma_minus_one * energy / volume; +#endif + + /* sanity checks */ + /* it would probably be safer to throw a warning if netive densities or + pressures occur */ + if (p->primitives.rho < 0.0f || p->primitives.P < 0.0f) { + p->primitives.rho = 0.0f; + p->primitives.P = 0.0f; + } } /** @@ -180,13 +293,11 @@ __attribute__((always_inline)) INLINE static void hydro_end_density( __attribute__((always_inline)) INLINE static void hydro_prepare_force( struct part* restrict p, struct xpart* restrict xp) { - /* Set the physical time step */ - p->force.dt = get_timestep(p->time_bin, 0.); // MATTHIEU 0 - /* Initialize time step criterion variables */ p->timestepvars.vmax = 0.0f; /* Set the actual velocity of the particle */ + /* if GIZMO_FIX_PARTICLES has been selected, v_full will always be zero */ p->force.v_full[0] = xp->v_full[0]; p->force.v_full[1] = xp->v_full[1]; p->force.v_full[2] = xp->v_full[2]; @@ -246,40 +357,14 @@ __attribute__((always_inline)) INLINE static void hydro_reset_predicted_values( * @brief Converts the hydrodynamic variables from the initial condition file to * conserved variables that can be used during the integration * - * Requires the volume to be known. - * - * The initial condition file contains a mixture of primitive and conserved - * variables. Mass is a conserved variable, and we just copy the particle - * mass into the corresponding conserved quantity. We need the volume to - * also derive a density, which is then used to convert the internal energy - * to a pressure. However, we do not actually use these variables anymore. - * We do need to initialize the linear momentum, based on the mass and the - * velocity of the particle. + * We no longer do this, as the mass needs to be provided in the initial + * condition file, and the mass alone is enough to initialize all conserved + * variables. This is now done in hydro_first_init_part. * * @param p The particle to act upon. */ __attribute__((always_inline)) INLINE static void hydro_convert_quantities( - struct part* p, struct xpart* xp) { - - const float volume = p->geometry.volume; - const float m = p->conserved.mass; - p->primitives.rho = m / volume; - - /* first get the initial velocities, as they were overwritten in end_density - */ - p->primitives.v[0] = p->v[0]; - p->primitives.v[1] = p->v[1]; - p->primitives.v[2] = p->v[2]; - - p->conserved.momentum[0] = m * p->primitives.v[0]; - p->conserved.momentum[1] = m * p->primitives.v[1]; - p->conserved.momentum[2] = m * p->primitives.v[2]; - - p->primitives.P = - hydro_gamma_minus_one * p->conserved.energy * p->primitives.rho; - - p->conserved.energy *= m; -} + struct part* p, struct xpart* xp) {} /** * @brief Extra operations to be done during the drift @@ -303,19 +388,33 @@ __attribute__((always_inline)) INLINE static void hydro_predict_extra( else p->h *= expf(w1); - const float w2 = -hydro_dimension * w1; - if (fabsf(w2) < 0.2f) { - p->primitives.rho *= approx_expf(w2); - } else { - p->primitives.rho *= expf(w2); +/* we temporarily disabled the primitive variable drift. + This should be reenabled once gravity works, and we have time to check that + drifting works properly. */ +// const float w2 = -hydro_dimension * w1; +// if (fabsf(w2) < 0.2f) { +// p->primitives.rho *= approx_expf(w2); +// } else { +// p->primitives.rho *= expf(w2); +// } + +// p->primitives.v[0] += (p->a_hydro[0] + p->gravity.old_a[0]) * dt; +// p->primitives.v[1] += (p->a_hydro[1] + p->gravity.old_a[1]) * dt; +// p->primitives.v[2] += (p->a_hydro[2] + p->gravity.old_a[2]) * dt; + +//#if !defined(EOS_ISOTHERMAL_GAS) +// if (p->conserved.mass > 0.) { +// const float u = p->conserved.energy + p->du_dt * dt; +// p->primitives.P = +// hydro_gamma_minus_one * u * p->primitives.rho / p->conserved.mass; +// } +//#endif + +#ifdef SWIFT_DEBUG_CHECKS + if (p->h <= 0.) { + error("Zero or negative smoothing length (%g)!", p->h); } - - p->primitives.v[0] += (p->a_hydro[0] + p->gravity.old_a[0]) * dt; - p->primitives.v[1] += (p->a_hydro[1] + p->gravity.old_a[1]) * dt; - p->primitives.v[2] += (p->a_hydro[2] + p->gravity.old_a[2]) * dt; - const float u = p->conserved.energy + p->du_dt * dt; - p->primitives.P = - hydro_gamma_minus_one * u * p->primitives.rho / p->conserved.mass; +#endif } /** @@ -333,35 +432,24 @@ __attribute__((always_inline)) INLINE static void hydro_predict_extra( __attribute__((always_inline)) INLINE static void hydro_end_force( struct part* p) { + /* set the variables that are used to drift the primitive variables */ + /* Add normalization to h_dt. */ p->force.h_dt *= p->h * hydro_dimension_inv; - /* Set the hydro acceleration, based on the new momentum and mass */ - /* NOTE: the momentum and mass are only correct for active particles, since - only active particles have received flux contributions from all their - neighbours. Since this method is only called for active particles, - this is indeed the case. */ if (p->force.dt) { - float mnew; - float vnew[3]; - - mnew = p->conserved.mass + p->conserved.flux.mass; - vnew[0] = (p->conserved.momentum[0] + p->conserved.flux.momentum[0]) / mnew; - vnew[1] = (p->conserved.momentum[1] + p->conserved.flux.momentum[1]) / mnew; - vnew[2] = (p->conserved.momentum[2] + p->conserved.flux.momentum[2]) / mnew; - - p->a_hydro[0] = (vnew[0] - p->force.v_full[0]) / p->force.dt; - p->a_hydro[1] = (vnew[1] - p->force.v_full[1]) / p->force.dt; - p->a_hydro[2] = (vnew[2] - p->force.v_full[2]) / p->force.dt; - p->du_dt = p->conserved.flux.energy / p->force.dt; } else { - p->a_hydro[0] = 0.0f; - p->a_hydro[1] = 0.0f; - p->a_hydro[2] = 0.0f; - p->du_dt = 0.0f; } + +#if defined(GIZMO_FIX_PARTICLES) + p->du_dt = 0.0f; + + /* disable the smoothing length update, since the smoothing lengths should + stay the same for all steps (particles don't move) */ + p->force.h_dt = 0.0f; +#endif } /** @@ -377,63 +465,54 @@ __attribute__((always_inline)) INLINE static void hydro_end_force( __attribute__((always_inline)) INLINE static void hydro_kick_extra( struct part* p, struct xpart* xp, float dt) { - float oldm, oldp[3], anew[3]; - const float half_dt = 0.5f * dt; // MATTHIEU - - /* Retrieve the current value of the gravitational acceleration from the - gpart. We are only allowed to do this because this is the kick. We still - need to check whether gpart exists though.*/ - if (p->gpart) { - anew[0] = p->gpart->a_grav[0]; - anew[1] = p->gpart->a_grav[1]; - anew[2] = p->gpart->a_grav[2]; - - /* Copy the old mass and momentum before updating the conserved variables */ - oldm = p->conserved.mass; - oldp[0] = p->conserved.momentum[0]; - oldp[1] = p->conserved.momentum[1]; - oldp[2] = p->conserved.momentum[2]; - } + float a_grav[3]; /* Update conserved variables. */ p->conserved.mass += p->conserved.flux.mass; p->conserved.momentum[0] += p->conserved.flux.momentum[0]; p->conserved.momentum[1] += p->conserved.flux.momentum[1]; p->conserved.momentum[2] += p->conserved.flux.momentum[2]; +#if defined(EOS_ISOTHERMAL_GAS) + p->conserved.energy = p->conserved.mass * const_isothermal_internal_energy; +#else p->conserved.energy += p->conserved.flux.energy; +#endif /* Add gravity. We only do this if we have gravity activated. */ if (p->gpart) { - p->conserved.momentum[0] += - half_dt * (oldm * p->gravity.old_a[0] + p->conserved.mass * anew[0]); - p->conserved.momentum[1] += - half_dt * (oldm * p->gravity.old_a[1] + p->conserved.mass * anew[1]); - p->conserved.momentum[2] += - half_dt * (oldm * p->gravity.old_a[2] + p->conserved.mass * anew[2]); - - float paold, panew; - paold = oldp[0] * p->gravity.old_a[0] + oldp[1] * p->gravity.old_a[1] + - oldp[2] * p->gravity.old_a[2]; - panew = p->conserved.momentum[0] * anew[0] + - p->conserved.momentum[1] * anew[1] + - p->conserved.momentum[2] * anew[2]; - p->conserved.energy += half_dt * (paold + panew); - - float fluxaold, fluxanew; - fluxaold = p->gravity.old_a[0] * p->gravity.old_mflux[0] + - p->gravity.old_a[1] * p->gravity.old_mflux[1] + - p->gravity.old_a[2] * p->gravity.old_mflux[2]; - fluxanew = anew[0] * p->gravity.mflux[0] + anew[1] * p->gravity.mflux[1] + - anew[2] * p->gravity.mflux[2]; - p->conserved.energy += half_dt * (fluxaold + fluxanew); - - /* Store gravitational acceleration and mass flux for next step */ - p->gravity.old_a[0] = anew[0]; - p->gravity.old_a[1] = anew[1]; - p->gravity.old_a[2] = anew[2]; - p->gravity.old_mflux[0] = p->gravity.mflux[0]; - p->gravity.old_mflux[1] = p->gravity.mflux[1]; - p->gravity.old_mflux[2] = p->gravity.mflux[2]; + /* Retrieve the current value of the gravitational acceleration from the + gpart. We are only allowed to do this because this is the kick. We still + need to check whether gpart exists though.*/ + a_grav[0] = p->gpart->a_grav[0]; + a_grav[1] = p->gpart->a_grav[1]; + a_grav[2] = p->gpart->a_grav[2]; + + /* Store the gravitational acceleration for later use. */ + /* This is currently only used for output purposes. */ + p->gravity.old_a[0] = a_grav[0]; + p->gravity.old_a[1] = a_grav[1]; + p->gravity.old_a[2] = a_grav[2]; + + /* Make sure the gpart knows the mass has changed. */ + p->gpart->mass = p->conserved.mass; + + /* Kick the momentum for half a time step */ + /* Note that this also affects the particle movement, as the velocity for + the particles is set after this. */ + p->conserved.momentum[0] += dt * p->conserved.mass * a_grav[0]; + p->conserved.momentum[1] += dt * p->conserved.mass * a_grav[1]; + p->conserved.momentum[2] += dt * p->conserved.mass * a_grav[2]; + +#if !defined(EOS_ISOTHERMAL_GAS) && defined(GIZMO_TOTAL_ENERGY) + /* This part still needs to be tested! */ + p->conserved.energy += dt * (p->conserved.momentum[0] * a_grav[0] + + p->conserved.momentum[1] * a_grav[1] + + p->conserved.momentum[2] * a_grav[2]); + + p->conserved.energy += dt * (a_grav[0] * p->gravity.mflux[0] + + a_grav[1] * p->gravity.mflux[1] + + a_grav[2] * p->gravity.mflux[2]); +#endif } /* reset fluxes */ @@ -444,6 +523,47 @@ __attribute__((always_inline)) INLINE static void hydro_kick_extra( p->conserved.flux.momentum[1] = 0.0f; p->conserved.flux.momentum[2] = 0.0f; p->conserved.flux.energy = 0.0f; + +#if defined(GIZMO_FIX_PARTICLES) + xp->v_full[0] = 0.; + xp->v_full[1] = 0.; + xp->v_full[2] = 0.; + + p->v[0] = 0.; + p->v[1] = 0.; + p->v[2] = 0.; + + if (p->gpart) { + p->gpart->v_full[0] = 0.; + p->gpart->v_full[1] = 0.; + p->gpart->v_full[2] = 0.; + } +#else + /* Set particle movement */ + if (p->conserved.mass > 0.) { + xp->v_full[0] = p->conserved.momentum[0] / p->conserved.mass; + xp->v_full[1] = p->conserved.momentum[1] / p->conserved.mass; + xp->v_full[2] = p->conserved.momentum[2] / p->conserved.mass; + } else { + /* vacuum particles don't move */ + xp->v_full[0] = 0.; + xp->v_full[1] = 0.; + xp->v_full[2] = 0.; + } + p->v[0] = xp->v_full[0]; + p->v[1] = xp->v_full[1]; + p->v[2] = xp->v_full[2]; + + /* Update gpart! */ + /* This is essential, as the gpart drift is done independently from the part + drift, and we don't want the gpart and the part to have different + positions! */ + if (p->gpart) { + p->gpart->v_full[0] = xp->v_full[0]; + p->gpart->v_full[1] = xp->v_full[1]; + p->gpart->v_full[2] = xp->v_full[2]; + } +#endif } /** @@ -454,7 +574,15 @@ __attribute__((always_inline)) INLINE static void hydro_kick_extra( __attribute__((always_inline)) INLINE static float hydro_get_internal_energy( const struct part* restrict p) { - return p->primitives.P / hydro_gamma_minus_one / p->primitives.rho; + if (p->primitives.rho > 0.) { +#ifdef EOS_ISOTHERMAL_GAS + return p->primitives.P / hydro_gamma_minus_one / p->primitives.rho; +#else + return const_isothermal_internal_energy; +#endif + } else { + return 0.; + } } /** @@ -465,7 +593,11 @@ __attribute__((always_inline)) INLINE static float hydro_get_internal_energy( __attribute__((always_inline)) INLINE static float hydro_get_entropy( const struct part* restrict p) { - return p->primitives.P / pow_gamma(p->primitives.rho); + if (p->primitives.rho > 0.) { + return p->primitives.P / pow_gamma(p->primitives.rho); + } else { + return 0.; + } } /** @@ -476,7 +608,16 @@ __attribute__((always_inline)) INLINE static float hydro_get_entropy( __attribute__((always_inline)) INLINE static float hydro_get_soundspeed( const struct part* restrict p) { - return sqrtf(hydro_gamma * p->primitives.P / p->primitives.rho); + if (p->primitives.rho > 0.) { +#ifdef EOS_ISOTHERMAL_GAS + return sqrtf(const_isothermal_internal_energy * hydro_gamma * + hydro_gamma_minus_one); +#else + return sqrtf(hydro_gamma * p->primitives.P / p->primitives.rho); +#endif + } else { + return 0.; + } } /** @@ -528,6 +669,13 @@ __attribute__((always_inline)) INLINE static void hydro_set_internal_energy( /* conserved.energy is NOT the specific energy (u), but the total thermal energy (u*m) */ p->conserved.energy = u * p->conserved.mass; +#ifdef GIZMO_TOTAL_ENERGY + /* add the kinetic energy */ + p->conserved.energy += 0.5f * p->conserved.mass * + (p->conserved.momentum[0] * p->primitives.v[0] + + p->conserved.momentum[1] * p->primitives.v[1] + + p->conserved.momentum[2] * p->primitives.v[2]); +#endif p->primitives.P = hydro_gamma_minus_one * p->primitives.rho * u; } @@ -543,7 +691,14 @@ __attribute__((always_inline)) INLINE static void hydro_set_internal_energy( __attribute__((always_inline)) INLINE static void hydro_set_entropy( struct part* restrict p, float S) { - p->conserved.energy = gas_internal_energy_from_entropy(p->primitives.rho, S) * - p->conserved.mass; - p->primitives.P = gas_pressure_from_entropy(p->primitives.rho, S); + p->conserved.energy = S * pow_gamma_minus_one(p->primitives.rho) * + hydro_one_over_gamma_minus_one * p->conserved.mass; +#ifdef GIZMO_TOTAL_ENERGY + /* add the kinetic energy */ + p->conserved.energy += 0.5f * p->conserved.mass * + (p->conserved.momentum[0] * p->primitives.v[0] + + p->conserved.momentum[1] * p->primitives.v[1] + + p->conserved.momentum[2] * p->primitives.v[2]); +#endif + p->primitives.P = S * pow_gamma(p->primitives.rho); } diff --git a/src/hydro/Gizmo/hydro_gradients.h b/src/hydro/Gizmo/hydro_gradients.h index 90448efc7adb8ccecaaa98c7388f89eaa8d16bcd..a5c1e9038d0d3de6896afe773e3193a2304a6b6b 100644 --- a/src/hydro/Gizmo/hydro_gradients.h +++ b/src/hydro/Gizmo/hydro_gradients.h @@ -1,3 +1,4 @@ + /******************************************************************************* * This file is part of SWIFT. * Copyright (c) 2016 Bert Vandenbroucke (bert.vandenbroucke@gmail.com) @@ -21,6 +22,7 @@ #define SWIFT_HYDRO_GRADIENTS_H #include "hydro_slope_limiters.h" +#include "riemann.h" #if defined(GRADIENTS_SPH) @@ -140,69 +142,144 @@ __attribute__((always_inline)) INLINE static void hydro_gradients_predict( hydro_slope_limit_face(Wi, Wj, dWi, dWj, xij_i, xij_j, r); /* time */ - dWi[0] -= 0.5 * mindt * (Wi[1] * pi->primitives.gradients.rho[0] + - Wi[2] * pi->primitives.gradients.rho[1] + - Wi[3] * pi->primitives.gradients.rho[2] + - Wi[0] * (pi->primitives.gradients.v[0][0] + - pi->primitives.gradients.v[1][1] + - pi->primitives.gradients.v[2][2])); - dWi[1] -= 0.5 * mindt * (Wi[1] * pi->primitives.gradients.v[0][0] + - Wi[2] * pi->primitives.gradients.v[0][1] + - Wi[3] * pi->primitives.gradients.v[0][2] + - pi->primitives.gradients.P[0] / Wi[0]); - dWi[2] -= 0.5 * mindt * (Wi[1] * pi->primitives.gradients.v[1][0] + - Wi[2] * pi->primitives.gradients.v[1][1] + - Wi[3] * pi->primitives.gradients.v[1][2] + - pi->primitives.gradients.P[1] / Wi[0]); - dWi[3] -= 0.5 * mindt * (Wi[1] * pi->primitives.gradients.v[2][0] + - Wi[2] * pi->primitives.gradients.v[2][1] + - Wi[3] * pi->primitives.gradients.v[2][2] + - pi->primitives.gradients.P[2] / Wi[0]); - dWi[4] -= - 0.5 * mindt * (Wi[1] * pi->primitives.gradients.P[0] + - Wi[2] * pi->primitives.gradients.P[1] + - Wi[3] * pi->primitives.gradients.P[2] + - hydro_gamma * Wi[4] * (pi->primitives.gradients.v[0][0] + - pi->primitives.gradients.v[1][1] + - pi->primitives.gradients.v[2][2])); - - dWj[0] -= 0.5 * mindt * (Wj[1] * pj->primitives.gradients.rho[0] + - Wj[2] * pj->primitives.gradients.rho[1] + - Wj[3] * pj->primitives.gradients.rho[2] + - Wj[0] * (pj->primitives.gradients.v[0][0] + - pj->primitives.gradients.v[1][1] + - pj->primitives.gradients.v[2][2])); - dWj[1] -= 0.5 * mindt * (Wj[1] * pj->primitives.gradients.v[0][0] + - Wj[2] * pj->primitives.gradients.v[0][1] + - Wj[3] * pj->primitives.gradients.v[0][2] + - pj->primitives.gradients.P[0] / Wj[0]); - dWj[2] -= 0.5 * mindt * (Wj[1] * pj->primitives.gradients.v[1][0] + - Wj[2] * pj->primitives.gradients.v[1][1] + - Wj[3] * pj->primitives.gradients.v[1][2] + - pj->primitives.gradients.P[1] / Wj[0]); - dWj[3] -= 0.5 * mindt * (Wj[1] * pj->primitives.gradients.v[2][0] + - Wj[2] * pj->primitives.gradients.v[2][1] + - Wj[3] * pj->primitives.gradients.v[2][2] + - pj->primitives.gradients.P[2] / Wj[0]); - dWj[4] -= - 0.5 * mindt * (Wj[1] * pj->primitives.gradients.P[0] + - Wj[2] * pj->primitives.gradients.P[1] + - Wj[3] * pj->primitives.gradients.P[2] + - hydro_gamma * Wj[4] * (pj->primitives.gradients.v[0][0] + - pj->primitives.gradients.v[1][1] + - pj->primitives.gradients.v[2][2])); - - Wi[0] += dWi[0]; + if (Wi[0] > 0.0f) { +#ifdef EOS_ISOTHERMAL_GAS + dWi[0] -= 0.5 * mindt * (Wi[1] * pi->primitives.gradients.rho[0] + + Wi[2] * pi->primitives.gradients.rho[1] + + Wi[3] * pi->primitives.gradients.rho[2] + + Wi[0] * (pi->primitives.gradients.v[0][0] + + pi->primitives.gradients.v[1][1] + + pi->primitives.gradients.v[2][2])); + dWi[1] -= 0.5 * mindt * + (Wi[1] * pi->primitives.gradients.v[0][0] + + Wi[2] * pi->primitives.gradients.v[0][1] + + Wi[3] * pi->primitives.gradients.v[0][2] + + const_isothermal_soundspeed * const_isothermal_soundspeed * + pi->primitives.gradients.rho[0] / Wi[0]); + dWi[2] -= 0.5 * mindt * + (Wi[1] * pi->primitives.gradients.v[1][0] + + Wi[2] * pi->primitives.gradients.v[1][1] + + Wi[3] * pi->primitives.gradients.v[1][2] + + const_isothermal_soundspeed * const_isothermal_soundspeed * + pi->primitives.gradients.rho[1] / Wi[0]); + dWi[3] -= 0.5 * mindt * + (Wi[1] * pi->primitives.gradients.v[2][0] + + Wi[2] * pi->primitives.gradients.v[2][1] + + Wi[3] * pi->primitives.gradients.v[2][2] + + const_isothermal_soundspeed * const_isothermal_soundspeed * + pi->primitives.gradients.rho[2] / Wi[0]); +/* we don't care about P in this case */ +#else + dWi[0] -= 0.5 * mindt * (Wi[1] * pi->primitives.gradients.rho[0] + + Wi[2] * pi->primitives.gradients.rho[1] + + Wi[3] * pi->primitives.gradients.rho[2] + + Wi[0] * (pi->primitives.gradients.v[0][0] + + pi->primitives.gradients.v[1][1] + + pi->primitives.gradients.v[2][2])); + dWi[1] -= 0.5 * mindt * (Wi[1] * pi->primitives.gradients.v[0][0] + + Wi[2] * pi->primitives.gradients.v[0][1] + + Wi[3] * pi->primitives.gradients.v[0][2] + + pi->primitives.gradients.P[0] / Wi[0]); + dWi[2] -= 0.5 * mindt * (Wi[1] * pi->primitives.gradients.v[1][0] + + Wi[2] * pi->primitives.gradients.v[1][1] + + Wi[3] * pi->primitives.gradients.v[1][2] + + pi->primitives.gradients.P[1] / Wi[0]); + dWi[3] -= 0.5 * mindt * (Wi[1] * pi->primitives.gradients.v[2][0] + + Wi[2] * pi->primitives.gradients.v[2][1] + + Wi[3] * pi->primitives.gradients.v[2][2] + + pi->primitives.gradients.P[2] / Wi[0]); + dWi[4] -= 0.5 * mindt * + (Wi[1] * pi->primitives.gradients.P[0] + + Wi[2] * pi->primitives.gradients.P[1] + + Wi[3] * pi->primitives.gradients.P[2] + + hydro_gamma * Wi[4] * (pi->primitives.gradients.v[0][0] + + pi->primitives.gradients.v[1][1] + + pi->primitives.gradients.v[2][2])); +#endif + } + + if (Wj[0] > 0.0f) { +#ifdef EOS_ISOTHERMAL_GAS + dWj[0] -= 0.5 * mindt * (Wj[1] * pj->primitives.gradients.rho[0] + + Wj[2] * pj->primitives.gradients.rho[1] + + Wj[3] * pj->primitives.gradients.rho[2] + + Wj[0] * (pj->primitives.gradients.v[0][0] + + pj->primitives.gradients.v[1][1] + + pj->primitives.gradients.v[2][2])); + dWj[1] -= 0.5 * mindt * + (Wj[1] * pj->primitives.gradients.v[0][0] + + Wj[2] * pj->primitives.gradients.v[0][1] + + Wj[3] * pj->primitives.gradients.v[0][2] + + const_isothermal_soundspeed * const_isothermal_soundspeed * + pj->primitives.gradients.rho[0] / Wj[0]); + dWj[2] -= 0.5 * mindt * + (Wj[1] * pj->primitives.gradients.v[1][0] + + Wj[2] * pj->primitives.gradients.v[1][1] + + Wj[3] * pj->primitives.gradients.v[1][2] + + const_isothermal_soundspeed * const_isothermal_soundspeed * + pj->primitives.gradients.rho[1] / Wj[0]); + dWj[3] -= 0.5 * mindt * + (Wj[1] * pj->primitives.gradients.v[2][0] + + Wj[2] * pj->primitives.gradients.v[2][1] + + Wj[3] * pj->primitives.gradients.v[2][2] + + const_isothermal_soundspeed * const_isothermal_soundspeed * + pj->primitives.gradients.rho[2] / Wj[0]); +#else + dWj[0] -= 0.5 * mindt * (Wj[1] * pj->primitives.gradients.rho[0] + + Wj[2] * pj->primitives.gradients.rho[1] + + Wj[3] * pj->primitives.gradients.rho[2] + + Wj[0] * (pj->primitives.gradients.v[0][0] + + pj->primitives.gradients.v[1][1] + + pj->primitives.gradients.v[2][2])); + dWj[1] -= 0.5 * mindt * (Wj[1] * pj->primitives.gradients.v[0][0] + + Wj[2] * pj->primitives.gradients.v[0][1] + + Wj[3] * pj->primitives.gradients.v[0][2] + + pj->primitives.gradients.P[0] / Wj[0]); + dWj[2] -= 0.5 * mindt * (Wj[1] * pj->primitives.gradients.v[1][0] + + Wj[2] * pj->primitives.gradients.v[1][1] + + Wj[3] * pj->primitives.gradients.v[1][2] + + pj->primitives.gradients.P[1] / Wj[0]); + dWj[3] -= 0.5 * mindt * (Wj[1] * pj->primitives.gradients.v[2][0] + + Wj[2] * pj->primitives.gradients.v[2][1] + + Wj[3] * pj->primitives.gradients.v[2][2] + + pj->primitives.gradients.P[2] / Wj[0]); + dWj[4] -= 0.5 * mindt * + (Wj[1] * pj->primitives.gradients.P[0] + + Wj[2] * pj->primitives.gradients.P[1] + + Wj[3] * pj->primitives.gradients.P[2] + + hydro_gamma * Wj[4] * (pj->primitives.gradients.v[0][0] + + pj->primitives.gradients.v[1][1] + + pj->primitives.gradients.v[2][2])); +#endif + } + + if (-dWi[0] > Wi[0]) { + Wi[0] = 0.0f; + } else { + Wi[0] += dWi[0]; + } Wi[1] += dWi[1]; Wi[2] += dWi[2]; Wi[3] += dWi[3]; - Wi[4] += dWi[4]; + if (-dWi[4] > Wi[4]) { + Wi[4] = 0.0f; + } else { + Wi[4] += dWi[4]; + } - Wj[0] += dWj[0]; + if (-dWj[0] > Wj[0]) { + Wj[0] = 0.0f; + } else { + Wj[0] += dWj[0]; + } Wj[1] += dWj[1]; Wj[2] += dWj[2]; Wj[3] += dWj[3]; - Wj[4] += dWj[4]; + if (-dWj[4] > Wj[4]) { + Wj[4] = 0.0f; + } else { + Wj[4] += dWj[4]; + } } #endif // SWIFT_HYDRO_GRADIENTS_H diff --git a/src/hydro/Gizmo/hydro_iact.h b/src/hydro/Gizmo/hydro_iact.h index aba6bd53c1c9557929426c11a0986e5f02888874..bdcea49e4fdc2a9ce17c43b99dab55d61f4095c8 100644 --- a/src/hydro/Gizmo/hydro_iact.h +++ b/src/hydro/Gizmo/hydro_iact.h @@ -231,9 +231,15 @@ __attribute__((always_inline)) INLINE static void runner_iact_fluxes_common( dtj = pj->force.dt; /* calculate the maximal signal velocity */ - if (Wi[0] && Wj[0]) { + if (Wi[0] > 0.0f && Wj[0] > 0.0f) { +#ifdef EOS_ISOTHERMAL_GAS + /* we use a value that is slightly higher than necessary, since the correct + value does not always work */ + vmax = 2.5 * const_isothermal_soundspeed; +#else vmax = sqrtf(hydro_gamma * Wi[4] / Wi[0]) + sqrtf(hydro_gamma * Wj[4] / Wj[0]); +#endif } else { vmax = 0.0f; } @@ -274,8 +280,10 @@ __attribute__((always_inline)) INLINE static void runner_iact_fluxes_common( float wi_dr = hidp1 * wi_dx; float wj_dr = hjdp1 * wj_dx; dvdr *= ri; - pi->force.h_dt -= pj->conserved.mass * dvdr / pj->primitives.rho * wi_dr; - if (mode == 1) { + if (pj->primitives.rho > 0.) { + pi->force.h_dt -= pj->conserved.mass * dvdr / pj->primitives.rho * wi_dr; + } + if (mode == 1 && pi->primitives.rho > 0.) { pj->force.h_dt -= pi->conserved.mass * dvdr / pi->primitives.rho * wj_dr; } @@ -391,6 +399,7 @@ __attribute__((always_inline)) INLINE static void runner_iact_fluxes_common( pi->conserved.flux.momentum[2] -= dti * Anorm * totflux[3]; pi->conserved.flux.energy -= dti * Anorm * totflux[4]; +#ifndef GIZMO_TOTAL_ENERGY float ekin = 0.5f * (pi->primitives.v[0] * pi->primitives.v[0] + pi->primitives.v[1] * pi->primitives.v[1] + pi->primitives.v[2] * pi->primitives.v[2]); @@ -398,6 +407,7 @@ __attribute__((always_inline)) INLINE static void runner_iact_fluxes_common( pi->conserved.flux.energy += dti * Anorm * totflux[2] * pi->primitives.v[1]; pi->conserved.flux.energy += dti * Anorm * totflux[3] * pi->primitives.v[2]; pi->conserved.flux.energy -= dti * Anorm * totflux[0] * ekin; +#endif /* here is how it works: Mode will only be 1 if both particles are ACTIVE and they are in the same @@ -412,11 +422,7 @@ __attribute__((always_inline)) INLINE static void runner_iact_fluxes_common( ==> we update particle j if (MODE IS 1) OR (j IS INACTIVE) */ - // MATTHIEU - const integertime_t pj_ti_end = 0; // get_integer_time_end(pj->time_bin); - const integertime_t pi_ti_end = 0; // get_integer_time_end(pi->time_bin); - - if (mode == 1 || pj_ti_end > pi_ti_end) { + if (mode == 1 || pj->force.active == 0) { /* Store mass flux */ mflux = dtj * Anorm * totflux[0]; pj->gravity.mflux[0] -= mflux * dx[0]; @@ -429,6 +435,7 @@ __attribute__((always_inline)) INLINE static void runner_iact_fluxes_common( pj->conserved.flux.momentum[2] += dtj * Anorm * totflux[3]; pj->conserved.flux.energy += dtj * Anorm * totflux[4]; +#ifndef GIZMO_TOTAL_ENERGY ekin = 0.5f * (pj->primitives.v[0] * pj->primitives.v[0] + pj->primitives.v[1] * pj->primitives.v[1] + pj->primitives.v[2] * pj->primitives.v[2]); @@ -436,6 +443,7 @@ __attribute__((always_inline)) INLINE static void runner_iact_fluxes_common( pj->conserved.flux.energy -= dtj * Anorm * totflux[2] * pj->primitives.v[1]; pj->conserved.flux.energy -= dtj * Anorm * totflux[3] * pj->primitives.v[2]; pj->conserved.flux.energy += dtj * Anorm * totflux[0] * ekin; +#endif } } diff --git a/src/hydro/Gizmo/hydro_io.h b/src/hydro/Gizmo/hydro_io.h index e5f221ae4345dc519a50d332131ecf296f318338..236106a1fb04cc2e5b84f997a2389d583ce17cff 100644 --- a/src/hydro/Gizmo/hydro_io.h +++ b/src/hydro/Gizmo/hydro_io.h @@ -23,6 +23,13 @@ #include "io_properties.h" #include "riemann.h" +/* Set the description of the particle movement. */ +#if defined(GIZMO_FIX_PARTICLES) +#define GIZMO_PARTICLE_MOVEMENT "Fixed particles." +#else +#define GIZMO_PARTICLE_MOVEMENT "Particles move with flow velocity." +#endif + /** * @brief Specifies which particle fields to read from a dataset * @@ -63,7 +70,15 @@ void hydro_read_particles(struct part* parts, struct io_props* list, * @return Internal energy of the particle */ float convert_u(struct engine* e, struct part* p) { - return p->primitives.P / hydro_gamma_minus_one / p->primitives.rho; + if (p->primitives.rho > 0.) { +#ifdef EOS_ISOTHERMAL_GAS + return const_isothermal_internal_energy; +#else + return p->primitives.P / hydro_gamma_minus_one / p->primitives.rho; +#endif + } else { + return 0.; + } } /** @@ -74,7 +89,11 @@ float convert_u(struct engine* e, struct part* p) { * @return Entropic function of the particle */ float convert_A(struct engine* e, struct part* p) { - return p->primitives.P / pow_gamma(p->primitives.rho); + if (p->primitives.rho > 0.) { + return p->primitives.P / pow_gamma(p->primitives.rho); + } else { + return 0.; + } } /** @@ -85,6 +104,9 @@ float convert_A(struct engine* e, struct part* p) { * @return Total energy of the particle */ float convert_Etot(struct engine* e, struct part* p) { +#ifdef GIZMO_TOTAL_ENERGY + return p->conserved.energy; +#else float momentum2; momentum2 = p->conserved.momentum[0] * p->conserved.momentum[0] + @@ -92,6 +114,7 @@ float convert_Etot(struct engine* e, struct part* p) { p->conserved.momentum[2] * p->conserved.momentum[2]; return p->conserved.energy + 0.5f * momentum2 / p->conserved.mass; +#endif } /** @@ -104,7 +127,7 @@ float convert_Etot(struct engine* e, struct part* p) { void hydro_write_particles(struct part* parts, struct io_props* list, int* num_fields) { - *num_fields = 13; + *num_fields = 14; /* List what we want to write */ list[0] = io_make_output_field("Coordinates", DOUBLE, 3, UNIT_CONV_LENGTH, @@ -135,6 +158,8 @@ void hydro_write_particles(struct part* parts, struct io_props* list, list[12] = io_make_output_field_convert_part("TotEnergy", FLOAT, 1, UNIT_CONV_ENERGY, parts, conserved.energy, convert_Etot); + list[13] = io_make_output_field("GravAcceleration", FLOAT, 3, + UNIT_CONV_ACCELERATION, parts, gravity.old_a); } /** @@ -143,18 +168,21 @@ void hydro_write_particles(struct part* parts, struct io_props* list, */ void writeSPHflavour(hid_t h_grpsph) { /* Gradient information */ - writeAttribute_s(h_grpsph, "Gradient reconstruction model", - HYDRO_GRADIENT_IMPLEMENTATION); + io_write_attribute_s(h_grpsph, "Gradient reconstruction model", + HYDRO_GRADIENT_IMPLEMENTATION); /* Slope limiter information */ - writeAttribute_s(h_grpsph, "Cell wide slope limiter model", - HYDRO_SLOPE_LIMITER_CELL_IMPLEMENTATION); - writeAttribute_s(h_grpsph, "Piecewise slope limiter model", - HYDRO_SLOPE_LIMITER_FACE_IMPLEMENTATION); + io_write_attribute_s(h_grpsph, "Cell wide slope limiter model", + HYDRO_SLOPE_LIMITER_CELL_IMPLEMENTATION); + io_write_attribute_s(h_grpsph, "Piecewise slope limiter model", + HYDRO_SLOPE_LIMITER_FACE_IMPLEMENTATION); /* Riemann solver information */ - writeAttribute_s(h_grpsph, "Riemann solver type", - RIEMANN_SOLVER_IMPLEMENTATION); + io_write_attribute_s(h_grpsph, "Riemann solver type", + RIEMANN_SOLVER_IMPLEMENTATION); + + /* Particle movement information */ + io_write_attribute_s(h_grpsph, "Particle movement", GIZMO_PARTICLE_MOVEMENT); } /** diff --git a/src/hydro/Gizmo/hydro_part.h b/src/hydro/Gizmo/hydro_part.h index f6592ca107d8d2c6970f34ebd3929e226b53a355..928011d8201f3f355b00d5fe064267d379278e64 100644 --- a/src/hydro/Gizmo/hydro_part.h +++ b/src/hydro/Gizmo/hydro_part.h @@ -178,6 +178,9 @@ struct part { /* Physical time step of the particle. */ float dt; + /* Flag keeping track of whether this is an active or inactive particle. */ + char active; + /* Actual velocity of the particle. */ float v_full[3]; diff --git a/src/hydro/Minimal/hydro.h b/src/hydro/Minimal/hydro.h index 20856b7e038855e22aa3776a74ba9f495ff6c93f..56078a82569fb0bc30347d5c01831e9eecfd48f4 100644 --- a/src/hydro/Minimal/hydro.h +++ b/src/hydro/Minimal/hydro.h @@ -165,6 +165,16 @@ __attribute__((always_inline)) INLINE static float hydro_compute_timestep( return dt_cfl; } +/** + * @brief Does some extra hydro operations once the actual physical time step + * for the particle is known. + * + * @param p The particle to act upon. + * @param dt Physical time step of the particle during the next step. + */ +__attribute__((always_inline)) INLINE static void hydro_timestep_extra( + struct part *p, float dt) {} + /** * @brief Prepares a particle for the density calculation. * diff --git a/src/hydro/Minimal/hydro_io.h b/src/hydro/Minimal/hydro_io.h index 8c83349a3e17d6b3375663698af7beeeab0636bc..2ec0cb11d1e3ccaaa09d9822c75b396364912df8 100644 --- a/src/hydro/Minimal/hydro_io.h +++ b/src/hydro/Minimal/hydro_io.h @@ -123,13 +123,13 @@ void writeSPHflavour(hid_t h_grpsph) { /* Viscosity and thermal conduction */ /* Nothing in this minimal model... */ - writeAttribute_s(h_grpsph, "Thermal Conductivity Model", "No treatment"); - writeAttribute_s(h_grpsph, "Viscosity Model", - "Minimal treatment as in Monaghan (1992)"); + io_write_attribute_s(h_grpsph, "Thermal Conductivity Model", "No treatment"); + io_write_attribute_s(h_grpsph, "Viscosity Model", + "Minimal treatment as in Monaghan (1992)"); /* Time integration properties */ - writeAttribute_f(h_grpsph, "Maximal Delta u change over dt", - const_max_u_change); + io_write_attribute_f(h_grpsph, "Maximal Delta u change over dt", + const_max_u_change); } /** diff --git a/src/hydro/PressureEntropy/hydro.h b/src/hydro/PressureEntropy/hydro.h index f22bb8a13a8ba4d896a77bd4c4f5e86bed5a5960..20238896f1458d0abebacca4865968a3a671c886 100644 --- a/src/hydro/PressureEntropy/hydro.h +++ b/src/hydro/PressureEntropy/hydro.h @@ -152,6 +152,16 @@ __attribute__((always_inline)) INLINE static float hydro_compute_timestep( return dt_cfl; } +/** + * @brief Does some extra hydro operations once the actual physical time step + * for the particle is known. + * + * @param p The particle to act upon. + * @param dt Physical time step of the particle during the next step. + */ +__attribute__((always_inline)) INLINE static void hydro_timestep_extra( + struct part *p, float dt) {} + /** * @brief Prepares a particle for the density calculation. * diff --git a/src/hydro/PressureEntropy/hydro_io.h b/src/hydro/PressureEntropy/hydro_io.h index fcc8439f64d299b7dcb59e819f8dd273112ce25a..10243750c01d6bc64664f9834bc4cc245c786f49 100644 --- a/src/hydro/PressureEntropy/hydro_io.h +++ b/src/hydro/PressureEntropy/hydro_io.h @@ -123,16 +123,16 @@ void writeSPHflavour(hid_t h_grpsph) { /* Viscosity and thermal conduction */ /* Nothing in this minimal model... */ - writeAttribute_s(h_grpsph, "Thermal Conductivity Model", "No treatment"); - writeAttribute_s( + io_write_attribute_s(h_grpsph, "Thermal Conductivity Model", "No treatment"); + io_write_attribute_s( h_grpsph, "Viscosity Model", "as in Springel (2005), i.e. Monaghan (1992) with Balsara (1995) switch"); - writeAttribute_f(h_grpsph, "Viscosity alpha", const_viscosity_alpha); - writeAttribute_f(h_grpsph, "Viscosity beta", 3.f); + io_write_attribute_f(h_grpsph, "Viscosity alpha", const_viscosity_alpha); + io_write_attribute_f(h_grpsph, "Viscosity beta", 3.f); /* Time integration properties */ - writeAttribute_f(h_grpsph, "Maximal Delta u change over dt", - const_max_u_change); + io_write_attribute_f(h_grpsph, "Maximal Delta u change over dt", + const_max_u_change); } /** diff --git a/src/hydro_properties.c b/src/hydro_properties.c index 815969975b4f5e6b39099e71bbbec4e43c875ddc..46785b4b2d5b958f6db3bd9813d139575217d6fe 100644 --- a/src/hydro_properties.c +++ b/src/hydro_properties.c @@ -34,6 +34,7 @@ #define hydro_props_default_max_iterations 30 #define hydro_props_default_volume_change 2.0f +#define hydro_props_default_h_max FLT_MAX void hydro_props_init(struct hydro_props *p, const struct swift_params *params) { @@ -43,7 +44,11 @@ void hydro_props_init(struct hydro_props *p, p->target_neighbours = pow_dimension(p->eta_neighbours) * kernel_norm; p->delta_neighbours = parser_get_param_float(params, "SPH:delta_neighbours"); - /* Ghost stuff */ + /* Maximal smoothing length */ + p->h_max = parser_get_opt_param_float(params, "SPH:h_max", + hydro_props_default_h_max); + + /* Number of iterations to converge h */ p->max_smoothing_iterations = parser_get_opt_param_int( params, "SPH:max_ghost_iterations", hydro_props_default_max_iterations); @@ -81,6 +86,9 @@ void hydro_props_print(const struct hydro_props *p) { "(max|dlog(h)/dt|=%f).", pow_dimension(expf(p->log_max_h_change)), p->log_max_h_change); + if (p->h_max != hydro_props_default_h_max) + message("Maximal smoothing length allowed: %.4f", p->h_max); + if (p->max_smoothing_iterations != hydro_props_default_max_iterations) message("Maximal iterations in ghost task set to %d (default is %d)", p->max_smoothing_iterations, hydro_props_default_max_iterations); @@ -89,18 +97,20 @@ void hydro_props_print(const struct hydro_props *p) { #if defined(HAVE_HDF5) void hydro_props_print_snapshot(hid_t h_grpsph, const struct hydro_props *p) { - writeAttribute_f(h_grpsph, "Adiabatic index", hydro_gamma); - writeAttribute_i(h_grpsph, "Dimension", (int)hydro_dimension); - writeAttribute_s(h_grpsph, "Scheme", SPH_IMPLEMENTATION); - writeAttribute_s(h_grpsph, "Kernel function", kernel_name); - writeAttribute_f(h_grpsph, "Kernel target N_ngb", p->target_neighbours); - writeAttribute_f(h_grpsph, "Kernel delta N_ngb", p->delta_neighbours); - writeAttribute_f(h_grpsph, "Kernel eta", p->eta_neighbours); - writeAttribute_f(h_grpsph, "CFL parameter", p->CFL_condition); - writeAttribute_f(h_grpsph, "Volume log(max(delta h))", p->log_max_h_change); - writeAttribute_f(h_grpsph, "Volume max change time-step", - pow_dimension(expf(p->log_max_h_change))); - writeAttribute_i(h_grpsph, "Max ghost iterations", - p->max_smoothing_iterations); + io_write_attribute_f(h_grpsph, "Adiabatic index", hydro_gamma); + io_write_attribute_i(h_grpsph, "Dimension", (int)hydro_dimension); + io_write_attribute_s(h_grpsph, "Scheme", SPH_IMPLEMENTATION); + io_write_attribute_s(h_grpsph, "Kernel function", kernel_name); + io_write_attribute_f(h_grpsph, "Kernel target N_ngb", p->target_neighbours); + io_write_attribute_f(h_grpsph, "Kernel delta N_ngb", p->delta_neighbours); + io_write_attribute_f(h_grpsph, "Kernel eta", p->eta_neighbours); + io_write_attribute_f(h_grpsph, "Maximal smoothing length", p->h_max); + io_write_attribute_f(h_grpsph, "CFL parameter", p->CFL_condition); + io_write_attribute_f(h_grpsph, "Volume log(max(delta h))", + p->log_max_h_change); + io_write_attribute_f(h_grpsph, "Volume max change time-step", + pow_dimension(expf(p->log_max_h_change))); + io_write_attribute_i(h_grpsph, "Max ghost iterations", + p->max_smoothing_iterations); } #endif diff --git a/src/hydro_properties.h b/src/hydro_properties.h index 6b151e2d038fc1ac30e77ad70bc9ef714cec2899..716c4c060c21eb95d05f9d50e13d4681a958a6fd 100644 --- a/src/hydro_properties.h +++ b/src/hydro_properties.h @@ -28,7 +28,6 @@ #endif /* Local includes. */ -#include "const.h" #include "parser.h" /** @@ -41,17 +40,15 @@ struct hydro_props { float target_neighbours; float delta_neighbours; - /* Kernel properties */ + /* Maximal smoothing length */ + float h_max; + + /* Number of iterations to converge h */ int max_smoothing_iterations; /* Time integration properties */ float CFL_condition; float log_max_h_change; - -/* Viscosity parameters */ -#ifdef GADGET_SPH - float const_viscosity_alpha; -#endif }; void hydro_props_print(const struct hydro_props *p); diff --git a/src/io_properties.h b/src/io_properties.h index af0d81aec8cf4901d2bfcce8cd023a2e04b804bf..9fcf1a1ac67cae6afab6870369e51d06c752fc11 100644 --- a/src/io_properties.h +++ b/src/io_properties.h @@ -38,7 +38,7 @@ struct io_props { char name[FIELD_BUFFER_SIZE]; /* Type of the field */ - enum DATA_TYPE type; + enum IO_DATA_TYPE type; /* Dimension (1D, 3D, ...) */ int dimension; @@ -47,7 +47,7 @@ struct io_props { enum DATA_IMPORTANCE importance; /* Units of the quantity */ - enum UnitConversionFactor units; + enum unit_conversion_factor units; /* Pointer to the field of the first particle in the array */ char* field; @@ -87,9 +87,9 @@ struct io_props { * Do not call this function directly. Use the macro defined above. */ struct io_props io_make_input_field_(char name[FIELD_BUFFER_SIZE], - enum DATA_TYPE type, int dimension, + enum IO_DATA_TYPE type, int dimension, enum DATA_IMPORTANCE importance, - enum UnitConversionFactor units, + enum unit_conversion_factor units, char* field, size_t partSize) { struct io_props r; strcpy(r.name, name); @@ -127,8 +127,8 @@ struct io_props io_make_input_field_(char name[FIELD_BUFFER_SIZE], * Do not call this function directly. Use the macro defined above. */ struct io_props io_make_output_field_(char name[FIELD_BUFFER_SIZE], - enum DATA_TYPE type, int dimension, - enum UnitConversionFactor units, + enum IO_DATA_TYPE type, int dimension, + enum unit_conversion_factor units, char* field, size_t partSize) { struct io_props r; strcpy(r.name, name); @@ -170,8 +170,8 @@ struct io_props io_make_output_field_(char name[FIELD_BUFFER_SIZE], * Do not call this function directly. Use the macro defined above. */ struct io_props io_make_output_field_convert_part_( - char name[FIELD_BUFFER_SIZE], enum DATA_TYPE type, int dimension, - enum UnitConversionFactor units, char* field, size_t partSize, + char name[FIELD_BUFFER_SIZE], enum IO_DATA_TYPE type, int dimension, + enum unit_conversion_factor units, char* field, size_t partSize, struct part* parts, float (*functionPtr)(struct engine*, struct part*)) { struct io_props r; @@ -214,8 +214,8 @@ struct io_props io_make_output_field_convert_part_( * Do not call this function directly. Use the macro defined above. */ struct io_props io_make_output_field_convert_gpart_( - char name[FIELD_BUFFER_SIZE], enum DATA_TYPE type, int dimension, - enum UnitConversionFactor units, char* field, size_t partSize, + char name[FIELD_BUFFER_SIZE], enum IO_DATA_TYPE type, int dimension, + enum unit_conversion_factor units, char* field, size_t partSize, struct gpart* gparts, float (*functionPtr)(struct engine*, struct gpart*)) { struct io_props r; diff --git a/src/kick.h b/src/kick.h index b7dea7ffe9593d92a1eeef38b878c57328bad083..7ccea7d26974297cfebc605808c4443633140ec1 100644 --- a/src/kick.h +++ b/src/kick.h @@ -25,6 +25,7 @@ /* Local headers. */ #include "const.h" #include "debug.h" +#include "stars.h" #include "timeline.h" /** @@ -42,6 +43,16 @@ __attribute__((always_inline)) INLINE static void kick_gpart( /* Time interval for this half-kick */ const float dt = (ti_end - ti_start) * timeBase; +#ifdef SWIFT_DEBUG_CHECKS + if (gp->ti_kick != ti_start) + error( + "g-particle has not been kicked to the current time gp->ti_kick=%lld, " + "ti_start=%lld, ti_end=%lld", + gp->ti_kick, ti_start, ti_end); + + gp->ti_kick = ti_end; +#endif + /* Kick particles in momentum space */ gp->v_full[0] += gp->a_grav[0] * dt; gp->v_full[1] += gp->a_grav[1] * dt; @@ -70,7 +81,7 @@ __attribute__((always_inline)) INLINE static void kick_part( #ifdef SWIFT_DEBUG_CHECKS if (p->ti_kick != ti_start) error( - "Particle has not been kicked to the current time p->ti_kick=%lld, " + "particle has not been kicked to the current time p->ti_kick=%lld, " "ti_start=%lld, ti_end=%lld", p->ti_kick, ti_start, ti_end); @@ -100,4 +111,45 @@ __attribute__((always_inline)) INLINE static void kick_part( if (p->gpart != NULL) gravity_kick_extra(p->gpart, dt); } +/** + * @brief Perform the 'kick' operation on a #spart + * + * @param sp The #spart to kick. + * @param ti_start The starting (integer) time of the kick + * @param ti_end The ending (integer) time of the kick + * @param timeBase The minimal allowed time-step size. + */ +__attribute__((always_inline)) INLINE static void kick_spart( + struct spart *restrict sp, integertime_t ti_start, integertime_t ti_end, + double timeBase) { + + /* Time interval for this half-kick */ + const float dt = (ti_end - ti_start) * timeBase; + +#ifdef SWIFT_DEBUG_CHECKS + if (sp->ti_kick != ti_start) + error( + "s-particle has not been kicked to the current time sp->ti_kick=%lld, " + "ti_start=%lld, ti_end=%lld", + sp->ti_kick, ti_start, ti_end); + + sp->ti_kick = ti_end; +#endif + + /* Acceleration from gravity */ + const float a[3] = {sp->gpart->a_grav[0], sp->gpart->a_grav[1], + sp->gpart->a_grav[2]}; + + /* Kick particles in momentum space */ + sp->v[0] += a[0] * dt; + sp->v[1] += a[1] * dt; + sp->v[2] += a[2] * dt; + sp->gpart->v_full[0] = sp->v[0]; + sp->gpart->v_full[1] = sp->v[1]; + sp->gpart->v_full[2] = sp->v[2]; + + /* Kick extra variables */ + star_kick_extra(sp, dt); +} + #endif /* SWIFT_KICK_H */ diff --git a/src/multipole.c b/src/multipole.c index f0d0c7084fd9bec366f2185cb24887da40591b17..bd5c6d6546fa0546108dcd53d7fe4060293c37a7 100644 --- a/src/multipole.c +++ b/src/multipole.c @@ -20,205 +20,3 @@ /* Config parameters. */ #include "../config.h" - -/* Some standard headers. */ -#include <strings.h> - -/* This object's header. */ -#include "multipole.h" - -/** -* @brief Reset the data of a #multipole. -* -* @param m The #multipole. -*/ -void multipole_reset(struct multipole *m) { - - /* Just bzero the struct. */ - bzero(m, sizeof(struct multipole)); -} - -/** -* @brief Init a multipole from a set of particles. -* -* @param m The #multipole. -* @param gparts The #gpart. -* @param gcount The number of particles. -*/ -void multipole_init(struct multipole *m, const struct gpart *gparts, - int gcount) { - -#if const_gravity_multipole_order > 2 -#error "Multipoles of order >2 not yet implemented." -#endif - - /* Zero everything */ - multipole_reset(m); - - /* Temporary variables */ - double mass = 0.0; - double com[3] = {0.0, 0.0, 0.0}; - -#if const_gravity_multipole_order >= 2 - double I_xx = 0.0, I_yy = 0.0, I_zz = 0.0; - double I_xy = 0.0, I_xz = 0.0, I_yz = 0.0; -#endif - - /* Collect the particle data. */ - for (int k = 0; k < gcount; k++) { - const float w = gparts[k].mass; - - mass += w; - com[0] += gparts[k].x[0] * w; - com[1] += gparts[k].x[1] * w; - com[2] += gparts[k].x[2] * w; - -#if const_gravity_multipole_order >= 2 - I_xx += gparts[k].x[0] * gparts[k].x[0] * w; - I_yy += gparts[k].x[1] * gparts[k].x[1] * w; - I_zz += gparts[k].x[2] * gparts[k].x[2] * w; - I_xy += gparts[k].x[0] * gparts[k].x[1] * w; - I_xz += gparts[k].x[0] * gparts[k].x[2] * w; - I_yz += gparts[k].x[1] * gparts[k].x[2] * w; -#endif - } - - const double imass = 1.0 / mass; - - /* Store the data on the multipole. */ - m->mass = mass; - m->CoM[0] = com[0] * imass; - m->CoM[1] = com[1] * imass; - m->CoM[2] = com[2] * imass; - -#if const_gravity_multipole_order >= 2 - m->I_xx = I_xx - imass * com[0] * com[0]; - m->I_yy = I_yy - imass * com[1] * com[1]; - m->I_zz = I_zz - imass * com[2] * com[2]; - m->I_xy = I_xy - imass * com[0] * com[1]; - m->I_xz = I_xz - imass * com[0] * com[2]; - m->I_yz = I_yz - imass * com[1] * com[2]; -#endif -} - -/** - * @brief Add the second multipole to the first one. - * - * @param ma The #multipole which will contain the sum. - * @param mb The #multipole to add. - */ - -void multipole_add(struct multipole *ma, const struct multipole *mb) { - -#if const_gravity_multipole_order > 2 -#error "Multipoles of order >2 not yet implemented." -#endif - - /* Correct the position. */ - const double ma_mass = ma->mass; - const double mb_mass = mb->mass; - const double M_tot = ma_mass + mb_mass; - const double M_tot_inv = 1.0 / M_tot; - - const double ma_CoM[3] = {ma->CoM[0], ma->CoM[1], ma->CoM[2]}; - const double mb_CoM[3] = {mb->CoM[0], mb->CoM[1], mb->CoM[2]}; - -#if const_gravity_multipole_order >= 2 - const double ma_I_xx = (double)ma->I_xx + ma_mass * ma_CoM[0] * ma_CoM[0]; - const double ma_I_yy = (double)ma->I_yy + ma_mass * ma_CoM[1] * ma_CoM[1]; - const double ma_I_zz = (double)ma->I_zz + ma_mass * ma_CoM[2] * ma_CoM[2]; - const double ma_I_xy = (double)ma->I_xy + ma_mass * ma_CoM[0] * ma_CoM[1]; - const double ma_I_xz = (double)ma->I_xz + ma_mass * ma_CoM[0] * ma_CoM[2]; - const double ma_I_yz = (double)ma->I_yz + ma_mass * ma_CoM[1] * ma_CoM[2]; - - const double mb_I_xx = (double)mb->I_xx + mb_mass * mb_CoM[0] * mb_CoM[0]; - const double mb_I_yy = (double)mb->I_yy + mb_mass * mb_CoM[1] * mb_CoM[1]; - const double mb_I_zz = (double)mb->I_zz + mb_mass * mb_CoM[2] * mb_CoM[2]; - const double mb_I_xy = (double)mb->I_xy + mb_mass * mb_CoM[0] * mb_CoM[1]; - const double mb_I_xz = (double)mb->I_xz + mb_mass * mb_CoM[0] * mb_CoM[2]; - const double mb_I_yz = (double)mb->I_yz + mb_mass * mb_CoM[1] * mb_CoM[2]; -#endif - - /* New mass */ - ma->mass = M_tot; - - /* New CoM */ - ma->CoM[0] = (ma_CoM[0] * ma_mass + mb_CoM[0] * mb_mass) * M_tot_inv; - ma->CoM[1] = (ma_CoM[1] * ma_mass + mb_CoM[1] * mb_mass) * M_tot_inv; - ma->CoM[2] = (ma_CoM[2] * ma_mass + mb_CoM[2] * mb_mass) * M_tot_inv; - -/* New quadrupole */ -#if const_gravity_multipole_order >= 2 - ma->I_xx = (ma_I_xx + mb_I_xx) - M_tot * ma->CoM[0] * ma->CoM[0]; - ma->I_yy = (ma_I_yy + mb_I_yy) - M_tot * ma->CoM[1] * ma->CoM[1]; - ma->I_zz = (ma_I_zz + mb_I_zz) - M_tot * ma->CoM[2] * ma->CoM[2]; - ma->I_xy = (ma_I_xy + mb_I_xy) - M_tot * ma->CoM[0] * ma->CoM[1]; - ma->I_xz = (ma_I_xz + mb_I_xz) - M_tot * ma->CoM[0] * ma->CoM[2]; - ma->I_yz = (ma_I_yz + mb_I_yz) - M_tot * ma->CoM[1] * ma->CoM[2]; -#endif -} - -/** - * @brief Add a particle to the given multipole. - * - * @param m The #multipole. - * @param p The #gpart. - */ - -void multipole_addpart(struct multipole *m, struct gpart *p) { - - /* #if const_gravity_multipole_order == 1 */ - - /* /\* Correct the position. *\/ */ - /* float mm = m->coeffs[0], mp = p->mass; */ - /* float w = 1.0f / (mm + mp); */ - /* for (int k = 0; k < 3; k++) m->x[k] = (m->x[k] * mm + p->x[k] * mp) * w; - */ - - /* /\* Add the particle to the moments. *\/ */ - /* m->coeffs[0] = mm + mp; */ - - /* #else */ - /* #error( "Multipoles of order %i not yet implemented." , - * const_gravity_multipole_order ) - */ - /* #endif */ -} - -/** - * @brief Add a group of particles to the given multipole. - * - * @param m The #multipole. - * @param p The #gpart array. - * @param N Number of parts to add. - */ - -void multipole_addparts(struct multipole *m, struct gpart *p, int N) { - - /* #if const_gravity_multipole_order == 1 */ - - /* /\* Get the combined mass and positions. *\/ */ - /* double xp[3] = {0.0, 0.0, 0.0}; */ - /* float mp = 0.0f, w; */ - /* for (int k = 0; k < N; k++) { */ - /* w = p[k].mass; */ - /* mp += w; */ - /* xp[0] += p[k].x[0] * w; */ - /* xp[1] += p[k].x[1] * w; */ - /* xp[2] += p[k].x[2] * w; */ - /* } */ - - /* /\* Correct the position. *\/ */ - /* float mm = m->coeffs[0]; */ - /* w = 1.0f / (mm + mp); */ - /* for (int k = 0; k < 3; k++) m->x[k] = (m->x[k] * mm + xp[k]) * w; */ - - /* /\* Add the particle to the moments. *\/ */ - /* m->coeffs[0] = mm + mp; */ - - /* #else */ - /* #error( "Multipoles of order %i not yet implemented." , - * const_gravity_multipole_order ) - */ - /* #endif */ -} diff --git a/src/multipole.h b/src/multipole.h index dc1f914dd73969bc63f3beffca7f34d0f889edb6..b5c9335ee8fabf46740cefe310fcfecbea3fd77e 100644 --- a/src/multipole.h +++ b/src/multipole.h @@ -22,34 +22,359 @@ /* Some standard headers. */ #include <math.h> +#include <string.h> /* Includes. */ +//#include "active.h" +#include "align.h" #include "const.h" +#include "error.h" +#include "gravity_derivatives.h" #include "inline.h" #include "kernel_gravity.h" +#include "minmax.h" #include "part.h" -/* Multipole struct. */ +#define multipole_align 128 + +struct acc_tensor { + + double F_000; +}; + +struct pot_tensor { + + double F_000; +}; + struct multipole { - /* Multipole location. */ + float mass; + + /*! Bulk velocity */ + float vel[3]; +}; + +/** + * @brief The multipole expansion of a mass distribution. + */ +struct gravity_tensors { + + /*! Linking pointer for "memory management". */ + struct gravity_tensors *next; + + /*! Centre of mass of the matter dsitribution */ double CoM[3]; - /* Multipole mass */ - float mass; + /*! The actual content */ + struct { + + /*! Multipole mass */ + struct multipole m_pole; + + /*! Field tensor for acceleration along x */ + struct acc_tensor a_x; + + /*! Field tensor for acceleration along y */ + struct acc_tensor a_y; + + /*! Field tensor for acceleration along z */ + struct acc_tensor a_z; + + /*! Field tensor for the potential */ + struct pot_tensor pot; + +#ifdef SWIFT_DEBUG_CHECKS + + /* Total mass this gpart interacted with */ + double mass_interacted; + +#endif + }; +} SWIFT_STRUCT_ALIGN; + +/** + * @brief Reset the data of a #multipole. + * + * @param m The #multipole. + */ +INLINE static void gravity_reset(struct gravity_tensors *m) { + + /* Just bzero the struct. */ + bzero(m, sizeof(struct gravity_tensors)); +} + +INLINE static void gravity_field_tensor_init(struct gravity_tensors *m) { + + bzero(&m->a_x, sizeof(struct acc_tensor)); + bzero(&m->a_y, sizeof(struct acc_tensor)); + bzero(&m->a_z, sizeof(struct acc_tensor)); + bzero(&m->pot, sizeof(struct pot_tensor)); + +#ifdef SWIFT_DEBUG_CHECKS + m->mass_interacted = 0.; +#endif +} + +/** + * @brief Prints the content of a #multipole to stdout. + * + * Note: Uses directly printf(), not a call to message(). + * + * @param m The #multipole to print. + */ +INLINE static void gravity_multipole_print(const struct multipole *m) { + + // printf("CoM= [%12.5e %12.5e %12.5e\n", m->CoM[0], m->CoM[1], m->CoM[2]); + printf("Mass= %12.5e\n", m->mass); + printf("Vel= [%12.5e %12.5e %12.5e\n", m->vel[0], m->vel[1], m->vel[2]); +} + +/** + * @brief Adds a #multipole to another one (i.e. does ma += mb). + * + * @param ma The multipole to add to. + * @param mb The multipole to add. + */ +INLINE static void gravity_multipole_add(struct multipole *ma, + const struct multipole *mb) { + + const float mass = ma->mass + mb->mass; + const float imass = 1.f / mass; + + /* Add the bulk velocities */ + ma->vel[0] = (ma->vel[0] * ma->mass + mb->vel[0] * mb->mass) * imass; + ma->vel[1] = (ma->vel[1] * ma->mass + mb->vel[1] * mb->mass) * imass; + ma->vel[2] = (ma->vel[2] * ma->mass + mb->vel[2] * mb->mass) * imass; + + /* Add the masses */ + ma->mass = mass; +} + +/** + * @brief Verifies whether two #multipole's are equal or not. + * + * @param ma The first #multipole. + * @param mb The second #multipole. + * @param tolerance The maximal allowed difference for the fields. + * @return 1 if the multipoles are equal 0 otherwise. + */ +INLINE static int gravity_multipole_equal(const struct multipole *ma, + const struct multipole *mb, + double tolerance) { + + /* Check CoM */ + /* if (fabs(ma->CoM[0] - mb->CoM[0]) / fabs(ma->CoM[0] + mb->CoM[0]) > + * tolerance) */ + /* return 0; */ + /* if (fabs(ma->CoM[1] - mb->CoM[1]) / fabs(ma->CoM[1] + mb->CoM[1]) > + * tolerance) */ + /* return 0; */ + /* if (fabs(ma->CoM[2] - mb->CoM[2]) / fabs(ma->CoM[2] + mb->CoM[2]) > + * tolerance) */ + /* return 0; */ + + /* Check bulk velocity (if non-zero)*/ + if (fabsf(ma->vel[0] + mb->vel[0]) > 0.f && + fabsf(ma->vel[0] - mb->vel[0]) / fabsf(ma->vel[0] + mb->vel[0]) > + tolerance) + return 0; + if (fabsf(ma->vel[1] + mb->vel[1]) > 0.f && + fabsf(ma->vel[1] - mb->vel[1]) / fabsf(ma->vel[1] + mb->vel[1]) > + tolerance) + return 0; + if (fabsf(ma->vel[2] + mb->vel[2]) > 0.f && + fabsf(ma->vel[2] - mb->vel[2]) / fabsf(ma->vel[2] + mb->vel[2]) > + tolerance) + return 0; + + /* Check mass */ + if (fabsf(ma->mass - mb->mass) / fabsf(ma->mass + mb->mass) > tolerance) + return 0; + + /* All is good */ + return 1; +} + +/** + * @brief Drifts a #multipole forward in time. + * + * @param m The #multipole. + * @param dt The drift time-step. + */ +INLINE static void gravity_multipole_drift(struct gravity_tensors *m, + double dt) { + + /* Move the whole thing according to bulk motion */ + m->CoM[0] += m->m_pole.vel[0]; + m->CoM[1] += m->m_pole.vel[1]; + m->CoM[2] += m->m_pole.vel[2]; +} + +/** + * @brief Applies the forces due to particles j onto particles i directly. + * + * @param gparts_i The #gpart to update. + * @param gcount_i The number of particles to update. + * @param gparts_j The #gpart that source the gravity field. + * @param gcount_j The number of sources. + */ +INLINE static void gravity_P2P(struct gpart *gparts_i, int gcount_i, + const struct gpart *gparts_j, int gcount_j) {} + +/** + * @brief Constructs the #multipole of a bunch of particles around their + * centre of mass. + * + * Corresponds to equation (28c). + * + * @param m The #multipole (content will be overwritten). + * @param gparts The #gpart. + * @param gcount The number of particles. + */ +INLINE static void gravity_P2M(struct gravity_tensors *m, + const struct gpart *gparts, int gcount) { #if const_gravity_multipole_order >= 2 - /* Quadrupole terms */ - float I_xx, I_yy, I_zz; - float I_xy, I_xz, I_yz; +#error "Implementation of P2M kernel missing for this order." #endif -}; + + /* Temporary variables */ + double mass = 0.0; + double com[3] = {0.0, 0.0, 0.0}; + float vel[3] = {0.f, 0.f, 0.f}; + + /* Collect the particle data. */ + for (int k = 0; k < gcount; k++) { + const float m = gparts[k].mass; + + mass += m; + com[0] += gparts[k].x[0] * m; + com[1] += gparts[k].x[1] * m; + com[2] += gparts[k].x[2] * m; + vel[0] += gparts[k].v_full[0] * m; + vel[1] += gparts[k].v_full[1] * m; + vel[2] += gparts[k].v_full[2] * m; + } + + const double imass = 1.0 / mass; + + /* Store the data on the multipole. */ + m->m_pole.mass = mass; + m->CoM[0] = com[0] * imass; + m->CoM[1] = com[1] * imass; + m->CoM[2] = com[2] * imass; + m->m_pole.vel[0] = vel[0] * imass; + m->m_pole.vel[1] = vel[1] * imass; + m->m_pole.vel[2] = vel[2] * imass; +} + +/** + * @brief Creates a copy of #multipole shifted to a new location. + * + * Corresponds to equation (28d). + * + * @param m_a The #multipole copy (content will be overwritten). + * @param m_b The #multipole to shift. + * @param pos_a The position to which m_b will be shifted. + * @param pos_b The current postion of the multipole to shift. + * @param periodic Is the calculation periodic ? + */ +INLINE static void gravity_M2M(struct multipole *m_a, + const struct multipole *m_b, + const double pos_a[3], const double pos_b[3], + int periodic) { + + m_a->mass = m_b->mass; + + m_a->vel[0] = m_b->vel[0]; + m_a->vel[1] = m_b->vel[1]; + m_a->vel[2] = m_b->vel[2]; +} +/** + * @brief Compute the field tensors due to a multipole. + * + * Corresponds to equation (28b). + * + * @param l_a The field tensor to compute. + * @param m_b The multipole creating the field. + * @param pos_a The position of the field tensor. + * @param pos_b The position of the multipole. + * @param periodic Is the calculation periodic ? + */ +INLINE static void gravity_M2L(struct gravity_tensors *l_a, + const struct multipole *m_b, + const double pos_a[3], const double pos_b[3], + int periodic) { + + double dx, dy, dz; + if (periodic) { + dx = box_wrap(pos_a[0] - pos_b[0], 0., 1.); + dy = box_wrap(pos_a[1] - pos_b[1], 0., 1.); + dz = box_wrap(pos_a[2] - pos_b[2], 0., 1.); + } else { + dx = pos_a[0] - pos_b[0]; + dy = pos_a[1] - pos_b[1]; + dz = pos_a[2] - pos_b[2]; + } + const double r2 = dx * dx + dy * dy + dz * dz; + + const double r_inv = 1. / sqrt(r2); + + /* 1st order multipole term */ + l_a->a_x.F_000 = D_100(dx, dy, dz, r_inv) * m_b->mass; + l_a->a_y.F_000 = D_010(dx, dy, dz, r_inv) * m_b->mass; + l_a->a_z.F_000 = D_001(dx, dy, dz, r_inv) * m_b->mass; + +#ifdef SWIFT_DEBUG_CHECKS + l_a->mass_interacted += m_b->mass; +#endif +} + +/** + * @brief Creates a copy of #acc_tensor shifted to a new location. + * + * Corresponds to equation (28e). + * + * @param l_a The #acc_tensor copy (content will be overwritten). + * @param l_b The #acc_tensor to shift. + * @param pos_a The position to which m_b will be shifted. + * @param pos_b The current postion of the multipole to shift. + * @param periodic Is the calculation periodic ? + */ +INLINE static void gravity_L2L(struct gravity_tensors *l_a, + const struct gravity_tensors *l_b, + const double pos_a[3], const double pos_b[3], + int periodic) {} + +/** + * @brief Applies the #acc_tensor to a set of #gpart. + * + * Corresponds to equation (28a). + */ +INLINE static void gravity_L2P(const struct gravity_tensors *l, + struct gpart *gparts, int gcount) { + + for (int i = 0; i < gcount; ++i) { + +#ifdef SWIFT_DEBUG_CHECKS + struct gpart *gp = &gparts[i]; + + // if(gpart_is_active(gp, e)){ + + gp->mass_interacted += l->mass_interacted; +#endif + //} + } +} + +#if 0 /* Multipole function prototypes. */ -void multipole_add(struct multipole *m_sum, const struct multipole *m_term); -void multipole_init(struct multipole *m, const struct gpart *gparts, +void multipole_add(struct gravity_tensors *m_sum, const struct gravity_tensors *m_term); +void multipole_init(struct gravity_tensors *m, const struct gpart *gparts, int gcount); -void multipole_reset(struct multipole *m); +void multipole_reset(struct gravity_tensors *m); /* static void multipole_iact_mm(struct multipole *ma, struct multipole *mb, */ /* double *shift); */ @@ -64,7 +389,7 @@ void multipole_reset(struct multipole *m); * @param shift The periodicity correction. */ __attribute__((always_inline)) INLINE static void multipole_iact_mm( - struct multipole *ma, struct multipole *mb, double *shift) { + struct gravity_tensors *ma, struct gravity_tensors *mb, double *shift) { /* float dx[3], ir, r, r2 = 0.0f, acc; */ /* int k; */ @@ -106,7 +431,7 @@ __attribute__((always_inline)) INLINE static void multipole_iact_mm( * @param shift The periodicity correction. */ __attribute__((always_inline)) INLINE static void multipole_iact_mp( - struct multipole *m, struct gpart *p, double *shift) { + struct gravity_tensors *m, struct gpart *p, double *shift) { /* float dx[3], ir, r, r2 = 0.0f, acc; */ /* int k; */ @@ -137,4 +462,6 @@ __attribute__((always_inline)) INLINE static void multipole_iact_mp( /* #endif */ } +#endif + #endif /* SWIFT_MULTIPOLE_H */ diff --git a/src/parallel_io.c b/src/parallel_io.c index ce46d8b2dde76e28be468ac89ba0e0bf74edf786..89945b58d96f22c93699ebe78935cd00a0fc3d54 100644 --- a/src/parallel_io.c +++ b/src/parallel_io.c @@ -41,12 +41,15 @@ #include "engine.h" #include "error.h" #include "gravity_io.h" +#include "gravity_properties.h" #include "hydro_io.h" #include "hydro_properties.h" #include "io_properties.h" #include "kernel_hydro.h" #include "part.h" +#include "stars_io.h" #include "units.h" +#include "xmf.h" /** * @brief Reads a data array from a given HDF5 group. @@ -69,10 +72,10 @@ */ void readArray(hid_t grp, const struct io_props prop, size_t N, long long N_total, long long offset, - const struct UnitSystem* internal_units, - const struct UnitSystem* ic_units) { + const struct unit_system* internal_units, + const struct unit_system* ic_units) { - const size_t typeSize = sizeOfType(prop.type); + const size_t typeSize = io_sizeof_type(prop.type); const size_t copySize = typeSize * prop.dimension; const size_t num_elements = N * prop.dimension; @@ -101,7 +104,7 @@ void readArray(hid_t grp, const struct io_props prop, size_t N, /* Check data type */ const hid_t h_type = H5Dget_type(h_data); if (h_type < 0) error("Unable to retrieve data type from the file"); - /* if (!H5Tequal(h_type, hdf5Type(type))) */ + /* if (!H5Tequal(h_type, hdf5_type(type))) */ /* error("Non-matching types between the code and the file"); */ /* Allocate temporary buffer */ @@ -139,7 +142,7 @@ void readArray(hid_t grp, const struct io_props prop, size_t N, /* Read HDF5 dataspace in temporary buffer */ /* Dirty version that happens to work for vectors but should be improved */ /* Using HDF5 dataspaces would be better */ - const hid_t h_err = H5Dread(h_data, hdf5Type(prop.type), h_memspace, + const hid_t h_err = H5Dread(h_data, io_hdf5_type(prop.type), h_memspace, h_filespace, h_plist_id, temp); if (h_err < 0) { error("Error while reading data array '%s'.", prop.name); @@ -152,7 +155,7 @@ void readArray(hid_t grp, const struct io_props prop, size_t N, /* message("Converting ! factor=%e", factor); */ - if (isDoublePrecision(prop.type)) { + if (io_is_double_precision(prop.type)) { double* temp_d = temp; for (size_t i = 0; i < num_elements; ++i) temp_d[i] *= factor; } else { @@ -189,8 +192,8 @@ void readArray(hid_t grp, const struct io_props prop, size_t N, * @param N The number of particles to write. * @param N_total Total number of particles across all cores * @param offset Offset in the array where this mpi task starts writing - * @param internal_units The #UnitSystem used internally - * @param snapshot_units The #UnitSystem used in the snapshots + * @param internal_units The #unit_system used internally + * @param snapshot_units The #unit_system used in the snapshots * * @todo A better version using HDF5 hyper-slabs to write the file directly from * the part array will be written once the structures have been stabilized. @@ -199,17 +202,17 @@ void readArray(hid_t grp, const struct io_props prop, size_t N, void writeArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile, char* partTypeGroupName, const struct io_props props, size_t N, long long N_total, int mpi_rank, long long offset, - const struct UnitSystem* internal_units, - const struct UnitSystem* snapshot_units) { + const struct unit_system* internal_units, + const struct unit_system* snapshot_units) { - const size_t typeSize = sizeOfType(props.type); + const size_t typeSize = io_sizeof_type(props.type); const size_t copySize = typeSize * props.dimension; const size_t num_elements = N * props.dimension; /* message("Writing '%s' array...", props.name); */ /* Allocate temporary buffer */ - void* temp = malloc(num_elements * sizeOfType(props.type)); + void* temp = malloc(num_elements * io_sizeof_type(props.type)); if (temp == NULL) error("Unable to allocate memory for temporary buffer"); /* Copy particle data to temporary buffer */ @@ -240,7 +243,7 @@ void writeArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile, /* message("Converting ! factor=%e", factor); */ - if (isDoublePrecision(props.type)) { + if (io_is_double_precision(props.type)) { double* temp_d = temp; for (size_t i = 0; i < num_elements; ++i) temp_d[i] *= factor; } else { @@ -299,8 +302,8 @@ void writeArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile, /* Create dataset */ const hid_t h_data = - H5Dcreate(grp, props.name, hdf5Type(props.type), h_filespace, H5P_DEFAULT, - H5P_DEFAULT, H5P_DEFAULT); + H5Dcreate(grp, props.name, io_hdf5_type(props.type), h_filespace, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); if (h_data < 0) { error("Error while creating dataset '%s'.", props.name); } @@ -314,7 +317,7 @@ void writeArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile, H5Pset_dxpl_mpio(h_plist_id, H5FD_MPIO_COLLECTIVE); /* Write temporary buffer to HDF5 dataspace */ - h_err = H5Dwrite(h_data, hdf5Type(props.type), h_memspace, h_filespace, + h_err = H5Dwrite(h_data, io_hdf5_type(props.type), h_memspace, h_filespace, h_plist_id, temp); if (h_err < 0) { error("Error while writing data array '%s'.", props.name); @@ -322,19 +325,20 @@ void writeArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile, /* Write XMF description for this data set */ if (mpi_rank == 0) - writeXMFline(xmfFile, fileName, partTypeGroupName, props.name, N_total, - props.dimension, props.type); + xmf_write_line(xmfFile, fileName, partTypeGroupName, props.name, N_total, + props.dimension, props.type); /* Write unit conversion factors for this data set */ char buffer[FIELD_BUFFER_SIZE]; units_cgs_conversion_string(buffer, snapshot_units, props.units); - writeAttribute_d(h_data, "CGS conversion factor", - units_cgs_conversion_factor(snapshot_units, props.units)); - writeAttribute_f(h_data, "h-scale exponent", - units_h_factor(snapshot_units, props.units)); - writeAttribute_f(h_data, "a-scale exponent", - units_a_factor(snapshot_units, props.units)); - writeAttribute_s(h_data, "Conversion factor", buffer); + io_write_attribute_d( + h_data, "CGS conversion factor", + units_cgs_conversion_factor(snapshot_units, props.units)); + io_write_attribute_f(h_data, "h-scale exponent", + units_h_factor(snapshot_units, props.units)); + io_write_attribute_f(h_data, "a-scale exponent", + units_a_factor(snapshot_units, props.units)); + io_write_attribute_s(h_data, "Conversion factor", buffer); /* Free and close everything */ free(temp); @@ -371,20 +375,24 @@ void writeArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile, * Calls #error() if an error occurs. * */ -void read_ic_parallel(char* fileName, const struct UnitSystem* internal_units, +void read_ic_parallel(char* fileName, const struct unit_system* internal_units, double dim[3], struct part** parts, struct gpart** gparts, - size_t* Ngas, size_t* Ngparts, int* periodic, - int* flag_entropy, int mpi_rank, int mpi_size, - MPI_Comm comm, MPI_Info info, int dry_run) { + struct spart** sparts, size_t* Ngas, size_t* Ngparts, + size_t* Nstars, int* periodic, int* flag_entropy, + int with_hydro, int with_gravity, int with_stars, + int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info, + int dry_run) { + hid_t h_file = 0, h_grp = 0; /* GADGET has only cubic boxes (in cosmological mode) */ double boxSize[3] = {0.0, -1.0, -1.0}; - int numParticles[NUM_PARTICLE_TYPES] = {0}; - int numParticles_highWord[NUM_PARTICLE_TYPES] = {0}; - size_t N[NUM_PARTICLE_TYPES] = {0}; - long long N_total[NUM_PARTICLE_TYPES] = {0}; - long long offset[NUM_PARTICLE_TYPES] = {0}; + long long numParticles[swift_type_count] = {0}; + long long numParticles_highWord[swift_type_count] = {0}; + size_t N[swift_type_count] = {0}; + long long N_total[swift_type_count] = {0}; + long long offset[swift_type_count] = {0}; int dimension = 3; /* Assume 3D if nothing is specified */ + size_t Ndm = 0; /* Open file */ /* message("Opening file '%s' as IC.", fileName); */ @@ -401,7 +409,7 @@ void read_ic_parallel(char* fileName, const struct UnitSystem* internal_units, if (h_grp < 0) error("Error while opening runtime parameters\n"); /* Read the relevant information */ - readAttribute(h_grp, "PeriodicBoundariesOn", INT, periodic); + io_read_attribute(h_grp, "PeriodicBoundariesOn", INT, periodic); /* Close runtime parameters */ H5Gclose(h_grp); @@ -415,34 +423,34 @@ void read_ic_parallel(char* fileName, const struct UnitSystem* internal_units, const hid_t hid_dim = H5Aexists(h_grp, "Dimension"); if (hid_dim < 0) error("Error while testing existance of 'Dimension' attribute"); - if (hid_dim > 0) readAttribute(h_grp, "Dimension", INT, &dimension); + if (hid_dim > 0) io_read_attribute(h_grp, "Dimension", INT, &dimension); if (dimension != hydro_dimension) error("ICs dimensionality (%dD) does not match code dimensionality (%dD)", dimension, (int)hydro_dimension); /* Read the relevant information and print status */ int flag_entropy_temp[6]; - readAttribute(h_grp, "Flag_Entropy_ICs", INT, flag_entropy_temp); + io_read_attribute(h_grp, "Flag_Entropy_ICs", INT, flag_entropy_temp); *flag_entropy = flag_entropy_temp[0]; - readAttribute(h_grp, "BoxSize", DOUBLE, boxSize); - readAttribute(h_grp, "NumPart_Total", UINT, numParticles); - readAttribute(h_grp, "NumPart_Total_HighWord", UINT, numParticles_highWord); + io_read_attribute(h_grp, "BoxSize", DOUBLE, boxSize); + io_read_attribute(h_grp, "NumPart_Total", LONGLONG, numParticles); + io_read_attribute(h_grp, "NumPart_Total_HighWord", LONGLONG, + numParticles_highWord); - for (int ptype = 0; ptype < NUM_PARTICLE_TYPES; ++ptype) - N_total[ptype] = ((long long)numParticles[ptype]) + - ((long long)numParticles_highWord[ptype] << 32); + for (int ptype = 0; ptype < swift_type_count; ++ptype) + N_total[ptype] = + (numParticles[ptype]) + (numParticles_highWord[ptype] << 32); dim[0] = boxSize[0]; dim[1] = (boxSize[1] < 0) ? boxSize[0] : boxSize[1]; dim[2] = (boxSize[2] < 0) ? boxSize[0] : boxSize[2]; - /* message("Found %d particles in a %speriodic box of size - * [%f %f %f].", */ - /* N_total, (periodic ? "": "non-"), dim[0], - * dim[1], dim[2]); */ + /* message("Found %lld particles in a %speriodic box of size [%f %f %f].", */ + /* N_total[0], (periodic ? "": "non-"), dim[0], */ + /* dim[1], dim[2]); */ /* Divide the particles among the tasks. */ - for (int ptype = 0; ptype < NUM_PARTICLE_TYPES; ++ptype) { + for (int ptype = 0; ptype < swift_type_count; ++ptype) { offset[ptype] = mpi_rank * N_total[ptype] / mpi_size; N[ptype] = (mpi_rank + 1) * N_total[ptype] / mpi_size - offset[ptype]; } @@ -451,9 +459,9 @@ void read_ic_parallel(char* fileName, const struct UnitSystem* internal_units, H5Gclose(h_grp); /* Read the unit system used in the ICs */ - struct UnitSystem* ic_units = malloc(sizeof(struct UnitSystem)); + struct unit_system* ic_units = malloc(sizeof(struct unit_system)); if (ic_units == NULL) error("Unable to allocate memory for IC unit system"); - readUnitSystem(h_file, ic_units); + io_read_unit_system(h_file, ic_units); /* Tell the user if a conversion will be needed */ if (mpi_rank == 0) { @@ -492,32 +500,43 @@ void read_ic_parallel(char* fileName, const struct UnitSystem* internal_units, units_conversion_factor(ic_units, internal_units, UNIT_CONV_LENGTH); /* Allocate memory to store SPH particles */ - *Ngas = N[0]; - if (posix_memalign((void*)parts, part_align, (*Ngas) * sizeof(struct part)) != - 0) - error("Error while allocating memory for particles"); - bzero(*parts, *Ngas * sizeof(struct part)); - - /* Allocate memory to store all particles */ - const size_t Ndm = N[1]; - *Ngparts = N[1] + N[0]; - if (posix_memalign((void*)gparts, gpart_align, - *Ngparts * sizeof(struct gpart)) != 0) - error( - "Error while allocating memory for gravity " - "particles"); - bzero(*gparts, *Ngparts * sizeof(struct gpart)); - - /* message("Allocated %8.2f MB for particles.", *N * - * sizeof(struct part) / + if (with_hydro) { + *Ngas = N[0]; + if (posix_memalign((void*)parts, part_align, + (*Ngas) * sizeof(struct part)) != 0) + error("Error while allocating memory for particles"); + bzero(*parts, *Ngas * sizeof(struct part)); + } + + /* Allocate memory to store star particles */ + if (with_stars) { + *Nstars = N[swift_type_star]; + if (posix_memalign((void*)sparts, spart_align, + *Nstars * sizeof(struct spart)) != 0) + error("Error while allocating memory for star particles"); + bzero(*sparts, *Nstars * sizeof(struct spart)); + } + + /* Allocate memory to store gravity particles */ + if (with_gravity) { + Ndm = N[1]; + *Ngparts = (with_hydro ? N[swift_type_gas] : 0) + + N[swift_type_dark_matter] + + (with_stars ? N[swift_type_star] : 0); + if (posix_memalign((void*)gparts, gpart_align, + *Ngparts * sizeof(struct gpart)) != 0) + error("Error while allocating memory for gravity particles"); + bzero(*gparts, *Ngparts * sizeof(struct gpart)); + } + + /* message("Allocated %8.2f MB for particles.", *N * sizeof(struct part) / * (1024.*1024.)); */ /* message("BoxSize = %lf", dim[0]); */ - /* message("NumPart = [%zd, %zd] Total = %zd", *Ngas, Ndm, - * *Ngparts); */ + /* message("NumPart = [%zd, %zd] Total = %zd", *Ngas, Ndm, *Ngparts); */ /* Loop over all particle types */ - for (int ptype = 0; ptype < NUM_PARTICLE_TYPES; ptype++) { + for (int ptype = 0; ptype < swift_type_count; ptype++) { /* Don't do anything if no particle of this kind */ if (N_total[ptype] == 0) continue; @@ -538,14 +557,25 @@ void read_ic_parallel(char* fileName, const struct UnitSystem* internal_units, /* Read particle fields into the particle structure */ switch (ptype) { - case GAS: - Nparticles = *Ngas; - hydro_read_particles(*parts, list, &num_fields); + case swift_type_gas: + if (with_hydro) { + Nparticles = *Ngas; + hydro_read_particles(*parts, list, &num_fields); + } break; - case DM: - Nparticles = Ndm; - darkmatter_read_particles(*gparts, list, &num_fields); + case swift_type_dark_matter: + if (with_gravity) { + Nparticles = Ndm; + darkmatter_read_particles(*gparts, list, &num_fields); + } + break; + + case swift_type_star: + if (with_stars) { + Nparticles = *Nstars; + star_read_particles(*sparts, list, &num_fields); + } break; default: @@ -563,10 +593,15 @@ void read_ic_parallel(char* fileName, const struct UnitSystem* internal_units, } /* Prepare the DM particles */ - if (!dry_run) prepare_dm_gparts(*gparts, Ndm); + if (!dry_run && with_gravity) io_prepare_dm_gparts(*gparts, Ndm); + + /* Duplicate the hydro particles into gparts */ + if (!dry_run && with_gravity && with_hydro) + io_duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm); - /* Now duplicate the hydro particle into gparts */ - if (!dry_run) duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm); + /* Duplicate the star particles into gparts */ + if (!dry_run && with_gravity && with_stars) + io_duplicate_star_gparts(*sparts, *gparts, *Nstars, Ndm + *Ngas); /* message("Done Reading particles..."); */ @@ -586,8 +621,8 @@ void read_ic_parallel(char* fileName, const struct UnitSystem* internal_units, * * @param e The engine containing all the system. * @param baseName The common part of the snapshot file name. - * @param internal_units The #UnitSystem used internally - * @param snapshot_units The #UnitSystem used in the snapshots + * @param internal_units The #unit_system used internally + * @param snapshot_units The #unit_system used in the snapshots * @param mpi_rank The MPI rank of this node. * @param mpi_size The number of MPI ranks. * @param comm The MPI communicator. @@ -602,24 +637,26 @@ void read_ic_parallel(char* fileName, const struct UnitSystem* internal_units, * */ void write_output_parallel(struct engine* e, const char* baseName, - const struct UnitSystem* internal_units, - const struct UnitSystem* snapshot_units, + const struct unit_system* internal_units, + const struct unit_system* snapshot_units, int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info) { hid_t h_file = 0, h_grp = 0; const size_t Ngas = e->s->nr_parts; + const size_t Nstars = e->s->nr_sparts; const size_t Ntot = e->s->nr_gparts; int periodic = e->s->periodic; int numFiles = 1; struct part* parts = e->s->parts; struct gpart* gparts = e->s->gparts; struct gpart* dmparts = NULL; + struct spart* sparts = e->s->sparts; static int outputCount = 0; FILE* xmfFile = 0; /* Number of unassociated gparts */ - const size_t Ndm = Ntot > 0 ? Ntot - Ngas : 0; + const size_t Ndm = Ntot > 0 ? Ntot - (Ngas + Nstars) : 0; /* File name */ char fileName[FILENAME_BUFFER_SIZE]; @@ -627,10 +664,10 @@ void write_output_parallel(struct engine* e, const char* baseName, outputCount); /* First time, we need to create the XMF file */ - if (outputCount == 0 && mpi_rank == 0) createXMFfile(baseName); + if (outputCount == 0 && mpi_rank == 0) xmf_create_file(baseName); /* Prepare the XMF file for the new entry */ - if (mpi_rank == 0) xmfFile = prepareXMFfile(baseName); + if (mpi_rank == 0) xmfFile = xmf_prepare_file(baseName); /* Open HDF5 file */ hid_t plist_id = H5Pcreate(H5P_FILE_ACCESS); @@ -642,11 +679,11 @@ void write_output_parallel(struct engine* e, const char* baseName, /* Compute offset in the file and total number of * particles */ - size_t N[NUM_PARTICLE_TYPES] = {Ngas, Ndm, 0}; - long long N_total[NUM_PARTICLE_TYPES] = {0}; - long long offset[NUM_PARTICLE_TYPES] = {0}; - MPI_Exscan(&N, &offset, NUM_PARTICLE_TYPES, MPI_LONG_LONG_INT, MPI_SUM, comm); - for (int ptype = 0; ptype < NUM_PARTICLE_TYPES; ++ptype) + size_t N[swift_type_count] = {Ngas, Ndm, 0, 0, Nstars, 0}; + long long N_total[swift_type_count] = {0}; + long long offset[swift_type_count] = {0}; + MPI_Exscan(&N, &offset, swift_type_count, MPI_LONG_LONG_INT, MPI_SUM, comm); + for (int ptype = 0; ptype < swift_type_count; ++ptype) N_total[ptype] = offset[ptype] + N[ptype]; /* The last rank now has the correct N_total. Let's @@ -659,7 +696,7 @@ void write_output_parallel(struct engine* e, const char* baseName, /* Write the part of the XMF file corresponding to this * specific output */ - if (mpi_rank == 0) writeXMFoutputheader(xmfFile, fileName, e->time); + if (mpi_rank == 0) xmf_write_outputheader(xmfFile, fileName, e->time); /* Open header to write simulation properties */ /* message("Writing runtime parameters..."); */ @@ -668,7 +705,7 @@ void write_output_parallel(struct engine* e, const char* baseName, if (h_grp < 0) error("Error while creating runtime parameters group\n"); /* Write the relevant information */ - writeAttribute(h_grp, "PeriodicBoundariesOn", INT, &periodic, 1); + io_write_attribute(h_grp, "PeriodicBoundariesOn", INT, &periodic, 1); /* Close runtime parameters */ H5Gclose(h_grp); @@ -679,47 +716,58 @@ void write_output_parallel(struct engine* e, const char* baseName, if (h_grp < 0) error("Error while creating file header\n"); /* Print the relevant information and print status */ - writeAttribute(h_grp, "BoxSize", DOUBLE, e->s->dim, 3); + io_write_attribute(h_grp, "BoxSize", DOUBLE, e->s->dim, 3); double dblTime = e->time; - writeAttribute(h_grp, "Time", DOUBLE, &dblTime, 1); + io_write_attribute(h_grp, "Time", DOUBLE, &dblTime, 1); int dimension = (int)hydro_dimension; - writeAttribute(h_grp, "Dimension", INT, &dimension, 1); + io_write_attribute(h_grp, "Dimension", INT, &dimension, 1); /* GADGET-2 legacy values */ /* Number of particles of each type */ - unsigned int numParticles[NUM_PARTICLE_TYPES] = {0}; - unsigned int numParticlesHighWord[NUM_PARTICLE_TYPES] = {0}; - for (int ptype = 0; ptype < NUM_PARTICLE_TYPES; ++ptype) { + unsigned int numParticles[swift_type_count] = {0}; + unsigned int numParticlesHighWord[swift_type_count] = {0}; + for (int ptype = 0; ptype < swift_type_count; ++ptype) { numParticles[ptype] = (unsigned int)N_total[ptype]; numParticlesHighWord[ptype] = (unsigned int)(N_total[ptype] >> 32); } - writeAttribute(h_grp, "NumPart_ThisFile", LONGLONG, N_total, - NUM_PARTICLE_TYPES); - writeAttribute(h_grp, "NumPart_Total", UINT, numParticles, - NUM_PARTICLE_TYPES); - writeAttribute(h_grp, "NumPart_Total_HighWord", UINT, numParticlesHighWord, - NUM_PARTICLE_TYPES); + io_write_attribute(h_grp, "NumPart_ThisFile", LONGLONG, N_total, + swift_type_count); + io_write_attribute(h_grp, "NumPart_Total", UINT, numParticles, + swift_type_count); + io_write_attribute(h_grp, "NumPart_Total_HighWord", UINT, + numParticlesHighWord, swift_type_count); double MassTable[6] = {0., 0., 0., 0., 0., 0.}; - writeAttribute(h_grp, "MassTable", DOUBLE, MassTable, NUM_PARTICLE_TYPES); - unsigned int flagEntropy[NUM_PARTICLE_TYPES] = {0}; + io_write_attribute(h_grp, "MassTable", DOUBLE, MassTable, swift_type_count); + unsigned int flagEntropy[swift_type_count] = {0}; flagEntropy[0] = writeEntropyFlag(); - writeAttribute(h_grp, "Flag_Entropy_ICs", UINT, flagEntropy, - NUM_PARTICLE_TYPES); - writeAttribute(h_grp, "NumFilesPerSnapshot", INT, &numFiles, 1); + io_write_attribute(h_grp, "Flag_Entropy_ICs", UINT, flagEntropy, + swift_type_count); + io_write_attribute(h_grp, "NumFilesPerSnapshot", INT, &numFiles, 1); /* Close header */ H5Gclose(h_grp); /* Print the code version */ - writeCodeDescription(h_file); + io_write_code_description(h_file); /* Print the SPH parameters */ - h_grp = - H5Gcreate(h_file, "/HydroScheme", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if (h_grp < 0) error("Error while creating SPH group"); - hydro_props_print_snapshot(h_grp, e->hydro_properties); - writeSPHflavour(h_grp); - H5Gclose(h_grp); + if (e->policy & engine_policy_hydro) { + h_grp = H5Gcreate(h_file, "/HydroScheme", H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT); + if (h_grp < 0) error("Error while creating SPH group"); + hydro_props_print_snapshot(h_grp, e->hydro_properties); + writeSPHflavour(h_grp); + H5Gclose(h_grp); + } + + /* Print the gravity parameters */ + if (e->policy & engine_policy_self_gravity) { + h_grp = H5Gcreate(h_file, "/GravityScheme", H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT); + if (h_grp < 0) error("Error while creating gravity group"); + gravity_props_print_snapshot(h_grp, e->gravity_properties); + H5Gclose(h_grp); + } /* Print the runtime parameters */ h_grp = @@ -729,10 +777,10 @@ void write_output_parallel(struct engine* e, const char* baseName, H5Gclose(h_grp); /* Print the system of Units used in the spashot */ - writeUnitSystem(h_file, snapshot_units, "Units"); + io_write_unit_system(h_file, snapshot_units, "Units"); /* Print the system of Units used internally */ - writeUnitSystem(h_file, internal_units, "InternalCodeUnits"); + io_write_unit_system(h_file, internal_units, "InternalCodeUnits"); /* Tell the user if a conversion will be needed */ if (e->verbose && mpi_rank == 0) { @@ -768,7 +816,7 @@ void write_output_parallel(struct engine* e, const char* baseName, } /* Loop over all particle types */ - for (int ptype = 0; ptype < NUM_PARTICLE_TYPES; ptype++) { + for (int ptype = 0; ptype < swift_type_count; ptype++) { /* Don't do anything if no particle of this kind */ if (N_total[ptype] == 0) continue; @@ -776,8 +824,8 @@ void write_output_parallel(struct engine* e, const char* baseName, /* Add the global information for that particle type to * the XMF meta-file */ if (mpi_rank == 0) - writeXMFgroupheader(xmfFile, fileName, N_total[ptype], - (enum PARTICLE_TYPE)ptype); + xmf_write_groupheader(xmfFile, fileName, N_total[ptype], + (enum part_type)ptype); /* Open the particle group in the file */ char partTypeGroupName[PARTICLE_GROUP_BUFFER_SIZE]; @@ -796,12 +844,12 @@ void write_output_parallel(struct engine* e, const char* baseName, /* Write particle fields from the particle structure */ switch (ptype) { - case GAS: + case swift_type_gas: Nparticles = Ngas; hydro_write_particles(parts, list, &num_fields); break; - case DM: + case swift_type_dark_matter: /* Allocate temporary array */ if (posix_memalign((void*)&dmparts, gpart_align, Ndm * sizeof(struct gpart)) != 0) @@ -811,14 +859,16 @@ void write_output_parallel(struct engine* e, const char* baseName, bzero(dmparts, Ndm * sizeof(struct gpart)); /* Collect the DM particles from gpart */ - collect_dm_gparts(gparts, Ntot, dmparts, Ndm); + io_collect_dm_gparts(gparts, Ntot, dmparts, Ndm); /* Write DM particles */ Nparticles = Ndm; darkmatter_write_particles(dmparts, list, &num_fields); + break; - /* Free temporary array */ - free(dmparts); + case swift_type_star: + Nparticles = Nstars; + star_write_particles(sparts, list, &num_fields); break; default: @@ -832,17 +882,20 @@ void write_output_parallel(struct engine* e, const char* baseName, internal_units, snapshot_units); /* Free temporary array */ - free(dmparts); + if (dmparts) { + free(dmparts); + dmparts = 0; + } /* Close particle group */ H5Gclose(h_grp); /* Close this particle group in the XMF file as well */ - if (mpi_rank == 0) writeXMFgroupfooter(xmfFile, (enum PARTICLE_TYPE)ptype); + if (mpi_rank == 0) xmf_write_groupfooter(xmfFile, (enum part_type)ptype); } /* Write LXMF file descriptor */ - if (mpi_rank == 0) writeXMFoutputfooter(xmfFile, outputCount, e->time); + if (mpi_rank == 0) xmf_write_outputfooter(xmfFile, outputCount, e->time); /* message("Done writing particles..."); */ diff --git a/src/parallel_io.h b/src/parallel_io.h index e5b12aa50c30b4d63ccc81835d2d8454e01b3889..d4cde2bd89313f6cf14ce059870925b9ac304b40 100644 --- a/src/parallel_io.h +++ b/src/parallel_io.h @@ -34,15 +34,17 @@ #if defined(HAVE_HDF5) && defined(WITH_MPI) && defined(HAVE_PARALLEL_HDF5) -void read_ic_parallel(char* fileName, const struct UnitSystem* internal_units, +void read_ic_parallel(char* fileName, const struct unit_system* internal_units, double dim[3], struct part** parts, struct gpart** gparts, - size_t* Ngas, size_t* Ngparts, int* periodic, - int* flag_entropy, int mpi_rank, int mpi_size, - MPI_Comm comm, MPI_Info info, int dry_run); + struct spart** sparts, size_t* Ngas, size_t* Ngparts, + size_t* Nsparts, int* periodic, int* flag_entropy, + int with_hydro, int with_gravity, int with_stars, + int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info, + int dry_run); void write_output_parallel(struct engine* e, const char* baseName, - const struct UnitSystem* internal_units, - const struct UnitSystem* snapshot_units, + const struct unit_system* internal_units, + const struct unit_system* snapshot_units, int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info); #endif diff --git a/src/parser.c b/src/parser.c index 879605760bedb5e4bb282788c2392edf0d3a64df..41a3e8637630eceb3beb9383acb3344028d38659 100644 --- a/src/parser.c +++ b/src/parser.c @@ -722,6 +722,6 @@ void parser_write_params_to_file(const struct swift_params *params, void parser_write_params_to_hdf5(const struct swift_params *params, hid_t grp) { for (int i = 0; i < params->paramCount; i++) - writeAttribute_s(grp, params->data[i].name, params->data[i].value); + io_write_attribute_s(grp, params->data[i].name, params->data[i].value); } #endif diff --git a/src/part.c b/src/part.c index b00eaccaae0e86f7c4e8019a307f0bf455687b7c..712a13d42a629355290bcf89f31b94fb9670a215 100644 --- a/src/part.c +++ b/src/part.c @@ -27,6 +27,7 @@ /* This object's header. */ #include "error.h" +#include "multipole.h" #include "part.h" /** @@ -36,7 +37,8 @@ * @param N The number of particles to re-link; * @param offset The offset of #part%s relative to the global parts list. */ -void part_relink_gparts(struct part *parts, size_t N, ptrdiff_t offset) { +void part_relink_gparts_to_parts(struct part *parts, size_t N, + ptrdiff_t offset) { for (size_t k = 0; k < N; k++) { if (parts[k].gpart) { parts[k].gpart->id_or_neg_offset = -(k + offset); @@ -45,28 +47,195 @@ void part_relink_gparts(struct part *parts, size_t N, ptrdiff_t offset) { } /** - * @brief Re-link the #gpart%s associated with the list of #part%s. + * @brief Re-link the #gpart%s associated with the list of #spart%s. + * + * @param sparts The list of #spart. + * @param N The number of s-particles to re-link; + * @param offset The offset of #spart%s relative to the global sparts list. + */ +void part_relink_gparts_to_sparts(struct spart *sparts, size_t N, + ptrdiff_t offset) { + for (size_t k = 0; k < N; k++) { + if (sparts[k].gpart) { + sparts[k].gpart->id_or_neg_offset = -(k + offset); + } + } +} + +/** + * @brief Re-link the #part%s associated with the list of #gpart%s. * * @param gparts The list of #gpart. * @param N The number of particles to re-link; - * @param parts The global part array in which to find the #gpart offsets. + * @param parts The global #part array in which to find the #gpart offsets. */ -void part_relink_parts(struct gpart *gparts, size_t N, struct part *parts) { +void part_relink_parts_to_gparts(struct gpart *gparts, size_t N, + struct part *parts) { for (size_t k = 0; k < N; k++) { - if (gparts[k].id_or_neg_offset <= 0) { + if (gparts[k].type == swift_type_gas) { parts[-gparts[k].id_or_neg_offset].gpart = &gparts[k]; } } } +/** + * @brief Re-link the #spart%s associated with the list of #gpart%s. + * + * @param gparts The list of #gpart. + * @param N The number of particles to re-link; + * @param sparts The global #spart array in which to find the #gpart offsets. + */ +void part_relink_sparts_to_gparts(struct gpart *gparts, size_t N, + struct spart *sparts) { + for (size_t k = 0; k < N; k++) { + if (gparts[k].type == swift_type_star) { + sparts[-gparts[k].id_or_neg_offset].gpart = &gparts[k]; + } + } +} + +/** + * @brief Verifies that the #gpart, #part and #spart are correctly linked + * together + * and that the particle poisitions match. + * + * This is a debugging function. + * + * @param parts The #part array. + * @param gparts The #gpart array. + * @param sparts The #spart array. + * @param nr_parts The number of #part in the array. + * @param nr_gparts The number of #gpart in the array. + * @param nr_sparts The number of #spart in the array. + * @param verbose Do we report verbosely in case of success ? + */ +void part_verify_links(struct part *parts, struct gpart *gparts, + struct spart *sparts, size_t nr_parts, size_t nr_gparts, + size_t nr_sparts, int verbose) { + + for (size_t k = 0; k < nr_gparts; ++k) { + + /* We have a DM particle */ + if (gparts[k].type == swift_type_dark_matter) { + + /* Check that it's not linked */ + if (gparts[k].id_or_neg_offset < 0) + error("DM gpart particle linked to something !"); + } + + /* We have a gas particle */ + else if (gparts[k].type == swift_type_gas) { + + /* Check that it is linked */ + if (gparts[k].id_or_neg_offset > 0) + error("Gas gpart not linked to anything !"); + + /* Find its link */ + const struct part *part = &parts[-gparts[k].id_or_neg_offset]; + + /* Check the reverse link */ + if (part->gpart != &gparts[k]) error("Linking problem !"); + + /* Check that the particles are at the same place */ + if (gparts[k].x[0] != part->x[0] || gparts[k].x[1] != part->x[1] || + gparts[k].x[2] != part->x[2]) + error( + "Linked particles are not at the same position !\n" + "gp->x=[%e %e %e] p->x=[%e %e %e] diff=[%e %e %e]", + gparts[k].x[0], gparts[k].x[1], gparts[k].x[2], part->x[0], + part->x[1], part->x[2], gparts[k].x[0] - part->x[0], + gparts[k].x[1] - part->x[1], gparts[k].x[2] - part->x[2]); + + /* Check that the particles are at the same time */ + if (gparts[k].time_bin != part->time_bin) + error("Linked particles are not at the same time !"); + } + + else if (gparts[k].type == swift_type_star) { + + /* Check that it is linked */ + if (gparts[k].id_or_neg_offset > 0) + error("Star gpart not linked to anything !"); + + /* Find its link */ + const struct spart *spart = &sparts[-gparts[k].id_or_neg_offset]; + + /* Check the reverse link */ + if (spart->gpart != &gparts[k]) error("Linking problem !"); + + /* Check that the particles are at the same place */ + if (gparts[k].x[0] != spart->x[0] || gparts[k].x[1] != spart->x[1] || + gparts[k].x[2] != spart->x[2]) + error( + "Linked particles are not at the same position !\n" + "gp->x=[%e %e %e] sp->x=[%e %e %e] diff=[%e %e %e]", + gparts[k].x[0], gparts[k].x[1], gparts[k].x[2], spart->x[0], + spart->x[1], spart->x[2], gparts[k].x[0] - spart->x[0], + gparts[k].x[1] - spart->x[1], gparts[k].x[2] - spart->x[2]); + + /* Check that the particles are at the same time */ + if (gparts[k].time_bin != spart->time_bin) + error("Linked particles are not at the same time !"); + } + } + + /* Now check that all parts are linked */ + for (size_t k = 0; k < nr_parts; ++k) { + + /* Ok, there is a link */ + if (parts[k].gpart != NULL) { + + /* Check the link */ + if (parts[k].gpart->id_or_neg_offset != -(ptrdiff_t)k) { + error("Linking problem !"); + } + + /* Check that the particles are at the same place */ + if (parts[k].x[0] != parts[k].gpart->x[0] || + parts[k].x[1] != parts[k].gpart->x[1] || + parts[k].x[2] != parts[k].gpart->x[2]) + error("Linked particles are not at the same position !"); + + /* Check that the particles are at the same time */ + if (parts[k].time_bin != parts[k].gpart->time_bin) + error("Linked particles are not at the same time !"); + } + } + + /* Now check that all sparts are linked */ + for (size_t k = 0; k < nr_sparts; ++k) { + + /* Ok, there is a link */ + if (sparts[k].gpart != NULL) { + + /* Check the link */ + if (sparts[k].gpart->id_or_neg_offset != -(ptrdiff_t)k) { + error("Linking problem !"); + + /* Check that the particles are at the same place */ + if (sparts[k].x[0] != sparts[k].gpart->x[0] || + sparts[k].x[1] != sparts[k].gpart->x[1] || + sparts[k].x[2] != sparts[k].gpart->x[2]) + error("Linked particles are not at the same position !"); + + /* Check that the particles are at the same time */ + if (sparts[k].time_bin != sparts[k].gpart->time_bin) + error("Linked particles are not at the same time !"); + } + } + } + + if (verbose) message("All links OK"); +} + #ifdef WITH_MPI /* MPI data type for the particle transfers */ MPI_Datatype part_mpi_type; MPI_Datatype xpart_mpi_type; MPI_Datatype gpart_mpi_type; -#endif +MPI_Datatype spart_mpi_type; +MPI_Datatype multipole_mpi_type; -#ifdef WITH_MPI /** * @brief Registers MPI particle types. */ @@ -93,5 +262,16 @@ void part_create_mpi_types() { MPI_Type_commit(&gpart_mpi_type) != MPI_SUCCESS) { error("Failed to create MPI type for gparts."); } + if (MPI_Type_contiguous(sizeof(struct spart) / sizeof(unsigned char), + MPI_BYTE, &spart_mpi_type) != MPI_SUCCESS || + MPI_Type_commit(&spart_mpi_type) != MPI_SUCCESS) { + error("Failed to create MPI type for sparts."); + } + if (MPI_Type_contiguous( + sizeof(struct gravity_tensors) / sizeof(unsigned char), MPI_BYTE, + &multipole_mpi_type) != MPI_SUCCESS || + MPI_Type_commit(&multipole_mpi_type) != MPI_SUCCESS) { + error("Failed to create MPI type for multipole."); + } } #endif diff --git a/src/part.h b/src/part.h index 71bad6019c182e4cdb72744d2076e165504fa874..e9a151f5b6dc6c2670ced942d195e108555c5a4b 100644 --- a/src/part.h +++ b/src/part.h @@ -32,11 +32,13 @@ /* Local headers. */ #include "align.h" +#include "part_type.h" #include "timeline.h" /* Some constants. */ #define part_align 128 #define xpart_align 128 +#define spart_align 128 #define gpart_align 128 /* Import the right hydro particle definition */ @@ -63,13 +65,28 @@ /* Import the right gravity particle definition */ #include "./gravity/Default/gravity_part.h" -void part_relink_gparts(struct part *parts, size_t N, ptrdiff_t offset); -void part_relink_parts(struct gpart *gparts, size_t N, struct part *parts); +/* Import the right star particle definition */ +#include "./stars/Default/star_part.h" + +void part_relink_gparts_to_parts(struct part *parts, size_t N, + ptrdiff_t offset); +void part_relink_gparts_to_sparts(struct spart *sparts, size_t N, + ptrdiff_t offset); +void part_relink_parts_to_gparts(struct gpart *gparts, size_t N, + struct part *parts); +void part_relink_sparts_to_gparts(struct gpart *gparts, size_t N, + struct spart *sparts); +void part_verify_links(struct part *parts, struct gpart *gparts, + struct spart *sparts, size_t nr_parts, size_t nr_gparts, + size_t nr_sparts, int verbose); + #ifdef WITH_MPI /* MPI data type for the particle transfers */ extern MPI_Datatype part_mpi_type; extern MPI_Datatype xpart_mpi_type; extern MPI_Datatype gpart_mpi_type; +extern MPI_Datatype spart_mpi_type; +extern MPI_Datatype multipole_mpi_type; void part_create_mpi_types(); #endif diff --git a/src/part_type.c b/src/part_type.c new file mode 100644 index 0000000000000000000000000000000000000000..af97bd34aaace93a9faa953c0c9345d83ca3bc34 --- /dev/null +++ b/src/part_type.c @@ -0,0 +1,24 @@ +/******************************************************************************* + * This file is part of SWIFT. + * Copyright (c) 2016 Matthieu Schaller (matthieu.schaller@durham.ac.uk). + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + ******************************************************************************/ + +/* This object's header. */ +#include "part_type.h" + +const char* part_type_names[swift_type_count] = {"Gas", "DM", "Dummy", + "Dummy", "Star", "BH"}; diff --git a/src/part_type.h b/src/part_type.h new file mode 100644 index 0000000000000000000000000000000000000000..fbe2b2aeaea37503635372b0f09f8edde4578721 --- /dev/null +++ b/src/part_type.h @@ -0,0 +1,37 @@ +/******************************************************************************* + * This file is part of SWIFT. + * Copyright (c) 2016 Matthieu Schaller (matthieu.schaller@durham.ac.uk). + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + ******************************************************************************/ +#ifndef SWIFT_PART_TYPES_H +#define SWIFT_PART_TYPES_H + +/** + * @brief The different types of particles a #gpart can link to. + * + * Note we use the historical values from Gadget for these fields. + */ +enum part_type { + swift_type_gas = 0, + swift_type_dark_matter = 1, + swift_type_star = 4, + swift_type_black_hole = 5, + swift_type_count +} __attribute__((packed)); + +extern const char* part_type_names[]; + +#endif /* SWIFT_PART_TYPES_H */ diff --git a/src/physical_constants.c b/src/physical_constants.c index b726678ec1c0f2c85e409da3898dc3b6b842750f..c851578c96e146261e9512f6899c7b82a8d91097 100644 --- a/src/physical_constants.c +++ b/src/physical_constants.c @@ -34,7 +34,8 @@ * @param us The current internal system of units. * @param internal_const The physical constants to initialize. */ -void phys_const_init(struct UnitSystem* us, struct phys_const* internal_const) { +void phys_const_init(struct unit_system* us, + struct phys_const* internal_const) { /* Units are declared as {U_M, U_L, U_t, U_I, U_T} */ diff --git a/src/physical_constants.h b/src/physical_constants.h index aac0fa0840e08a1140bbab67e944f7f134f4222b..3731c0ef565c6159592ad2de96a222efc6cf43f2 100644 --- a/src/physical_constants.h +++ b/src/physical_constants.h @@ -78,7 +78,7 @@ struct phys_const { double const_earth_mass; }; -void phys_const_init(struct UnitSystem* us, struct phys_const* internal_const); +void phys_const_init(struct unit_system* us, struct phys_const* internal_const); void phys_const_print(struct phys_const* internal_const); diff --git a/src/potential.c b/src/potential.c index 6ee80900952c032d019ad5d20ec086d05d34ef29..94c2a6cc9412d1fc76b70ddebb2edba7878e6209 100644 --- a/src/potential.c +++ b/src/potential.c @@ -36,7 +36,7 @@ */ void potential_init(const struct swift_params* parameter_file, const struct phys_const* phys_const, - const struct UnitSystem* us, const struct space* s, + const struct unit_system* us, const struct space* s, struct external_potential* potential) { potential_init_backend(parameter_file, phys_const, us, s, potential); diff --git a/src/potential.h b/src/potential.h index 116ea8302e7f706cdb861540a89d562174d73408..e6ab9a5bd6bd91801eae0b3f1e3d8f65778f5065 100644 --- a/src/potential.h +++ b/src/potential.h @@ -36,6 +36,8 @@ #include "./potential/isothermal/potential.h" #elif defined(EXTERNAL_POTENTIAL_DISC_PATCH) #include "./potential/disc_patch/potential.h" +#elif defined(EXTERNAL_POTENTIAL_SINE_WAVE) +#include "./potential/sine_wave/potential.h" #else #error "Invalid choice of external potential" #endif @@ -43,7 +45,7 @@ /* Now, some generic functions, defined in the source file */ void potential_init(const struct swift_params* parameter_file, const struct phys_const* phys_const, - const struct UnitSystem* us, const struct space* s, + const struct unit_system* us, const struct space* s, struct external_potential* potential); void potential_print(const struct external_potential* potential); diff --git a/src/potential/disc_patch/potential.h b/src/potential/disc_patch/potential.h index 400539a8d02d29a8d383bb1c523d064f733267c5..8fa40ecd4e6503cde8be00db8c6fb8a70c84ebdf 100644 --- a/src/potential/disc_patch/potential.h +++ b/src/potential/disc_patch/potential.h @@ -195,7 +195,7 @@ external_gravity_get_potential_energy( */ static INLINE void potential_init_backend( const struct swift_params* parameter_file, - const struct phys_const* phys_const, const struct UnitSystem* us, + const struct phys_const* phys_const, const struct unit_system* us, const struct space* s, struct external_potential* potential) { potential->surface_density = parser_get_param_double( diff --git a/src/potential/isothermal/potential.h b/src/potential/isothermal/potential.h index 9c07f3eb67528a003788ca94bd1e2e52dd985a2c..c974618d7b581884f871863bb83200b8cecee7a5 100644 --- a/src/potential/isothermal/potential.h +++ b/src/potential/isothermal/potential.h @@ -163,7 +163,7 @@ external_gravity_get_potential_energy( */ static INLINE void potential_init_backend( const struct swift_params* parameter_file, - const struct phys_const* phys_const, const struct UnitSystem* us, + const struct phys_const* phys_const, const struct unit_system* us, const struct space* s, struct external_potential* potential) { potential->x = diff --git a/src/potential/none/potential.h b/src/potential/none/potential.h index cb6254b4a23b336637cb3c9f36a2dd01170eabad..a8550cad702891ff211539c95c42eca57418c464 100644 --- a/src/potential/none/potential.h +++ b/src/potential/none/potential.h @@ -99,7 +99,7 @@ external_gravity_get_potential_energy( */ static INLINE void potential_init_backend( const struct swift_params* parameter_file, - const struct phys_const* phys_const, const struct UnitSystem* us, + const struct phys_const* phys_const, const struct unit_system* us, const struct space* s, struct external_potential* potential) {} /** diff --git a/src/potential/point_mass/potential.h b/src/potential/point_mass/potential.h index 81b51ab2009ad599d0201708d78c8c64cac991dc..adea9d912056fd07e134fb98a7603030e897ec7a 100644 --- a/src/potential/point_mass/potential.h +++ b/src/potential/point_mass/potential.h @@ -24,6 +24,7 @@ #include "../config.h" /* Some standard headers. */ +#include <float.h> #include <math.h> /* Local includes. */ @@ -84,7 +85,10 @@ __attribute__((always_inline)) INLINE static float external_gravity_timestep( const float a_2 = g->a_grav[0] * g->a_grav[0] + g->a_grav[1] * g->a_grav[1] + g->a_grav[2] * g->a_grav[2]; - return potential->timestep_mult * sqrtf(a_2 / dota_2); + if (fabsf(dota_2) > 0.f) + return potential->timestep_mult * sqrtf(a_2 / dota_2); + else + return FLT_MAX; } /** @@ -149,7 +153,7 @@ external_gravity_get_potential_energy( */ static INLINE void potential_init_backend( const struct swift_params* parameter_file, - const struct phys_const* phys_const, const struct UnitSystem* us, + const struct phys_const* phys_const, const struct unit_system* us, const struct space* s, struct external_potential* potential) { potential->x = diff --git a/src/potential/sine_wave/potential.h b/src/potential/sine_wave/potential.h new file mode 100644 index 0000000000000000000000000000000000000000..e2e2b8ffcc170c28a5facc8373a81746811a9991 --- /dev/null +++ b/src/potential/sine_wave/potential.h @@ -0,0 +1,133 @@ +/******************************************************************************* + * This file is part of SWIFT. + * Copyright (c) 2017 Bert Vandenbroucke (bert.vandenbroucke@gmail.com) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + ******************************************************************************/ +#ifndef SWIFT_SINE_WAVE_H +#define SWIFT_SINE_WAVE_H + +/* Config parameters. */ +#include "../config.h" + +/* Some standard headers. */ +#include <float.h> +#include <math.h> + +/* Local includes. */ +#include "const.h" +#include "error.h" +#include "parser.h" +#include "part.h" +#include "physical_constants.h" +#include "space.h" +#include "units.h" + +/** + * @brief External Potential Properties - Sine wave case + */ +struct external_potential { + + /*! Amplitude of the sine wave. */ + double amplitude; + + /*! Time-step limiting factor. */ + double timestep_limit; +}; + +/** + * @brief Computes the time-step from the acceleration due to a sine wave. + * + * @param time The current time. + * @param potential The properties of the potential. + * @param phys_const The physical constants in internal units. + * @param g Pointer to the g-particle data. + */ +__attribute__((always_inline)) INLINE static float external_gravity_timestep( + double time, const struct external_potential* restrict potential, + const struct phys_const* restrict phys_const, + const struct gpart* restrict g) { + + return potential->timestep_limit; +} + +/** + * @brief Computes the gravitational acceleration along x given by the sine + * wave. + * + * @param time The current time in internal units. + * @param potential The properties of the potential. + * @param phys_const The physical constants in internal units. + * @param g Pointer to the g-particle data. + */ +__attribute__((always_inline)) INLINE static void external_gravity_acceleration( + double time, const struct external_potential* restrict potential, + const struct phys_const* restrict phys_const, struct gpart* restrict g) { + + g->a_grav[0] = potential->amplitude * sin(2. * M_PI * g->x[0]) / + phys_const->const_newton_G; +} + +/** + * @brief Computes the gravitational potential energy of a particle in the + * sine wave. + * + * @param time The current time. + * @param potential The #external_potential used in the run. + * @param phys_const Physical constants in internal units. + * @param gp Pointer to the particle data. + */ +__attribute__((always_inline)) INLINE static float +external_gravity_get_potential_energy( + double time, const struct external_potential* potential, + const struct phys_const* const phys_const, const struct gpart* gp) { + + /* this potential does not really have a potential energy */ + return 0.; +} + +/** + * @brief Initialises the external potential properties in the internal system + * of units. + * + * @param parameter_file The parsed parameter file + * @param phys_const Physical constants in internal units + * @param us The current internal system of units + * @param potential The external potential properties to initialize + */ +static INLINE void potential_init_backend( + const struct swift_params* parameter_file, + const struct phys_const* phys_const, const struct unit_system* us, + const struct space* s, struct external_potential* potential) { + + potential->amplitude = + parser_get_param_double(parameter_file, "SineWavePotential:amplitude"); + potential->timestep_limit = parser_get_param_double( + parameter_file, "SineWavePotential:timestep_limit"); +} + +/** + * @brief Prints the properties of the external potential to stdout. + * + * @param potential The external potential properties. + */ +static INLINE void potential_print_backend( + const struct external_potential* potential) { + + message("External potential is a sine wave with amplitude %g", + potential->amplitude); +} + +#endif /* SWIFT_SINE_WAVE_H */ diff --git a/src/proxy.c b/src/proxy.c index 6a3dcaca2851ab39b832a054511577266599cad0..dd6faa3055cb17a0a3050d9e62d107d7489a4326 100644 --- a/src/proxy.c +++ b/src/proxy.c @@ -45,7 +45,6 @@ * * @param p The #proxy. */ - void proxy_cells_exch1(struct proxy *p) { #ifdef WITH_MPI @@ -126,7 +125,6 @@ void proxy_cells_exch2(struct proxy *p) { * @param p The #proxy. * @param c The #cell. */ - void proxy_addcell_in(struct proxy *p, struct cell *c) { /* Check if the cell is already registered with the proxy. */ @@ -155,7 +153,6 @@ void proxy_addcell_in(struct proxy *p, struct cell *c) { * @param p The #proxy. * @param c The #cell. */ - void proxy_addcell_out(struct proxy *p, struct cell *c) { /* Check if the cell is already registered with the proxy. */ @@ -183,7 +180,6 @@ void proxy_addcell_out(struct proxy *p, struct cell *c) { * * @param p The #proxy. */ - void proxy_parts_exch1(struct proxy *p) { #ifdef WITH_MPI @@ -191,7 +187,8 @@ void proxy_parts_exch1(struct proxy *p) { /* Send the number of particles. */ p->buff_out[0] = p->nr_parts_out; p->buff_out[1] = p->nr_gparts_out; - if (MPI_Isend(p->buff_out, 2, MPI_INT, p->nodeID, + p->buff_out[2] = p->nr_sparts_out; + if (MPI_Isend(p->buff_out, 3, MPI_INT, p->nodeID, p->mynodeID * proxy_tag_shift + proxy_tag_count, MPI_COMM_WORLD, &p->req_parts_count_out) != MPI_SUCCESS) error("Failed to isend nr of parts."); @@ -218,13 +215,22 @@ void proxy_parts_exch1(struct proxy *p) { if (MPI_Isend(p->gparts_out, p->nr_gparts_out, gpart_mpi_type, p->nodeID, p->mynodeID * proxy_tag_shift + proxy_tag_gparts, MPI_COMM_WORLD, &p->req_gparts_out) != MPI_SUCCESS) - error("Failed to isend part data."); + error("Failed to isend gpart data."); + // message( "isent gpart data (%i) to node %i." , p->nr_parts_out , + // p->nodeID ); fflush(stdout); + } + + if (p->nr_sparts_out > 0) { + if (MPI_Isend(p->sparts_out, p->nr_sparts_out, spart_mpi_type, p->nodeID, + p->mynodeID * proxy_tag_shift + proxy_tag_sparts, + MPI_COMM_WORLD, &p->req_sparts_out) != MPI_SUCCESS) + error("Failed to isend spart data."); // message( "isent gpart data (%i) to node %i." , p->nr_parts_out , // p->nodeID ); fflush(stdout); } /* Receive the number of particles. */ - if (MPI_Irecv(p->buff_in, 2, MPI_INT, p->nodeID, + if (MPI_Irecv(p->buff_in, 3, MPI_INT, p->nodeID, p->nodeID * proxy_tag_shift + proxy_tag_count, MPI_COMM_WORLD, &p->req_parts_count_in) != MPI_SUCCESS) error("Failed to irecv nr of parts."); @@ -241,8 +247,9 @@ void proxy_parts_exch2(struct proxy *p) { /* Unpack the incomming parts counts. */ p->nr_parts_in = p->buff_in[0]; p->nr_gparts_in = p->buff_in[1]; + p->nr_sparts_in = p->buff_in[2]; - /* Is there enough space in the buffer? */ + /* Is there enough space in the buffers? */ if (p->nr_parts_in > p->size_parts_in) { do { p->size_parts_in *= proxy_buffgrow; @@ -264,6 +271,15 @@ void proxy_parts_exch2(struct proxy *p) { p->size_gparts_in)) == NULL) error("Failed to re-allocate gparts_in buffers."); } + if (p->nr_sparts_in > p->size_sparts_in) { + do { + p->size_sparts_in *= proxy_buffgrow; + } while (p->nr_sparts_in > p->size_sparts_in); + free(p->sparts_in); + if ((p->sparts_in = (struct spart *)malloc(sizeof(struct spart) * + p->size_sparts_in)) == NULL) + error("Failed to re-allocate sparts_in buffers."); + } /* Receive the particle buffers. */ if (p->nr_parts_in > 0) { @@ -285,6 +301,14 @@ void proxy_parts_exch2(struct proxy *p) { // message( "irecv gpart data (%i) from node %i." , p->nr_gparts_in , // p->nodeID ); fflush(stdout); } + if (p->nr_sparts_in > 0) { + if (MPI_Irecv(p->sparts_in, p->nr_sparts_in, spart_mpi_type, p->nodeID, + p->nodeID * proxy_tag_shift + proxy_tag_sparts, + MPI_COMM_WORLD, &p->req_sparts_in) != MPI_SUCCESS) + error("Failed to irecv spart data."); + // message( "irecv gpart data (%i) from node %i." , p->nr_gparts_in , + // p->nodeID ); fflush(stdout); + } #else error("SWIFT was not compiled with MPI support."); @@ -299,7 +323,6 @@ void proxy_parts_exch2(struct proxy *p) { * @param xparts Pointer to an array of #xpart to send. * @param N The number of parts. */ - void proxy_parts_load(struct proxy *p, const struct part *parts, const struct xpart *xparts, int N) { @@ -332,13 +355,12 @@ void proxy_parts_load(struct proxy *p, const struct part *parts, } /** - * @brief Load parts onto a proxy for exchange. + * @brief Load gparts onto a proxy for exchange. * * @param p The #proxy. * @param gparts Pointer to an array of #gpart to send. - * @param N The number of parts. + * @param N The number of gparts. */ - void proxy_gparts_load(struct proxy *p, const struct gpart *gparts, int N) { /* Is there enough space in the buffer? */ @@ -362,6 +384,36 @@ void proxy_gparts_load(struct proxy *p, const struct gpart *gparts, int N) { p->nr_gparts_out += N; } +/** + * @brief Load sparts onto a proxy for exchange. + * + * @param p The #proxy. + * @param sparts Pointer to an array of #spart to send. + * @param N The number of sparts. + */ +void proxy_sparts_load(struct proxy *p, const struct spart *sparts, int N) { + + /* Is there enough space in the buffer? */ + if (p->nr_sparts_out + N > p->size_sparts_out) { + do { + p->size_sparts_out *= proxy_buffgrow; + } while (p->nr_sparts_out + N > p->size_sparts_out); + struct spart *tp; + if ((tp = (struct spart *)malloc(sizeof(struct spart) * + p->size_sparts_out)) == NULL) + error("Failed to re-allocate sparts_out buffers."); + memcpy(tp, p->sparts_out, sizeof(struct spart) * p->nr_sparts_out); + free(p->sparts_out); + p->sparts_out = tp; + } + + /* Copy the parts and xparts data to the buffer. */ + memcpy(&p->sparts_out[p->nr_sparts_out], sparts, sizeof(struct spart) * N); + + /* Increase the counters. */ + p->nr_sparts_out += N; +} + /** * @brief Initialize the given proxy. * @@ -369,7 +421,6 @@ void proxy_gparts_load(struct proxy *p, const struct gpart *gparts, int N) { * @param mynodeID The node this proxy is running on. * @param nodeID The node with which this proxy will communicate. */ - void proxy_init(struct proxy *p, int mynodeID, int nodeID) { /* Set the nodeID. */ @@ -427,4 +478,20 @@ void proxy_init(struct proxy *p, int mynodeID, int nodeID) { error("Failed to allocate gparts_out buffers."); } p->nr_gparts_out = 0; + + /* Allocate the spart send and receive buffers, if needed. */ + if (p->sparts_in == NULL) { + p->size_sparts_in = proxy_buffinit; + if ((p->sparts_in = (struct spart *)malloc(sizeof(struct spart) * + p->size_sparts_in)) == NULL) + error("Failed to allocate sparts_in buffers."); + } + p->nr_sparts_in = 0; + if (p->sparts_out == NULL) { + p->size_sparts_out = proxy_buffinit; + if ((p->sparts_out = (struct spart *)malloc(sizeof(struct spart) * + p->size_sparts_out)) == NULL) + error("Failed to allocate sparts_out buffers."); + } + p->nr_sparts_out = 0; } diff --git a/src/proxy.h b/src/proxy.h index 5a747187e05a78a109ce4523ebb3c9d5fe2ad717..a245077193878bb669b474944965badceffcee80 100644 --- a/src/proxy.h +++ b/src/proxy.h @@ -33,7 +33,8 @@ #define proxy_tag_parts 1 #define proxy_tag_xparts 2 #define proxy_tag_gparts 3 -#define proxy_tag_cells 4 +#define proxy_tag_sparts 4 +#define proxy_tag_cells 5 /* Data structure for the proxy. */ struct proxy { @@ -55,13 +56,16 @@ struct proxy { struct part *parts_in, *parts_out; struct xpart *xparts_in, *xparts_out; struct gpart *gparts_in, *gparts_out; + struct spart *sparts_in, *sparts_out; int size_parts_in, size_parts_out; int nr_parts_in, nr_parts_out; int size_gparts_in, size_gparts_out; int nr_gparts_in, nr_gparts_out; + int size_sparts_in, size_sparts_out; + int nr_sparts_in, nr_sparts_out; /* Buffer to hold the incomming/outgoing particle counts. */ - int buff_out[2], buff_in[2]; + int buff_out[3], buff_in[3]; /* MPI request handles. */ #ifdef WITH_MPI @@ -69,6 +73,7 @@ struct proxy { MPI_Request req_parts_out, req_parts_in; MPI_Request req_xparts_out, req_xparts_in; MPI_Request req_gparts_out, req_gparts_in; + MPI_Request req_sparts_out, req_sparts_in; MPI_Request req_cells_count_out, req_cells_count_in; MPI_Request req_cells_out, req_cells_in; #endif @@ -79,6 +84,7 @@ void proxy_init(struct proxy *p, int mynodeID, int nodeID); void proxy_parts_load(struct proxy *p, const struct part *parts, const struct xpart *xparts, int N); void proxy_gparts_load(struct proxy *p, const struct gpart *gparts, int N); +void proxy_sparts_load(struct proxy *p, const struct spart *sparts, int N); void proxy_parts_exch1(struct proxy *p); void proxy_parts_exch2(struct proxy *p); void proxy_addcell_in(struct proxy *p, struct cell *c); diff --git a/src/riemann/riemann_exact_isothermal.h b/src/riemann/riemann_exact_isothermal.h index 8e8a6a4dcce456b55f85148bed7342ad4e651c1b..26fb649f02ddd62c102e31c9191c20f9003c0133 100644 --- a/src/riemann/riemann_exact_isothermal.h +++ b/src/riemann/riemann_exact_isothermal.h @@ -112,7 +112,7 @@ __attribute__((always_inline)) INLINE static float riemann_guess_rho(float* WL, /* Currently three possibilities and not really an algorithm to decide which one to choose: */ /* just the average */ - return 0.5f * (WL[0] + WR[0]); + // return 0.5f * (WL[0] + WR[0]); /* two rarefaction approximation */ return sqrtf(WL[0] * WR[0] * expf((vL - vR) / const_isothermal_soundspeed)); @@ -276,12 +276,24 @@ __attribute__((always_inline)) INLINE static void riemann_solver_solve( vL = WL[1] * n_unit[0] + WL[2] * n_unit[1] + WL[3] * n_unit[2]; vR = WR[1] * n_unit[0] + WR[2] * n_unit[1] + WR[3] * n_unit[2]; - /* VACUUM... */ + if (WL[0] == 0. || WR[0] == 0.) { + error( + "One of the states is vacuum, the isothermal solver cannot solve " + "this!"); + } rho = 0.; /* obtain a first guess for p */ rhoguess = riemann_guess_rho(WL, WR, vL, vR); - frho = riemann_f(rho, WL, WR, vL, vR); + +#ifdef SWIFT_DEBUG_CHECKS + if (rhoguess <= 0.) { + error("Zero or negative initial density guess."); + } +#endif + + /* we know that the value of riemann_f for rho=0 is negative (it is -inf). */ + frho = -1.; frhoguess = riemann_f(rhoguess, WL, WR, vL, vR); /* ok, rhostar is close to 0, better use Brent's method... */ /* we use Newton-Raphson until we find a suitable interval */ @@ -289,14 +301,18 @@ __attribute__((always_inline)) INLINE static void riemann_solver_solve( /* Newton-Raphson until convergence or until suitable interval is found to use Brent's method */ unsigned int counter = 0; - while (fabs(rho - rhoguess) > 1.e-6f * 0.5f * (rho + rhoguess) && + while (fabs(rho - rhoguess) > 5.e-7f * (rho + rhoguess) && frhoguess < 0.0f) { rho = rhoguess; rhoguess = rhoguess - frhoguess / riemann_fprime(rhoguess, WL, WR); frhoguess = riemann_f(rhoguess, WL, WR, vL, vR); counter++; if (counter > 1000) { - error("Stuck in Newton-Raphson!\n"); + error( + "Stuck in Newton-Raphson (rho: %g, rhoguess: %g, frhoguess: %g, " + "fprime: %g, rho-rhoguess: %g, WL: %g %g %g, WR: %g %g %g)!\n", + rho, rhoguess, frhoguess, riemann_fprime(rhoguess, WL, WR), + (rho - rhoguess), WL[0], vL, WL[4], WR[0], vR, WR[4]); } } } @@ -304,7 +320,7 @@ __attribute__((always_inline)) INLINE static void riemann_solver_solve( if (1.e6 * fabs(rho - rhoguess) > 0.5f * (rho + rhoguess) && frhoguess > 0.0f) { rho = 0.0f; - frho = riemann_f(rho, WL, WR, vL, vR); + frho = -1.; /* use Brent's method to find the zeropoint */ rho = riemann_solve_brent(rho, rhoguess, frho, frhoguess, 1.e-6, WL, WR, vL, vR); diff --git a/src/runner.c b/src/runner.c index f1ab18ec8af3504f96d7a39eede13f8ba843f188..7308d85a2bd9d37c231bab44a2518294e04d3190 100644 --- a/src/runner.c +++ b/src/runner.c @@ -57,6 +57,7 @@ #include "scheduler.h" #include "sourceterms.h" #include "space.h" +#include "stars.h" #include "task.h" #include "timers.h" #include "timestep.h" @@ -209,7 +210,7 @@ void runner_do_cooling(struct runner *r, struct cell *c, int timer) { const struct engine *e = r->e; const struct cooling_function_data *cooling_func = e->cooling_func; const struct phys_const *constants = e->physical_constants; - const struct UnitSystem *us = e->internalUnits; + const struct unit_system *us = e->internal_units; const double timeBase = e->timeBase; TIMER_TIC; @@ -453,9 +454,11 @@ void runner_do_sort(struct runner *r, struct cell *c, int flags, int clock) { /* Fill the sort array. */ for (int k = 0; k < count; k++) { - xparts[k].x_diff_sort[0] = 0.0f; - xparts[k].x_diff_sort[1] = 0.0f; - xparts[k].x_diff_sort[2] = 0.0f; + if (xparts != NULL) { + xparts[k].x_diff_sort[0] = 0.0f; + xparts[k].x_diff_sort[1] = 0.0f; + xparts[k].x_diff_sort[2] = 0.0f; + } const float px[3] = {parts[k].x[0], parts[k].x[1], parts[k].x[2]}; for (int j = 0; j < 13; j++) if (flags & (1 << j)) { @@ -524,6 +527,10 @@ void runner_do_init(struct runner *r, struct cell *c, int timer) { /* Anything to do here? */ if (!cell_is_active(c, e)) return; + /* Reset the gravity acceleration tensors */ + if (e->policy & engine_policy_self_gravity) + gravity_field_tensor_init(c->multipole); + /* Recurse? */ if (c->split) { for (int k = 0; k < 8; k++) @@ -620,14 +627,15 @@ void runner_do_ghost(struct runner *r, struct cell *c, int timer) { struct part *restrict parts = c->parts; struct xpart *restrict xparts = c->xparts; - int redo, count = c->count; const struct engine *e = r->e; + const float hydro_h_max = e->hydro_properties->h_max; const float target_wcount = e->hydro_properties->target_neighbours; const float max_wcount = target_wcount + e->hydro_properties->delta_neighbours; const float min_wcount = target_wcount - e->hydro_properties->delta_neighbours; const int max_smoothing_iter = e->hydro_properties->max_smoothing_iterations; + int redo = 0, count = 0; TIMER_TIC; @@ -640,11 +648,15 @@ void runner_do_ghost(struct runner *r, struct cell *c, int timer) { if (c->progeny[k] != NULL) runner_do_ghost(r, c->progeny[k], 0); } else { - /* Init the IDs that have to be updated. */ + /* Init the list of active particles that have to be updated. */ int *pid = NULL; - if ((pid = malloc(sizeof(int) * count)) == NULL) + if ((pid = malloc(sizeof(int) * c->count)) == NULL) error("Can't allocate memory for pid."); - for (int k = 0; k < count; k++) pid[k] = k; + for (int k = 0; k < c->count; k++) + if (part_is_active(&parts[k], e)) { + pid[count] = k; + ++count; + } /* While there are particles that need to be updated... */ for (int num_reruns = 0; count > 0 && num_reruns < max_smoothing_iter; @@ -653,18 +665,23 @@ void runner_do_ghost(struct runner *r, struct cell *c, int timer) { /* Reset the redo-count. */ redo = 0; - /* Loop over the parts in this cell. */ + /* Loop over the remaining active parts in this cell. */ for (int i = 0; i < count; i++) { /* Get a direct pointer on the part. */ struct part *restrict p = &parts[pid[i]]; struct xpart *restrict xp = &xparts[pid[i]]; +#ifdef SWIFT_DEBUG_CHECKS /* Is this part within the timestep? */ - if (part_is_active(p, e)) { + if (!part_is_active(p, e)) error("Ghost applied to inactive particle"); +#endif + + /* Finish the density calculation */ + hydro_end_density(p); - /* Finish the density calculation */ - hydro_end_density(p); + /* Did we get the right number of neighbours? */ + if (p->density.wcount > max_wcount || p->density.wcount < min_wcount) { float h_corr = 0.f; @@ -680,12 +697,11 @@ void runner_do_ghost(struct runner *r, struct cell *c, int timer) { h_corr = (h_corr > -0.5f * p->h) ? h_corr : -0.5f * p->h; } - /* Did we get the right number density? */ - if (p->density.wcount > max_wcount || - p->density.wcount < min_wcount) { + /* Ok, correct then */ + p->h += h_corr; - /* Ok, correct then */ - p->h += h_corr; + /* If below the absolute maximum, try again */ + if (p->h < hydro_h_max) { /* Flag for another round of fun */ pid[redo] = pid[i]; @@ -696,21 +712,25 @@ void runner_do_ghost(struct runner *r, struct cell *c, int timer) { /* Off we go ! */ continue; + } else { + + /* Ok, this particle is a lost cause... */ + p->h = hydro_h_max; } + } - /* We now have a particle whose smoothing length has converged */ + /* We now have a particle whose smoothing length has converged */ - /* As of here, particle force variables will be set. */ + /* As of here, particle force variables will be set. */ - /* Compute variables required for the force loop */ - hydro_prepare_force(p, xp); + /* Compute variables required for the force loop */ + hydro_prepare_force(p, xp); - /* The particle force values are now set. Do _NOT_ - try to read any particle density variables! */ + /* The particle force values are now set. Do _NOT_ + try to read any particle density variables! */ - /* Prepare the particle for the force loop over neighbours */ - hydro_reset_acceleration(p); - } + /* Prepare the particle for the force loop over neighbours */ + hydro_reset_acceleration(p); } /* We now need to treat the particles whose smoothing length had not @@ -834,34 +854,15 @@ void runner_do_unskip_mapper(void *map_data, int num_elements, * @param c The cell. * @param timer Are we timing this ? */ -void runner_do_drift(struct runner *r, struct cell *c, int timer) { +void runner_do_drift_particles(struct runner *r, struct cell *c, int timer) { TIMER_TIC; - cell_drift(c, r->e); + cell_drift_particles(c, r->e); if (timer) TIMER_TOC(timer_drift); } -/** - * @brief Mapper function to drift ALL particle and g-particles forward in time. - * - * @param map_data An array of #cell%s. - * @param num_elements Chunk size. - * @param extra_data Pointer to an #engine. - */ -void runner_do_drift_mapper(void *map_data, int num_elements, - void *extra_data) { - - struct engine *e = (struct engine *)extra_data; - struct cell *cells = (struct cell *)map_data; - - for (int ind = 0; ind < num_elements; ind++) { - struct cell *c = &cells[ind]; - if (c != NULL && c->nodeID == e->nodeID) cell_drift(c, e); - } -} - /** * @brief Perform the first half-kick on all the active particles in a cell. * @@ -875,15 +876,17 @@ void runner_do_kick1(struct runner *r, struct cell *c, int timer) { struct part *restrict parts = c->parts; struct xpart *restrict xparts = c->xparts; struct gpart *restrict gparts = c->gparts; + struct spart *restrict sparts = c->sparts; const int count = c->count; const int gcount = c->gcount; + const int scount = c->scount; const integertime_t ti_current = e->ti_current; const double timeBase = e->timeBase; TIMER_TIC; /* Anything to do here? */ - if (!cell_is_active(c, e)) return; + if (!cell_is_starting(c, e)) return; /* Recurse? */ if (c->split) { @@ -899,17 +902,17 @@ void runner_do_kick1(struct runner *r, struct cell *c, int timer) { struct xpart *restrict xp = &xparts[k]; /* If particle needs to be kicked */ - if (part_is_active(p, e)) { + if (part_is_starting(p, e)) { const integertime_t ti_step = get_integer_timestep(p->time_bin); const integertime_t ti_begin = - get_integer_time_begin(ti_current, p->time_bin); + get_integer_time_begin(ti_current + 1, p->time_bin); #ifdef SWIFT_DEBUG_CHECKS const integertime_t ti_end = - get_integer_time_end(ti_current, p->time_bin); + get_integer_time_end(ti_current + 1, p->time_bin); - if (ti_end - ti_begin != ti_step) + if (ti_begin != ti_current) error( "Particle in wrong time-bin, ti_end=%lld, ti_begin=%lld, " "ti_step=%lld time_bin=%d ti_current=%lld", @@ -928,24 +931,58 @@ void runner_do_kick1(struct runner *r, struct cell *c, int timer) { struct gpart *restrict gp = &gparts[k]; /* If the g-particle has no counterpart and needs to be kicked */ - if (gp->id_or_neg_offset > 0 && gpart_is_active(gp, e)) { + if (gp->type == swift_type_dark_matter && gpart_is_starting(gp, e)) { const integertime_t ti_step = get_integer_timestep(gp->time_bin); const integertime_t ti_begin = - get_integer_time_begin(ti_current, gp->time_bin); + get_integer_time_begin(ti_current + 1, gp->time_bin); #ifdef SWIFT_DEBUG_CHECKS const integertime_t ti_end = - get_integer_time_end(ti_current, gp->time_bin); + get_integer_time_end(ti_current + 1, gp->time_bin); - if (ti_end - ti_begin != ti_step) error("Particle in wrong time-bin"); + if (ti_begin != ti_current) + error( + "Particle in wrong time-bin, ti_end=%lld, ti_begin=%lld, " + "ti_step=%lld time_bin=%d ti_current=%lld", + ti_end, ti_begin, ti_step, gp->time_bin, ti_current); #endif /* do the kick */ kick_gpart(gp, ti_begin, ti_begin + ti_step / 2, timeBase); } } + + /* Loop over the star particles in this cell. */ + for (int k = 0; k < scount; k++) { + + /* Get a handle on the s-part. */ + struct spart *restrict sp = &sparts[k]; + + /* If particle needs to be kicked */ + if (spart_is_starting(sp, e)) { + + const integertime_t ti_step = get_integer_timestep(sp->time_bin); + const integertime_t ti_begin = + get_integer_time_begin(ti_current + 1, sp->time_bin); + +#ifdef SWIFT_DEBUG_CHECKS + const integertime_t ti_end = + get_integer_time_end(ti_current + 1, sp->time_bin); + + if (ti_begin != ti_current) + error( + "Particle in wrong time-bin, ti_end=%lld, ti_begin=%lld, " + "ti_step=%lld time_bin=%d ti_current=%lld", + ti_end, ti_begin, ti_step, sp->time_bin, ti_current); +#endif + + /* do the kick */ + kick_spart(sp, ti_begin, ti_begin + ti_step / 2, timeBase); + } + } } + if (timer) TIMER_TOC(timer_kick1); } @@ -965,9 +1002,11 @@ void runner_do_kick2(struct runner *r, struct cell *c, int timer) { const double timeBase = e->timeBase; const int count = c->count; const int gcount = c->gcount; + const int scount = c->scount; struct part *restrict parts = c->parts; struct xpart *restrict xparts = c->xparts; struct gpart *restrict gparts = c->gparts; + struct spart *restrict sparts = c->sparts; TIMER_TIC; @@ -1005,6 +1044,11 @@ void runner_do_kick2(struct runner *r, struct cell *c, int timer) { /* Finish the time-step with a second half-kick */ kick_part(p, xp, ti_begin + ti_step / 2, ti_begin + ti_step, timeBase); +#ifdef SWIFT_DEBUG_CHECKS + /* Check that kick and the drift are synchronized */ + if (p->ti_drift != p->ti_kick) error("Error integrating part in time."); +#endif + /* Prepare the values to be drifted */ hydro_reset_predicted_values(p, xp); } @@ -1017,7 +1061,7 @@ void runner_do_kick2(struct runner *r, struct cell *c, int timer) { struct gpart *restrict gp = &gparts[k]; /* If the g-particle has no counterpart and needs to be kicked */ - if (gp->id_or_neg_offset > 0 && gpart_is_active(gp, e)) { + if (gp->type == swift_type_dark_matter && gpart_is_active(gp, e)) { const integertime_t ti_step = get_integer_timestep(gp->time_bin); const integertime_t ti_begin = @@ -1030,6 +1074,47 @@ void runner_do_kick2(struct runner *r, struct cell *c, int timer) { /* Finish the time-step with a second half-kick */ kick_gpart(gp, ti_begin + ti_step / 2, ti_begin + ti_step, timeBase); + +#ifdef SWIFT_DEBUG_CHECKS + /* Check that kick and the drift are synchronized */ + if (gp->ti_drift != gp->ti_kick) + error("Error integrating g-part in time."); +#endif + + /* Prepare the values to be drifted */ + gravity_reset_predicted_values(gp); + } + } + + /* Loop over the particles in this cell. */ + for (int k = 0; k < scount; k++) { + + /* Get a handle on the part. */ + struct spart *restrict sp = &sparts[k]; + + /* If particle needs to be kicked */ + if (spart_is_active(sp, e)) { + + const integertime_t ti_step = get_integer_timestep(sp->time_bin); + const integertime_t ti_begin = + get_integer_time_begin(ti_current, sp->time_bin); + +#ifdef SWIFT_DEBUG_CHECKS + if (ti_begin + ti_step != ti_current) + error("Particle in wrong time-bin"); +#endif + + /* Finish the time-step with a second half-kick */ + kick_spart(sp, ti_begin + ti_step / 2, ti_begin + ti_step, timeBase); + +#ifdef SWIFT_DEBUG_CHECKS + /* Check that kick and the drift are synchronized */ + if (sp->ti_drift != sp->ti_kick) + error("Error integrating s-part in time."); +#endif + + /* Prepare the values to be drifted */ + star_reset_predicted_values(sp); } } } @@ -1050,14 +1135,17 @@ void runner_do_timestep(struct runner *r, struct cell *c, int timer) { const integertime_t ti_current = e->ti_current; const int count = c->count; const int gcount = c->gcount; + const int scount = c->scount; struct part *restrict parts = c->parts; struct xpart *restrict xparts = c->xparts; struct gpart *restrict gparts = c->gparts; + struct spart *restrict sparts = c->sparts; + const double timeBase = e->timeBase; TIMER_TIC; - int updated = 0, g_updated = 0; - integertime_t ti_end_min = max_nr_timesteps, ti_end_max = 0; + int updated = 0, g_updated = 0, s_updated = 0; + integertime_t ti_end_min = max_nr_timesteps, ti_end_max = 0, ti_beg_max = 0; /* No children? */ if (!c->split) { @@ -1088,6 +1176,10 @@ void runner_do_timestep(struct runner *r, struct cell *c, int timer) { p->time_bin = get_time_bin(ti_new_step); if (p->gpart != NULL) p->gpart->time_bin = get_time_bin(ti_new_step); + /* Tell the particle what the new physical time step is */ + float dt = get_timestep(p->time_bin, timeBase); + hydro_timestep_extra(p, dt); + /* Number of updated particles */ updated++; if (p->gpart != NULL) g_updated++; @@ -1095,6 +1187,9 @@ void runner_do_timestep(struct runner *r, struct cell *c, int timer) { /* What is the next sync-point ? */ ti_end_min = min(ti_current + ti_new_step, ti_end_min); ti_end_max = max(ti_current + ti_new_step, ti_end_max); + + /* What is the next starting point for this cell ? */ + ti_beg_max = max(ti_current, ti_beg_max); } else { /* part is inactive */ @@ -1105,6 +1200,12 @@ void runner_do_timestep(struct runner *r, struct cell *c, int timer) { /* What is the next sync-point ? */ ti_end_min = min(ti_end, ti_end_min); ti_end_max = max(ti_end, ti_end_max); + + const integertime_t ti_beg = + get_integer_time_begin(ti_current + 1, p->time_bin); + + /* What is the next starting point for this cell ? */ + ti_beg_max = max(ti_beg, ti_beg_max); } } @@ -1115,7 +1216,7 @@ void runner_do_timestep(struct runner *r, struct cell *c, int timer) { struct gpart *restrict gp = &gparts[k]; /* If the g-particle has no counterpart */ - if (gp->id_or_neg_offset > 0) { + if (gp->type == swift_type_dark_matter) { /* need to be updated ? */ if (gpart_is_active(gp, e)) { @@ -1141,6 +1242,10 @@ void runner_do_timestep(struct runner *r, struct cell *c, int timer) { /* What is the next sync-point ? */ ti_end_min = min(ti_current + ti_new_step, ti_end_min); ti_end_max = max(ti_current + ti_new_step, ti_end_max); + + /* What is the next starting point for this cell ? */ + ti_beg_max = max(ti_current, ti_beg_max); + } else { /* gpart is inactive */ const integertime_t ti_end = @@ -1149,9 +1254,67 @@ void runner_do_timestep(struct runner *r, struct cell *c, int timer) { /* What is the next sync-point ? */ ti_end_min = min(ti_end, ti_end_min); ti_end_max = max(ti_end, ti_end_max); + + const integertime_t ti_beg = + get_integer_time_begin(ti_current + 1, gp->time_bin); + + /* What is the next starting point for this cell ? */ + ti_beg_max = max(ti_beg, ti_beg_max); } } } + + /* Loop over the star particles in this cell. */ + for (int k = 0; k < scount; k++) { + + /* Get a handle on the part. */ + struct spart *restrict sp = &sparts[k]; + + /* need to be updated ? */ + if (spart_is_active(sp, e)) { + +#ifdef SWIFT_DEBUG_CHECKS + /* Current end of time-step */ + const integertime_t ti_end = + get_integer_time_end(ti_current, sp->time_bin); + + if (ti_end != ti_current) + error("Computing time-step of rogue particle."); +#endif + /* Get new time-step */ + const integertime_t ti_new_step = get_spart_timestep(sp, e); + + /* Update particle */ + sp->time_bin = get_time_bin(ti_new_step); + sp->gpart->time_bin = get_time_bin(ti_new_step); + + /* Number of updated s-particles */ + s_updated++; + g_updated++; + + /* What is the next sync-point ? */ + ti_end_min = min(ti_current + ti_new_step, ti_end_min); + ti_end_max = max(ti_current + ti_new_step, ti_end_max); + + /* What is the next starting point for this cell ? */ + ti_beg_max = max(ti_current, ti_beg_max); + + } else { /* star particle is inactive */ + + const integertime_t ti_end = + get_integer_time_end(ti_current, sp->time_bin); + + /* What is the next sync-point ? */ + ti_end_min = min(ti_end, ti_end_min); + ti_end_max = max(ti_end, ti_end_max); + + const integertime_t ti_beg = + get_integer_time_begin(ti_current + 1, sp->time_bin); + + /* What is the next starting point for this cell ? */ + ti_beg_max = max(ti_beg, ti_beg_max); + } + } } else { /* Loop over the progeny. */ @@ -1165,16 +1328,20 @@ void runner_do_timestep(struct runner *r, struct cell *c, int timer) { /* And aggregate */ updated += cp->updated; g_updated += cp->g_updated; + s_updated += cp->s_updated; ti_end_min = min(cp->ti_end_min, ti_end_min); ti_end_max = max(cp->ti_end_max, ti_end_max); + ti_beg_max = max(cp->ti_beg_max, ti_beg_max); } } /* Store the values. */ c->updated = updated; c->g_updated = g_updated; + c->s_updated = s_updated; c->ti_end_min = ti_end_min; c->ti_end_max = ti_end_max; + c->ti_beg_max = ti_beg_max; if (timer) TIMER_TOC(timer_timestep); } @@ -1192,8 +1359,10 @@ void runner_do_end_force(struct runner *r, struct cell *c, int timer) { const struct engine *e = r->e; const int count = c->count; const int gcount = c->gcount; + const int scount = c->scount; struct part *restrict parts = c->parts; struct gpart *restrict gparts = c->gparts; + struct spart *restrict sparts = c->sparts; const double const_G = e->physical_constants->const_newton_G; TIMER_TIC; @@ -1207,7 +1376,7 @@ void runner_do_end_force(struct runner *r, struct cell *c, int timer) { if (c->progeny[k] != NULL) runner_do_end_force(r, c->progeny[k], 0); } else { - /* Loop over the particles in this cell. */ + /* Loop over the gas particles in this cell. */ for (int k = 0; k < count; k++) { /* Get a handle on the part. */ @@ -1227,8 +1396,33 @@ void runner_do_end_force(struct runner *r, struct cell *c, int timer) { /* Get a handle on the gpart. */ struct gpart *restrict gp = &gparts[k]; - if (gp->id_or_neg_offset > 0 && gpart_is_active(gp, e)) { - gravity_end_force(gp, const_G); + if (gp->type == swift_type_dark_matter) { + + if (gpart_is_active(gp, e)) gravity_end_force(gp, const_G); + } + +#ifdef SWIFT_DEBUG_CHECKS + if (e->policy & engine_policy_self_gravity) { + gp->mass_interacted += gp->mass; + if (fabs(gp->mass_interacted - e->s->total_mass) > gp->mass) + error( + "g-particle did not interact gravitationally with all other " + "particles gp->mass_interacted=%e, total_mass=%e, gp->mass=%e", + gp->mass_interacted, e->s->total_mass, gp->mass); + } +#endif + } + + /* Loop over the star particles in this cell. */ + for (int k = 0; k < scount; k++) { + + /* Get a handle on the spart. */ + struct spart *restrict sp = &sparts[k]; + if (spart_is_active(sp, e)) { + + /* First, finish the force loop */ + star_end_force(sp); + gravity_end_force(sp->gpart, const_G); } } } @@ -1255,6 +1449,8 @@ void runner_do_recv_part(struct runner *r, struct cell *c, int timer) { integertime_t ti_end_min = max_nr_timesteps; integertime_t ti_end_max = 0; + timebin_t time_bin_min = num_time_bins; + timebin_t time_bin_max = 0; float h_max = 0.f; /* If this cell is a leaf, collect the particle data. */ @@ -1262,10 +1458,9 @@ void runner_do_recv_part(struct runner *r, struct cell *c, int timer) { /* Collect everything... */ for (size_t k = 0; k < nr_parts; k++) { - const integertime_t ti_end = - get_integer_time_end(ti_current, parts[k].time_bin); - ti_end_min = min(ti_end_min, ti_end); - ti_end_max = max(ti_end_max, ti_end); + if (parts[k].time_bin == time_bin_inhibited) continue; + time_bin_min = min(time_bin_min, parts[k].time_bin); + time_bin_max = max(time_bin_max, parts[k].time_bin); h_max = max(h_max, parts[k].h); #ifdef SWIFT_DEBUG_CHECKS @@ -1273,6 +1468,10 @@ void runner_do_recv_part(struct runner *r, struct cell *c, int timer) { error("Received un-drifted particle !"); #endif } + + /* Convert into a time */ + ti_end_min = get_integer_time_end(ti_current, time_bin_min); + ti_end_max = get_integer_time_end(ti_current, time_bin_max); } /* Otherwise, recurse and collect. */ @@ -1327,17 +1526,27 @@ void runner_do_recv_gpart(struct runner *r, struct cell *c, int timer) { integertime_t ti_end_min = max_nr_timesteps; integertime_t ti_end_max = 0; + timebin_t time_bin_min = num_time_bins; + timebin_t time_bin_max = 0; /* If this cell is a leaf, collect the particle data. */ if (!c->split) { /* Collect everything... */ for (size_t k = 0; k < nr_gparts; k++) { - const integertime_t ti_end = - get_integer_time_end(ti_current, gparts[k].time_bin); - ti_end_min = min(ti_end_min, ti_end); - ti_end_max = max(ti_end_max, ti_end); + if (gparts[k].time_bin == time_bin_inhibited) continue; + time_bin_min = min(time_bin_min, gparts[k].time_bin); + time_bin_max = max(time_bin_max, gparts[k].time_bin); + +#ifdef SWIFT_DEBUG_CHECKS + if (gparts[k].ti_drift != ti_current) + error("Received un-drifted g-particle !"); +#endif } + + /* Convert into a time */ + ti_end_min = get_integer_time_end(ti_current, time_bin_min); + ti_end_max = get_integer_time_end(ti_current, time_bin_max); } /* Otherwise, recurse and collect. */ @@ -1371,6 +1580,79 @@ void runner_do_recv_gpart(struct runner *r, struct cell *c, int timer) { #endif } +/** + * @brief Construct the cell properties from the received #spart. + * + * @param r The runner thread. + * @param c The cell. + * @param timer Are we timing this ? + */ +void runner_do_recv_spart(struct runner *r, struct cell *c, int timer) { + +#ifdef WITH_MPI + + const struct spart *restrict sparts = c->sparts; + const size_t nr_sparts = c->scount; + const integertime_t ti_current = r->e->ti_current; + + TIMER_TIC; + + integertime_t ti_end_min = max_nr_timesteps; + integertime_t ti_end_max = 0; + timebin_t time_bin_min = num_time_bins; + timebin_t time_bin_max = 0; + + /* If this cell is a leaf, collect the particle data. */ + if (!c->split) { + + /* Collect everything... */ + for (size_t k = 0; k < nr_sparts; k++) { + if (sparts[k].time_bin == time_bin_inhibited) continue; + time_bin_min = min(time_bin_min, sparts[k].time_bin); + time_bin_max = max(time_bin_max, sparts[k].time_bin); + +#ifdef SWIFT_DEBUG_CHECKS + if (sparts[k].ti_drift != ti_current) + error("Received un-drifted s-particle !"); +#endif + } + + /* Convert into a time */ + ti_end_min = get_integer_time_end(ti_current, time_bin_min); + ti_end_max = get_integer_time_end(ti_current, time_bin_max); + } + + /* Otherwise, recurse and collect. */ + else { + for (int k = 0; k < 8; k++) { + if (c->progeny[k] != NULL) { + runner_do_recv_spart(r, c->progeny[k], 0); + ti_end_min = min(ti_end_min, c->progeny[k]->ti_end_min); + ti_end_max = max(ti_end_max, c->progeny[k]->ti_end_max); + } + } + } + +#ifdef SWIFT_DEBUG_CHECKS + if (ti_end_min < ti_current) + error( + "Received a cell at an incorrect time c->ti_end_min=%lld, " + "e->ti_current=%lld.", + ti_end_min, ti_current); +#endif + + /* ... and store. */ + c->ti_end_min = ti_end_min; + c->ti_end_max = ti_end_max; + c->ti_old = ti_current; + + if (timer) TIMER_TOC(timer_dorecv_spart); + +#else + error("SWIFT was not compiled with MPI support."); +#endif +} + /** * @brief The #runner main thread routine. * @@ -1410,26 +1692,25 @@ void *runner_main(void *data) { /* Get the cells. */ struct cell *ci = t->ci; struct cell *cj = t->cj; + +/* Mark the thread we run on */ #ifdef SWIFT_DEBUG_TASKS t->rid = r->cpuid; #endif -#ifdef SWIFT_DEBUG_CHECKS - t->ti_run = e->ti_current; -#endif /* Check that we haven't scheduled an inactive task */ #ifdef SWIFT_DEBUG_CHECKS + t->ti_run = e->ti_current; #ifndef WITH_MPI if (ci == NULL && cj == NULL) { - if (t->type != task_type_grav_gather_m && t->type != task_type_grav_fft) - error("Task not associated with cells!"); + error("Task not associated with cells!"); } else if (cj == NULL) { /* self */ if (!cell_is_active(ci, e) && t->type != task_type_sort && - t->type != task_type_drift && t->type != task_type_send && - t->type != task_type_recv) + t->type != task_type_send && t->type != task_type_recv && + t->type != task_type_kick1 && t->type != task_type_drift) error( "Task (type='%s/%s') should have been skipped ti_current=%lld " "c->ti_end_min=%lld", @@ -1445,6 +1726,15 @@ void *runner_main(void *data) { taskID_names[t->type], subtaskID_names[t->subtype], e->ti_current, ci->ti_end_min, t->flags); + /* Special case for kick1 */ + if (!cell_is_starting(ci, e) && t->type == task_type_kick1 && + t->flags == 0) + error( + "Task (type='%s/%s') should have been skipped ti_current=%lld " + "c->ti_end_min=%lld t->flags=%d", + taskID_names[t->type], subtaskID_names[t->subtype], e->ti_current, + ci->ti_end_min, t->flags); + } else { /* pair */ if (!cell_is_active(ci, e) && !cell_is_active(cj, e)) @@ -1544,7 +1834,7 @@ void *runner_main(void *data) { break; #endif case task_type_drift: - runner_do_drift(r, ci, 1); + runner_do_drift_particles(r, ci, 1); break; case task_type_kick1: runner_do_kick1(r, ci, 1); @@ -1572,19 +1862,26 @@ void *runner_main(void *data) { runner_do_recv_part(r, ci, 1); } else if (t->subtype == task_subtype_gpart) { runner_do_recv_gpart(r, ci, 1); + } else if (t->subtype == task_subtype_spart) { + runner_do_recv_spart(r, ci, 1); + } else if (t->subtype == task_subtype_multipole) { + ci->ti_old_multipole = e->ti_current; + } else { + error("Unknown/invalid task subtype (%d).", t->subtype); } break; #endif case task_type_grav_mm: - runner_do_grav_mm(r, t->ci, 1); + // runner_do_grav_mm(r, t->ci, 1); break; - case task_type_grav_up: - runner_do_grav_up(r, t->ci); + case task_type_grav_down: + runner_do_grav_down(r, t->ci); break; - case task_type_grav_gather_m: + case task_type_grav_top_level: + // runner_do_grav_top_level(r); break; - case task_type_grav_fft: - runner_do_grav_fft(r); + case task_type_grav_long_range: + runner_do_grav_long_range(r, t->ci, 1); break; case task_type_cooling: if (e->policy & engine_policy_cooling) runner_do_end_force(r, ci, 1); @@ -1597,6 +1894,18 @@ void *runner_main(void *data) { error("Unknown/invalid task type (%d).", t->type); } +/* Mark that we have run this task on these cells */ +#ifdef SWIFT_DEBUG_CHECKS + if (ci != NULL) { + ci->tasks_executed[t->type]++; + ci->subtasks_executed[t->subtype]++; + } + if (cj != NULL) { + cj->tasks_executed[t->type]++; + cj->subtasks_executed[t->subtype]++; + } +#endif + /* We're done with this task, see if we get a next one. */ prev = t; t = scheduler_done(sched, t); diff --git a/src/runner.h b/src/runner.h index 53e78b00657385c7185e0730d421707c87ccf382..ec63eb3ec98547859f7da75809555f231f277f62 100644 --- a/src/runner.h +++ b/src/runner.h @@ -56,7 +56,7 @@ struct runner { void runner_do_ghost(struct runner *r, struct cell *c, int timer); void runner_do_extra_ghost(struct runner *r, struct cell *c, int timer); void runner_do_sort(struct runner *r, struct cell *c, int flag, int clock); -void runner_do_drift(struct runner *r, struct cell *c, int timer); +void runner_do_drift_particles(struct runner *r, struct cell *c, int timer); void runner_do_kick1(struct runner *r, struct cell *c, int timer); void runner_do_kick2(struct runner *r, struct cell *c, int timer); void runner_do_end_force(struct runner *r, struct cell *c, int timer); @@ -66,6 +66,7 @@ void runner_do_grav_external(struct runner *r, struct cell *c, int timer); void *runner_main(void *data); void runner_do_unskip_mapper(void *map_data, int num_elements, void *extra_data); -void runner_do_drift_mapper(void *map_data, int num_elements, void *extra_data); +void runner_do_drift_all_mapper(void *map_data, int num_elements, + void *extra_data); #endif /* SWIFT_RUNNER_H */ diff --git a/src/runner_doiact.h b/src/runner_doiact.h index ffa6492232d3e8387ea6d3eddce605ef8a249999..c0185737eefbdad008f46fe83fbdbf41794e1465 100644 --- a/src/runner_doiact.h +++ b/src/runner_doiact.h @@ -125,7 +125,7 @@ void DOPAIR1_NAIVE(struct runner *r, struct cell *restrict ci, const struct engine *e = r->e; #ifndef SWIFT_DEBUG_CHECKS -// error("Don't use in actual runs ! Slow code !"); + error("Don't use in actual runs ! Slow code !"); #endif #ifdef WITH_VECTORIZATION @@ -360,7 +360,7 @@ void DOSELF_NAIVE(struct runner *r, struct cell *restrict c) { const struct engine *e = r->e; #ifndef SWIFT_DEBUG_CHECKS -// error("Don't use in actual runs ! Slow code !"); + error("Don't use in actual runs ! Slow code !"); #endif #ifdef WITH_OLD_VECTORIZATION @@ -616,8 +616,6 @@ void DOPAIR_SUBSET(struct runner *r, struct cell *restrict ci, /* Have the cells been sorted? */ if (!(cj->sorted & (1 << sid)) || cj->dx_max_sort > space_maxreldx * cj->dmin) runner_do_sort(r, cj, (1 << sid), 1); - // if (!(cj->sorted & (1 << sid))) error("Trying to interact unsorted - // cells."); /* Pick-out the sorted lists. */ const struct entry *restrict sort_j = &cj->sort[sid * (cj->count + 1)]; @@ -900,8 +898,6 @@ void DOPAIR1(struct runner *r, struct cell *ci, struct cell *cj) { runner_do_sort(r, ci, (1 << sid), 1); if (!(cj->sorted & (1 << sid)) || cj->dx_max_sort > space_maxreldx * cj->dmin) runner_do_sort(r, cj, (1 << sid), 1); - // if (!(ci->sorted & (1 << sid)) || !(cj->sorted & (1 << sid))) - // error("Trying to interact unsorted cells."); /* Get the cutoff shift. */ double rshift = 0.0; @@ -943,145 +939,153 @@ void DOPAIR1(struct runner *r, struct cell *ci, struct cell *cj) { const double dj_min = sort_j[0].d; const float dx_max = (ci->dx_max_sort + cj->dx_max_sort); - /* Loop over the parts in ci. */ - for (int pid = count_i - 1; - pid >= 0 && sort_i[pid].d + hi_max + dx_max > dj_min; pid--) { + if (cell_is_active(ci, e)) { - /* Get a hold of the ith part in ci. */ - struct part *restrict pi = &parts_i[sort_i[pid].i]; - if (!part_is_active(pi, e)) continue; - const float hi = pi->h; - const double di = sort_i[pid].d + hi * kernel_gamma + dx_max - rshift; - if (di < dj_min) continue; + /* Loop over the parts in ci. */ + for (int pid = count_i - 1; + pid >= 0 && sort_i[pid].d + hi_max + dx_max > dj_min; pid--) { - double pix[3]; - for (int k = 0; k < 3; k++) pix[k] = pi->x[k] - shift[k]; - const float hig2 = hi * hi * kernel_gamma2; + /* Get a hold of the ith part in ci. */ + struct part *restrict pi = &parts_i[sort_i[pid].i]; + if (!part_is_active(pi, e)) continue; + const float hi = pi->h; + const double di = sort_i[pid].d + hi * kernel_gamma + dx_max - rshift; + if (di < dj_min) continue; - /* Loop over the parts in cj. */ - for (int pjd = 0; pjd < count_j && sort_j[pjd].d < di; pjd++) { + double pix[3]; + for (int k = 0; k < 3; k++) pix[k] = pi->x[k] - shift[k]; + const float hig2 = hi * hi * kernel_gamma2; - /* Get a pointer to the jth particle. */ - struct part *restrict pj = &parts_j[sort_j[pjd].i]; + /* Loop over the parts in cj. */ + for (int pjd = 0; pjd < count_j && sort_j[pjd].d < di; pjd++) { - /* Compute the pairwise distance. */ - float r2 = 0.0f; - float dx[3]; - for (int k = 0; k < 3; k++) { - dx[k] = pix[k] - pj->x[k]; - r2 += dx[k] * dx[k]; - } + /* Get a pointer to the jth particle. */ + struct part *restrict pj = &parts_j[sort_j[pjd].i]; + + /* Compute the pairwise distance. */ + float r2 = 0.0f; + float dx[3]; + for (int k = 0; k < 3; k++) { + dx[k] = pix[k] - pj->x[k]; + r2 += dx[k] * dx[k]; + } #ifdef SWIFT_DEBUG_CHECKS - /* Check that particles have been drifted to the current time */ - if (pi->ti_drift != e->ti_current) - error("Particle pi not drifted to current time"); - if (pj->ti_drift != e->ti_current) - error("Particle pj not drifted to current time"); + /* Check that particles have been drifted to the current time */ + if (pi->ti_drift != e->ti_current) + error("Particle pi not drifted to current time"); + if (pj->ti_drift != e->ti_current) + error("Particle pj not drifted to current time"); #endif - /* Hit or miss? */ - if (r2 < hig2) { + /* Hit or miss? */ + if (r2 < hig2) { #ifndef WITH_OLD_VECTORIZATION - IACT_NONSYM(r2, dx, hi, pj->h, pi, pj); + IACT_NONSYM(r2, dx, hi, pj->h, pi, pj); #else - /* Add this interaction to the queue. */ - r2q[icount] = r2; - dxq[3 * icount + 0] = dx[0]; - dxq[3 * icount + 1] = dx[1]; - dxq[3 * icount + 2] = dx[2]; - hiq[icount] = hi; - hjq[icount] = pj->h; - piq[icount] = pi; - pjq[icount] = pj; - icount += 1; + /* Add this interaction to the queue. */ + r2q[icount] = r2; + dxq[3 * icount + 0] = dx[0]; + dxq[3 * icount + 1] = dx[1]; + dxq[3 * icount + 2] = dx[2]; + hiq[icount] = hi; + hjq[icount] = pj->h; + piq[icount] = pi; + pjq[icount] = pj; + icount += 1; - /* Flush? */ - if (icount == VEC_SIZE) { - IACT_NONSYM_VEC(r2q, dxq, hiq, hjq, piq, pjq); - icount = 0; - } + /* Flush? */ + if (icount == VEC_SIZE) { + IACT_NONSYM_VEC(r2q, dxq, hiq, hjq, piq, pjq); + icount = 0; + } #endif - } + } - } /* loop over the parts in cj. */ + } /* loop over the parts in cj. */ - } /* loop over the parts in ci. */ + } /* loop over the parts in ci. */ - /* Loop over the parts in cj. */ - for (int pjd = 0; pjd < count_j && sort_j[pjd].d - hj_max - dx_max < di_max; - pjd++) { + } /* Cell ci is active */ - /* Get a hold of the jth part in cj. */ - struct part *restrict pj = &parts_j[sort_j[pjd].i]; - if (!part_is_active(pj, e)) continue; - const float hj = pj->h; - const double dj = sort_j[pjd].d - hj * kernel_gamma - dx_max - rshift; - if (dj > di_max) continue; + if (cell_is_active(cj, e)) { - double pjx[3]; - for (int k = 0; k < 3; k++) pjx[k] = pj->x[k] + shift[k]; - const float hjg2 = hj * hj * kernel_gamma2; + /* Loop over the parts in cj. */ + for (int pjd = 0; pjd < count_j && sort_j[pjd].d - hj_max - dx_max < di_max; + pjd++) { - /* Loop over the parts in ci. */ - for (int pid = count_i - 1; pid >= 0 && sort_i[pid].d > dj; pid--) { + /* Get a hold of the jth part in cj. */ + struct part *restrict pj = &parts_j[sort_j[pjd].i]; + if (!part_is_active(pj, e)) continue; + const float hj = pj->h; + const double dj = sort_j[pjd].d - hj * kernel_gamma - dx_max - rshift; + if (dj > di_max) continue; - /* Get a pointer to the jth particle. */ - struct part *restrict pi = &parts_i[sort_i[pid].i]; + double pjx[3]; + for (int k = 0; k < 3; k++) pjx[k] = pj->x[k] + shift[k]; + const float hjg2 = hj * hj * kernel_gamma2; - /* Compute the pairwise distance. */ - float r2 = 0.0f; - float dx[3]; - for (int k = 0; k < 3; k++) { - dx[k] = pjx[k] - pi->x[k]; - r2 += dx[k] * dx[k]; - } + /* Loop over the parts in ci. */ + for (int pid = count_i - 1; pid >= 0 && sort_i[pid].d > dj; pid--) { + + /* Get a pointer to the jth particle. */ + struct part *restrict pi = &parts_i[sort_i[pid].i]; + + /* Compute the pairwise distance. */ + float r2 = 0.0f; + float dx[3]; + for (int k = 0; k < 3; k++) { + dx[k] = pjx[k] - pi->x[k]; + r2 += dx[k] * dx[k]; + } #ifdef SWIFT_DEBUG_CHECKS - /* Check that particles have been drifted to the current time */ - if (pi->ti_drift != e->ti_current) - error("Particle pi not drifted to current time"); - if (pj->ti_drift != e->ti_current) - error("Particle pj not drifted to current time"); + /* Check that particles have been drifted to the current time */ + if (pi->ti_drift != e->ti_current) + error("Particle pi not drifted to current time"); + if (pj->ti_drift != e->ti_current) + error("Particle pj not drifted to current time"); #endif - /* Hit or miss? */ - if (r2 < hjg2) { + /* Hit or miss? */ + if (r2 < hjg2) { #ifndef WITH_OLD_VECTORIZATION - IACT_NONSYM(r2, dx, hj, pi->h, pj, pi); + IACT_NONSYM(r2, dx, hj, pi->h, pj, pi); #else - /* Add this interaction to the queue. */ - r2q[icount] = r2; - dxq[3 * icount + 0] = dx[0]; - dxq[3 * icount + 1] = dx[1]; - dxq[3 * icount + 2] = dx[2]; - hiq[icount] = hj; - hjq[icount] = pi->h; - piq[icount] = pj; - pjq[icount] = pi; - icount += 1; + /* Add this interaction to the queue. */ + r2q[icount] = r2; + dxq[3 * icount + 0] = dx[0]; + dxq[3 * icount + 1] = dx[1]; + dxq[3 * icount + 2] = dx[2]; + hiq[icount] = hj; + hjq[icount] = pi->h; + piq[icount] = pj; + pjq[icount] = pi; + icount += 1; - /* Flush? */ - if (icount == VEC_SIZE) { - IACT_NONSYM_VEC(r2q, dxq, hiq, hjq, piq, pjq); - icount = 0; - } + /* Flush? */ + if (icount == VEC_SIZE) { + IACT_NONSYM_VEC(r2q, dxq, hiq, hjq, piq, pjq); + icount = 0; + } #endif - } + } + + } /* loop over the parts in ci. */ } /* loop over the parts in cj. */ - } /* loop over the parts in ci. */ + } /* Cell cj is active */ #ifdef WITH_OLD_VECTORIZATION /* Pick up any leftovers. */ @@ -1143,8 +1147,6 @@ void DOPAIR2(struct runner *r, struct cell *ci, struct cell *cj) { runner_do_sort(r, ci, (1 << sid), 1); if (!(cj->sorted & (1 << sid)) || cj->dx_max_sort > space_maxreldx * cj->dmin) runner_do_sort(r, cj, (1 << sid), 1); - // if (!(ci->sorted & (1 << sid)) || !(cj->sorted & (1 << sid))) - // error("Trying to interact unsorted cells."); /* Get the cutoff shift. */ double rshift = 0.0; diff --git a/src/runner_doiact_grav.h b/src/runner_doiact_grav.h index 9d2606ceb06fd6d32592010376e867a6ae582bf0..9988b7d553a962ee541f20cab64cc534c991957a 100644 --- a/src/runner_doiact_grav.h +++ b/src/runner_doiact_grav.h @@ -34,25 +34,78 @@ */ void runner_do_grav_up(struct runner *r, struct cell *c) { - if (c->split) { /* Regular node */ + /* if (c->split) { /\* Regular node *\/ */ + + /* /\* Recurse. *\/ */ + /* for (int k = 0; k < 8; k++) */ + /* if (c->progeny[k] != NULL) runner_do_grav_up(r, c->progeny[k]); */ + + /* /\* Collect the multipoles from the progeny. *\/ */ + /* multipole_reset(&c->multipole); */ + /* for (int k = 0; k < 8; k++) { */ + /* if (c->progeny[k] != NULL) */ + /* multipole_add(&c->multipole, &c->progeny[k]->multipole); */ + /* } */ + + /* } else { /\* Leaf node. *\/ */ + /* /\* Just construct the multipole from the gparts. *\/ */ + /* multipole_init(&c->multipole, c->gparts, c->gcount); */ + /* } */ +} + +void runner_do_grav_down(struct runner *r, struct cell *c) { + + if (c->split) { - /* Recurse. */ - for (int k = 0; k < 8; k++) - if (c->progeny[k] != NULL) runner_do_grav_up(r, c->progeny[k]); + for (int k = 0; k < 8; ++k) { + struct cell *cp = c->progeny[k]; + struct gravity_tensors temp; - /* Collect the multipoles from the progeny. */ - multipole_reset(&c->multipole); - for (int k = 0; k < 8; k++) { - if (c->progeny[k] != NULL) - multipole_add(&c->multipole, &c->progeny[k]->multipole); + if (cp != NULL) { + gravity_L2L(&temp, c->multipole, cp->multipole->CoM, c->multipole->CoM, + 1); + } } - } else { /* Leaf node. */ - /* Just construct the multipole from the gparts. */ - multipole_init(&c->multipole, c->gparts, c->gcount); + } else { + + gravity_L2P(c->multipole, c->gparts, c->gcount); } } +/** + * @brief Computes the interaction of the field tensor in a cell with the + * multipole of another cell. + * + * @param r The #runner. + * @param ci The #cell with field tensor to interact. + * @param cj The #cell with the multipole. + */ +__attribute__((always_inline)) INLINE static void runner_dopair_grav_mm( + const struct runner *r, const struct cell *restrict ci, + const struct cell *restrict cj) { + + const struct engine *e = r->e; + const int periodic = e->s->periodic; + const struct multipole *multi_j = &cj->multipole->m_pole; + // const float a_smooth = e->gravity_properties->a_smooth; + // const float rlr_inv = 1. / (a_smooth * ci->super->width[0]); + + TIMER_TIC; + +#ifdef SWIFT_DEBUG_CHECKS + if (multi_j->mass == 0.0) error("Multipole does not seem to have been set."); +#endif + + /* Anything to do here? */ + if (!cell_is_active(ci, e)) return; + + gravity_M2L(ci->multipole, multi_j, ci->multipole->CoM, cj->multipole->CoM, + periodic); + + TIMER_TOC(timer_dopair_grav_mm); +} + /** * @brief Computes the interaction of all the particles in a cell with the * multipole of another cell. @@ -68,15 +121,16 @@ __attribute__((always_inline)) INLINE static void runner_dopair_grav_pm( const struct engine *e = r->e; const int gcount = ci->gcount; struct gpart *restrict gparts = ci->gparts; - const struct multipole multi = cj->multipole; - const float rlr_inv = 1. / (const_gravity_a_smooth * ci->super->width[0]); + const struct gravity_tensors *multi = cj->multipole; + const float a_smooth = e->gravity_properties->a_smooth; + const float rlr_inv = 1. / (a_smooth * ci->super->width[0]); TIMER_TIC; #ifdef SWIFT_DEBUG_CHECKS - if (gcount == 0) error("Empty cell!"); // MATTHIEU sanity check + if (gcount == 0) error("Empty cell!"); - if (multi.mass == 0.0) // MATTHIEU sanity check + if (multi->m_pole.mass == 0.0) error("Multipole does not seem to have been set."); #endif @@ -105,13 +159,17 @@ __attribute__((always_inline)) INLINE static void runner_dopair_grav_pm( if (!gpart_is_active(gp, e)) continue; /* Compute the pairwise distance. */ - const float dx[3] = {multi.CoM[0] - gp->x[0], // x - multi.CoM[1] - gp->x[1], // y - multi.CoM[2] - gp->x[2]}; // z + const float dx[3] = {multi->CoM[0] - gp->x[0], // x + multi->CoM[1] - gp->x[1], // y + multi->CoM[2] - gp->x[2]}; // z const float r2 = dx[0] * dx[0] + dx[1] * dx[1] + dx[2] * dx[2]; /* Interact !*/ - runner_iact_grav_pm(rlr_inv, r2, dx, gp, &multi); + runner_iact_grav_pm(rlr_inv, r2, dx, gp, &multi->m_pole); + +#ifdef SWIFT_DEBUG_CHECKS + gp->mass_interacted += multi->m_pole.mass; +#endif } TIMER_TOC(timer_dopair_grav_pm); @@ -135,7 +193,8 @@ __attribute__((always_inline)) INLINE static void runner_dopair_grav_pp( const int gcount_j = cj->gcount; struct gpart *restrict gparts_i = ci->gparts; struct gpart *restrict gparts_j = cj->gparts; - const float rlr_inv = 1. / (const_gravity_a_smooth * ci->super->width[0]); + const float a_smooth = e->gravity_properties->a_smooth; + const float rlr_inv = 1. / (a_smooth * ci->super->width[0]); TIMER_TIC; @@ -195,18 +254,30 @@ __attribute__((always_inline)) INLINE static void runner_dopair_grav_pp( runner_iact_grav_pp(rlr_inv, r2, dx, gpi, gpj); +#ifdef SWIFT_DEBUG_CHECKS + gpi->mass_interacted += gpj->mass; + gpj->mass_interacted += gpi->mass; +#endif + } else { if (gpart_is_active(gpi, e)) { runner_iact_grav_pp_nonsym(rlr_inv, r2, dx, gpi, gpj); +#ifdef SWIFT_DEBUG_CHECKS + gpi->mass_interacted += gpj->mass; +#endif } else if (gpart_is_active(gpj, e)) { dx[0] = -dx[0]; dx[1] = -dx[1]; dx[2] = -dx[2]; runner_iact_grav_pp_nonsym(rlr_inv, r2, dx, gpj, gpi); + +#ifdef SWIFT_DEBUG_CHECKS + gpj->mass_interacted += gpi->mass; +#endif } } } @@ -229,7 +300,8 @@ __attribute__((always_inline)) INLINE static void runner_doself_grav_pp( const struct engine *e = r->e; const int gcount = c->gcount; struct gpart *restrict gparts = c->gparts; - const float rlr_inv = 1. / (const_gravity_a_smooth * c->super->width[0]); + const float a_smooth = e->gravity_properties->a_smooth; + const float rlr_inv = 1. / (a_smooth * c->super->width[0]); TIMER_TIC; @@ -277,18 +349,31 @@ __attribute__((always_inline)) INLINE static void runner_doself_grav_pp( runner_iact_grav_pp(rlr_inv, r2, dx, gpi, gpj); +#ifdef SWIFT_DEBUG_CHECKS + gpi->mass_interacted += gpj->mass; + gpj->mass_interacted += gpi->mass; +#endif + } else { if (gpart_is_active(gpi, e)) { runner_iact_grav_pp_nonsym(rlr_inv, r2, dx, gpi, gpj); +#ifdef SWIFT_DEBUG_CHECKS + gpi->mass_interacted += gpj->mass; +#endif + } else if (gpart_is_active(gpj, e)) { dx[0] = -dx[0]; dx[1] = -dx[1]; dx[2] = -dx[2]; runner_iact_grav_pp_nonsym(rlr_inv, r2, dx, gpj, gpi); + +#ifdef SWIFT_DEBUG_CHECKS + gpj->mass_interacted += gpi->mass; +#endif } } } @@ -466,7 +551,8 @@ static void runner_dosub_grav(struct runner *r, struct cell *ci, } } -static void runner_do_grav_mm(struct runner *r, struct cell *ci, int timer) { +static void runner_do_grav_long_range(struct runner *r, struct cell *ci, + int timer) { #if ICHECK > 0 for (int pid = 0; pid < ci->gcount; pid++) { @@ -485,10 +571,10 @@ static void runner_do_grav_mm(struct runner *r, struct cell *ci, int timer) { const struct engine *e = r->e; struct cell *cells = e->s->cells_top; const int nr_cells = e->s->nr_cells; - const double max_d = - const_gravity_a_smooth * const_gravity_r_cut * ci->width[0]; - const double max_d2 = max_d * max_d; - const double pos_i[3] = {ci->loc[0], ci->loc[1], ci->loc[2]}; + /* const double max_d = */ + /* const_gravity_a_smooth * const_gravity_r_cut * ci->width[0]; */ + /* const double max_d2 = max_d * max_d; */ + // const double pos_i[3] = {ci->loc[0], ci->loc[1], ci->loc[2]}; /* Anything to do here? */ if (!cell_is_active(ci, e)) return; @@ -501,14 +587,14 @@ static void runner_do_grav_mm(struct runner *r, struct cell *ci, int timer) { if (ci == cj) continue; - const double dx[3] = {cj->loc[0] - pos_i[0], // x - cj->loc[1] - pos_i[1], // y - cj->loc[2] - pos_i[2]}; // z - const double r2 = dx[0] * dx[0] + dx[1] * dx[1] + dx[2] * dx[2]; + /* const double dx[3] = {cj->loc[0] - pos_i[0], // x */ + /* cj->loc[1] - pos_i[1], // y */ + /* cj->loc[2] - pos_i[2]}; // z */ + /* const double r2 = dx[0] * dx[0] + dx[1] * dx[1] + dx[2] * dx[2]; */ - if (r2 > max_d2) continue; + // if (r2 > max_d2) continue; - if (!cell_are_neighbours(ci, cj)) runner_dopair_grav_pm(r, ci, cj); + if (!cell_are_neighbours(ci, cj)) runner_dopair_grav_mm(r, ci, cj); } } diff --git a/src/runner_doiact_nosort.h b/src/runner_doiact_nosort.h index 057dd756b4a8d636253eccd8808417ac452e193d..6a442afbdb0c7dfb24c5e9cb386783746a1f05ed 100644 --- a/src/runner_doiact_nosort.h +++ b/src/runner_doiact_nosort.h @@ -15,103 +15,111 @@ void DOPAIR1_NOSORT(struct runner *r, struct cell *ci, struct cell *cj) { /* Anything to do here? */ if (!cell_is_active(ci, e) && !cell_is_active(cj, e)) return; - if (!cell_is_drifted(ci, e)) cell_drift(ci, e); - if (!cell_is_drifted(cj, e)) cell_drift(cj, e); + if (!cell_is_drifted(ci, e)) cell_drift_particles(ci, e); + if (!cell_is_drifted(cj, e)) cell_drift_particles(cj, e); + + /* Get the relative distance between the pairs, wrapping. */ + double shift[3] = {0.0, 0.0, 0.0}; + space_getsid(e->s, &ci, &cj, shift); const int count_i = ci->count; const int count_j = cj->count; struct part *restrict parts_i = ci->parts; struct part *restrict parts_j = cj->parts; - /* Get the relative distance between the pairs, wrapping. */ - double shift[3] = {0.0, 0.0, 0.0}; - space_getsid(e->s, &ci, &cj, shift); + if (cell_is_active(ci, e)) { - /* Loop over the parts in ci. */ - for (int pid = 0; pid < count_i; pid++) { + /* Loop over the parts in ci. */ + for (int pid = 0; pid < count_i; pid++) { - /* Get a hold of the ith part in ci. */ - struct part *restrict pi = &parts_i[pid]; - if (!part_is_active(pi, e)) continue; - const float hi = pi->h; + /* Get a hold of the ith part in ci. */ + struct part *restrict pi = &parts_i[pid]; + if (!part_is_active(pi, e)) continue; + const float hi = pi->h; - double pix[3]; - for (int k = 0; k < 3; k++) pix[k] = pi->x[k] - shift[k]; - const float hig2 = hi * hi * kernel_gamma2; + double pix[3]; + for (int k = 0; k < 3; k++) pix[k] = pi->x[k] - shift[k]; + const float hig2 = hi * hi * kernel_gamma2; - /* Loop over the parts in cj. */ - for (int pjd = 0; pjd < count_j; pjd++) { + /* Loop over the parts in cj. */ + for (int pjd = 0; pjd < count_j; pjd++) { - /* Get a pointer to the jth particle. */ - struct part *restrict pj = &parts_j[pjd]; + /* Get a pointer to the jth particle. */ + struct part *restrict pj = &parts_j[pjd]; - /* Compute the pairwise distance. */ - float r2 = 0.0f; - float dx[3]; - for (int k = 0; k < 3; k++) { - dx[k] = pix[k] - pj->x[k]; - r2 += dx[k] * dx[k]; - } + /* Compute the pairwise distance. */ + float r2 = 0.0f; + float dx[3]; + for (int k = 0; k < 3; k++) { + dx[k] = pix[k] - pj->x[k]; + r2 += dx[k] * dx[k]; + } #ifdef SWIFT_DEBUG_CHECKS - /* Check that particles have been drifted to the current time */ - if (pi->ti_drift != e->ti_current) - error("Particle pi not drifted to current time"); - if (pj->ti_drift != e->ti_current) - error("Particle pj not drifted to current time"); + /* Check that particles have been drifted to the current time */ + if (pi->ti_drift != e->ti_current) + error("Particle pi not drifted to current time"); + if (pj->ti_drift != e->ti_current) + error("Particle pj not drifted to current time"); #endif - /* Hit or miss? */ - if (r2 < hig2) { - IACT_NONSYM(r2, dx, hi, pj->h, pi, pj); - } + /* Hit or miss? */ + if (r2 < hig2) { + IACT_NONSYM(r2, dx, hi, pj->h, pi, pj); + } - } /* loop over the parts in cj. */ + } /* loop over the parts in cj. */ - } /* loop over the parts in ci. */ + } /* loop over the parts in ci. */ - /* Loop over the parts in cj. */ - for (int pjd = 0; pjd < count_j; pjd++) { + } /* Cell ci is active */ - /* Get a hold of the ith part in ci. */ - struct part *restrict pj = &parts_j[pjd]; - if (!part_is_active(pj, e)) continue; - const float hj = pj->h; + if (cell_is_active(cj, e)) { - double pjx[3]; - for (int k = 0; k < 3; k++) pjx[k] = pj->x[k] + shift[k]; - const float hjg2 = hj * hj * kernel_gamma2; + /* Loop over the parts in cj. */ + for (int pjd = 0; pjd < count_j; pjd++) { - /* Loop over the parts in ci. */ - for (int pid = 0; pid < count_i; pid++) { + /* Get a hold of the ith part in ci. */ + struct part *restrict pj = &parts_j[pjd]; + if (!part_is_active(pj, e)) continue; + const float hj = pj->h; - /* Get a pointer to the jth particle. */ - struct part *restrict pi = &parts_i[pid]; + double pjx[3]; + for (int k = 0; k < 3; k++) pjx[k] = pj->x[k] + shift[k]; + const float hjg2 = hj * hj * kernel_gamma2; - /* Compute the pairwise distance. */ - float r2 = 0.0f; - float dx[3]; - for (int k = 0; k < 3; k++) { - dx[k] = pjx[k] - pi->x[k]; - r2 += dx[k] * dx[k]; - } + /* Loop over the parts in ci. */ + for (int pid = 0; pid < count_i; pid++) { + + /* Get a pointer to the jth particle. */ + struct part *restrict pi = &parts_i[pid]; + + /* Compute the pairwise distance. */ + float r2 = 0.0f; + float dx[3]; + for (int k = 0; k < 3; k++) { + dx[k] = pjx[k] - pi->x[k]; + r2 += dx[k] * dx[k]; + } #ifdef SWIFT_DEBUG_CHECKS - /* Check that particles have been drifted to the current time */ - if (pj->ti_drift != e->ti_current) - error("Particle pj not drifted to current time"); - if (pi->ti_drift != e->ti_current) - error("Particle pi not drifted to current time"); + /* Check that particles have been drifted to the current time */ + if (pj->ti_drift != e->ti_current) + error("Particle pj not drifted to current time"); + if (pi->ti_drift != e->ti_current) + error("Particle pi not drifted to current time"); #endif - /* Hit or miss? */ - if (r2 < hjg2) { - IACT_NONSYM(r2, dx, hj, pi->h, pj, pi); - } + /* Hit or miss? */ + if (r2 < hjg2) { + IACT_NONSYM(r2, dx, hj, pi->h, pj, pi); + } - } /* loop over the parts in ci. */ + } /* loop over the parts in ci. */ + + } /* loop over the parts in cj. */ - } /* loop over the parts in cj. */ + } /* Cell cj is active */ TIMER_TOC(TIMER_DOPAIR); } @@ -132,105 +140,113 @@ void DOPAIR2_NOSORT(struct runner *r, struct cell *ci, struct cell *cj) { /* Anything to do here? */ if (!cell_is_active(ci, e) && !cell_is_active(cj, e)) return; - if (!cell_is_drifted(ci, e)) cell_drift(ci, e); - if (!cell_is_drifted(cj, e)) cell_drift(cj, e); + if (!cell_is_drifted(ci, e)) cell_drift_particles(ci, e); + if (!cell_is_drifted(cj, e)) cell_drift_particles(cj, e); + + /* Get the relative distance between the pairs, wrapping. */ + double shift[3] = {0.0, 0.0, 0.0}; + space_getsid(e->s, &ci, &cj, shift); const int count_i = ci->count; const int count_j = cj->count; struct part *restrict parts_i = ci->parts; struct part *restrict parts_j = cj->parts; - /* Get the relative distance between the pairs, wrapping. */ - double shift[3] = {0.0, 0.0, 0.0}; - space_getsid(e->s, &ci, &cj, shift); + if (cell_is_active(ci, e)) { - /* Loop over the parts in ci. */ - for (int pid = 0; pid < count_i; pid++) { + /* Loop over the parts in ci. */ + for (int pid = 0; pid < count_i; pid++) { - /* Get a hold of the ith part in ci. */ - struct part *restrict pi = &parts_i[pid]; - if (!part_is_active(pi, e)) continue; - const float hi = pi->h; + /* Get a hold of the ith part in ci. */ + struct part *restrict pi = &parts_i[pid]; + if (!part_is_active(pi, e)) continue; + const float hi = pi->h; - double pix[3]; - for (int k = 0; k < 3; k++) pix[k] = pi->x[k] - shift[k]; - const float hig2 = hi * hi * kernel_gamma2; + double pix[3]; + for (int k = 0; k < 3; k++) pix[k] = pi->x[k] - shift[k]; + const float hig2 = hi * hi * kernel_gamma2; - /* Loop over the parts in cj. */ - for (int pjd = 0; pjd < count_j; pjd++) { + /* Loop over the parts in cj. */ + for (int pjd = 0; pjd < count_j; pjd++) { - /* Get a pointer to the jth particle. */ - struct part *restrict pj = &parts_j[pjd]; - const float hjg2 = pj->h * pj->h * kernel_gamma2; + /* Get a pointer to the jth particle. */ + struct part *restrict pj = &parts_j[pjd]; + const float hjg2 = pj->h * pj->h * kernel_gamma2; - /* Compute the pairwise distance. */ - float r2 = 0.0f; - float dx[3]; - for (int k = 0; k < 3; k++) { - dx[k] = pix[k] - pj->x[k]; - r2 += dx[k] * dx[k]; - } + /* Compute the pairwise distance. */ + float r2 = 0.0f; + float dx[3]; + for (int k = 0; k < 3; k++) { + dx[k] = pix[k] - pj->x[k]; + r2 += dx[k] * dx[k]; + } #ifdef SWIFT_DEBUG_CHECKS - /* Check that particles have been drifted to the current time */ - if (pi->ti_drift != e->ti_current) - error("Particle pi not drifted to current time"); - if (pj->ti_drift != e->ti_current) - error("Particle pj not drifted to current time"); + /* Check that particles have been drifted to the current time */ + if (pi->ti_drift != e->ti_current) + error("Particle pi not drifted to current time"); + if (pj->ti_drift != e->ti_current) + error("Particle pj not drifted to current time"); #endif - /* Hit or miss? */ - if (r2 < hig2 || r2 < hjg2) { - IACT_NONSYM(r2, dx, hi, pj->h, pi, pj); - } + /* Hit or miss? */ + if (r2 < hig2 || r2 < hjg2) { + IACT_NONSYM(r2, dx, hi, pj->h, pi, pj); + } - } /* loop over the parts in cj. */ + } /* loop over the parts in cj. */ - } /* loop over the parts in ci. */ + } /* loop over the parts in ci. */ - /* Loop over the parts in cj. */ - for (int pjd = 0; pjd < count_j; pjd++) { + } /* Cell ci is active */ - /* Get a hold of the ith part in ci. */ - struct part *restrict pj = &parts_j[pjd]; - if (!part_is_active(pj, e)) continue; - const float hj = pj->h; + if (cell_is_active(cj, e)) { - double pjx[3]; - for (int k = 0; k < 3; k++) pjx[k] = pj->x[k] + shift[k]; - const float hjg2 = hj * hj * kernel_gamma2; + /* Loop over the parts in cj. */ + for (int pjd = 0; pjd < count_j; pjd++) { - /* Loop over the parts in ci. */ - for (int pid = 0; pid < count_i; pid++) { + /* Get a hold of the ith part in ci. */ + struct part *restrict pj = &parts_j[pjd]; + if (!part_is_active(pj, e)) continue; + const float hj = pj->h; - /* Get a pointer to the jth particle. */ - struct part *restrict pi = &parts_i[pid]; - const float hig2 = pi->h * pi->h * kernel_gamma2; + double pjx[3]; + for (int k = 0; k < 3; k++) pjx[k] = pj->x[k] + shift[k]; + const float hjg2 = hj * hj * kernel_gamma2; - /* Compute the pairwise distance. */ - float r2 = 0.0f; - float dx[3]; - for (int k = 0; k < 3; k++) { - dx[k] = pjx[k] - pi->x[k]; - r2 += dx[k] * dx[k]; - } + /* Loop over the parts in ci. */ + for (int pid = 0; pid < count_i; pid++) { + + /* Get a pointer to the jth particle. */ + struct part *restrict pi = &parts_i[pid]; + const float hig2 = pi->h * pi->h * kernel_gamma2; + + /* Compute the pairwise distance. */ + float r2 = 0.0f; + float dx[3]; + for (int k = 0; k < 3; k++) { + dx[k] = pjx[k] - pi->x[k]; + r2 += dx[k] * dx[k]; + } #ifdef SWIFT_DEBUG_CHECKS - /* Check that particles have been drifted to the current time */ - if (pj->ti_drift != e->ti_current) - error("Particle pj not drifted to current time"); - if (pi->ti_drift != e->ti_current) - error("Particle pi not drifted to current time"); + /* Check that particles have been drifted to the current time */ + if (pj->ti_drift != e->ti_current) + error("Particle pj not drifted to current time"); + if (pi->ti_drift != e->ti_current) + error("Particle pi not drifted to current time"); #endif - /* Hit or miss? */ - if (r2 < hjg2 || r2 < hig2) { - IACT_NONSYM(r2, dx, hj, pi->h, pj, pi); - } + /* Hit or miss? */ + if (r2 < hjg2 || r2 < hig2) { + IACT_NONSYM(r2, dx, hj, pi->h, pj, pi); + } - } /* loop over the parts in ci. */ + } /* loop over the parts in ci. */ + + } /* loop over the parts in cj. */ - } /* loop over the parts in cj. */ + } /* Cell cj is active */ TIMER_TOC(TIMER_DOPAIR); } diff --git a/src/runner_doiact_vec.c b/src/runner_doiact_vec.c index b91d288529c0706693d74b0c54d688ee0944aa29..d78de2cd3c508fd17a816316dc931010f2febf80 100644 --- a/src/runner_doiact_vec.c +++ b/src/runner_doiact_vec.c @@ -287,7 +287,7 @@ __attribute__((always_inline)) INLINE void runner_doself1_density_vec( if (!cell_is_active(c, e)) return; - if (!cell_is_drifted(c, e)) cell_drift(c, e); + if (!cell_is_drifted(c, e)) cell_drift_particles(c, e); /* Get the particle cache from the runner and re-allocate * the cache if it is not big enough for the cell. */ @@ -536,7 +536,7 @@ __attribute__((always_inline)) INLINE void runner_doself1_density_vec_2( if (!cell_is_active(c, e)) return; - if (!cell_is_drifted(c, e)) cell_drift(c, e); + if (!cell_is_drifted(c, e)) cell_drift_particles(c, e); /* TODO: Need to find two active particles, not just one. */ diff --git a/src/scheduler.c b/src/scheduler.c index 604f4b0a5660e0140726e2f2c4a3baa96feea0c7..0c2e07f5d8fe69893a97862d7f94b145fdcf6a70 100644 --- a/src/scheduler.c +++ b/src/scheduler.c @@ -1104,8 +1104,7 @@ void scheduler_start(struct scheduler *s) { if (ci == NULL && cj == NULL) { - if (t->type != task_type_grav_gather_m && t->type != task_type_grav_fft) - error("Task not associated with cells!"); + error("Task not associated with cells!"); } else if (cj == NULL) { /* self */ @@ -1230,6 +1229,12 @@ void scheduler_enqueue(struct scheduler *s, struct task *t) { } else if (t->subtype == task_subtype_gpart) { err = MPI_Irecv(t->ci->gparts, t->ci->gcount, gpart_mpi_type, t->ci->nodeID, t->flags, MPI_COMM_WORLD, &t->req); + } else if (t->subtype == task_subtype_spart) { + err = MPI_Irecv(t->ci->sparts, t->ci->scount, spart_mpi_type, + t->ci->nodeID, t->flags, MPI_COMM_WORLD, &t->req); + } else if (t->subtype == task_subtype_multipole) { + err = MPI_Irecv(t->ci->multipole, 1, multipole_mpi_type, + t->ci->nodeID, t->flags, MPI_COMM_WORLD, &t->req); } else { error("Unknown communication sub-type"); } @@ -1258,13 +1263,18 @@ void scheduler_enqueue(struct scheduler *s, struct task *t) { #endif err = MPI_Isend(t->ci->parts, t->ci->count, part_mpi_type, t->cj->nodeID, t->flags, MPI_COMM_WORLD, &t->req); - // message( "sending %i parts with tag=%i from %i to %i." , // t->ci->count , t->flags , s->nodeID , t->cj->nodeID ); // fflush(stdout); } else if (t->subtype == task_subtype_gpart) { err = MPI_Isend(t->ci->gparts, t->ci->gcount, gpart_mpi_type, t->cj->nodeID, t->flags, MPI_COMM_WORLD, &t->req); + } else if (t->subtype == task_subtype_spart) { + err = MPI_Isend(t->ci->sparts, t->ci->scount, spart_mpi_type, + t->cj->nodeID, t->flags, MPI_COMM_WORLD, &t->req); + } else if (t->subtype == task_subtype_multipole) { + err = MPI_Isend(t->ci->multipole, 1, multipole_mpi_type, + t->cj->nodeID, t->flags, MPI_COMM_WORLD, &t->req); } else { error("Unknown communication sub-type"); } diff --git a/src/serial_io.c b/src/serial_io.c index 49dd945006d5907c43d58babc28f72914484a742..52c52ff24c186a04da3e3945aea6684abcd95476 100644 --- a/src/serial_io.c +++ b/src/serial_io.c @@ -41,12 +41,15 @@ #include "engine.h" #include "error.h" #include "gravity_io.h" +#include "gravity_properties.h" #include "hydro_io.h" #include "hydro_properties.h" #include "io_properties.h" #include "kernel_hydro.h" #include "part.h" +#include "stars_io.h" #include "units.h" +#include "xmf.h" /*----------------------------------------------------------------------------- * Routines reading an IC file @@ -72,10 +75,10 @@ */ void readArray(hid_t grp, const struct io_props props, size_t N, long long N_total, long long offset, - const struct UnitSystem* internal_units, - const struct UnitSystem* ic_units) { + const struct unit_system* internal_units, + const struct unit_system* ic_units) { - const size_t typeSize = sizeOfType(props.type); + const size_t typeSize = io_sizeof_type(props.type); const size_t copySize = typeSize * props.dimension; const size_t num_elements = N * props.dimension; @@ -104,7 +107,7 @@ void readArray(hid_t grp, const struct io_props props, size_t N, /* Check data type */ const hid_t h_type = H5Dget_type(h_data); if (h_type < 0) error("Unable to retrieve data type from the file"); - /* if (!H5Tequal(h_type, hdf5Type(type))) */ + /* if (!H5Tequal(h_type, hdf5_type(type))) */ /* error("Non-matching types between the code and the file"); */ /* Allocate temporary buffer */ @@ -138,7 +141,7 @@ void readArray(hid_t grp, const struct io_props props, size_t N, /* Read HDF5 dataspace in temporary buffer */ /* Dirty version that happens to work for vectors but should be improved */ /* Using HDF5 dataspaces would be better */ - const hid_t h_err = H5Dread(h_data, hdf5Type(props.type), h_memspace, + const hid_t h_err = H5Dread(h_data, io_hdf5_type(props.type), h_memspace, h_filespace, H5P_DEFAULT, temp); if (h_err < 0) { error("Error while reading data array '%s'.", props.name); @@ -151,7 +154,7 @@ void readArray(hid_t grp, const struct io_props props, size_t N, /* message("Converting ! factor=%e", factor); */ - if (isDoublePrecision(props.type)) { + if (io_is_double_precision(props.type)) { double* temp_d = temp; for (size_t i = 0; i < num_elements; ++i) temp_d[i] *= factor; } else { @@ -180,8 +183,8 @@ void readArray(hid_t grp, const struct io_props props, size_t N, void prepareArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile, char* partTypeGroupName, const struct io_props props, unsigned long long N_total, - const struct UnitSystem* internal_units, - const struct UnitSystem* snapshot_units) { + const struct unit_system* internal_units, + const struct unit_system* snapshot_units) { /* Create data space */ const hid_t h_space = H5Screate(H5S_SIMPLE); @@ -235,26 +238,27 @@ void prepareArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile, } /* Create dataset */ - const hid_t h_data = H5Dcreate(grp, props.name, hdf5Type(props.type), h_space, - H5P_DEFAULT, h_prop, H5P_DEFAULT); + const hid_t h_data = H5Dcreate(grp, props.name, io_hdf5_type(props.type), + h_space, H5P_DEFAULT, h_prop, H5P_DEFAULT); if (h_data < 0) { error("Error while creating dataspace '%s'.", props.name); } /* Write XMF description for this data set */ - writeXMFline(xmfFile, fileName, partTypeGroupName, props.name, N_total, - props.dimension, props.type); + xmf_write_line(xmfFile, fileName, partTypeGroupName, props.name, N_total, + props.dimension, props.type); /* Write unit conversion factors for this data set */ char buffer[FIELD_BUFFER_SIZE]; units_cgs_conversion_string(buffer, snapshot_units, props.units); - writeAttribute_d(h_data, "CGS conversion factor", - units_cgs_conversion_factor(snapshot_units, props.units)); - writeAttribute_f(h_data, "h-scale exponent", - units_h_factor(snapshot_units, props.units)); - writeAttribute_f(h_data, "a-scale exponent", - units_a_factor(snapshot_units, props.units)); - writeAttribute_s(h_data, "Conversion factor", buffer); + io_write_attribute_d( + h_data, "CGS conversion factor", + units_cgs_conversion_factor(snapshot_units, props.units)); + io_write_attribute_f(h_data, "h-scale exponent", + units_h_factor(snapshot_units, props.units)); + io_write_attribute_f(h_data, "a-scale exponent", + units_a_factor(snapshot_units, props.units)); + io_write_attribute_s(h_data, "Conversion factor", buffer); /* Close everything */ H5Pclose(h_prop); @@ -278,16 +282,16 @@ void prepareArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile, * @param part_c A (char*) pointer on the first occurrence of the field of *interest in the parts array * @param partSize The size in bytes of the particle structure. - * @param us The UnitSystem currently in use + * @param us The unit_system currently in use * @param convFactor The UnitConversionFactor for this arrayo */ void writeArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile, char* partTypeGroupName, const struct io_props props, size_t N, long long N_total, int mpi_rank, long long offset, - const struct UnitSystem* internal_units, - const struct UnitSystem* snapshot_units) { + const struct unit_system* internal_units, + const struct unit_system* snapshot_units) { - const size_t typeSize = sizeOfType(props.type); + const size_t typeSize = io_sizeof_type(props.type); const size_t copySize = typeSize * props.dimension; const size_t num_elements = N * props.dimension; @@ -299,7 +303,7 @@ void writeArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile, internal_units, snapshot_units); /* Allocate temporary buffer */ - void* temp = malloc(num_elements * sizeOfType(props.type)); + void* temp = malloc(num_elements * io_sizeof_type(props.type)); if (temp == NULL) error("Unable to allocate memory for temporary buffer"); /* Copy particle data to temporary buffer */ @@ -330,7 +334,7 @@ void writeArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile, /* message("Converting ! factor=%e", factor); */ - if (isDoublePrecision(props.type)) { + if (io_is_double_precision(props.type)) { double* temp_d = temp; for (size_t i = 0; i < num_elements; ++i) temp_d[i] *= factor; } else { @@ -378,7 +382,7 @@ void writeArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile, H5Sselect_hyperslab(h_filespace, H5S_SELECT_SET, offsets, NULL, shape, NULL); /* Write temporary buffer to HDF5 dataspace */ - h_err = H5Dwrite(h_data, hdf5Type(props.type), h_memspace, h_filespace, + h_err = H5Dwrite(h_data, io_hdf5_type(props.type), h_memspace, h_filespace, H5P_DEFAULT, temp); if (h_err < 0) error("Error while writing data array '%s'.", props.name); @@ -397,11 +401,16 @@ void writeArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile, * @param dim (output) The dimension of the volume read from the file. * @param parts (output) The array of #part (gas particles) read from the file. * @param gparts (output) The array of #gpart read from the file. + * @param sparts (output) Array of #spart particles. * @param Ngas (output) The number of #part read from the file on that node. * @param Ngparts (output) The number of #gpart read from the file on that node. + * @param Nstars (output) The number of #spart read from the file on that node. * @param periodic (output) 1 if the volume is periodic, 0 if not. * @param flag_entropy (output) 1 if the ICs contained Entropy in the * InternalEnergy field + * @param with_hydro Are we reading gas particles ? + * @param with_gravity Are we reading/creating #gpart arrays ? + * @param with_stars Are we reading star particles ? * @param mpi_rank The MPI rank of this node * @param mpi_size The number of MPI ranks * @param comm The MPI communicator @@ -416,22 +425,26 @@ void writeArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile, * @todo Read snapshots distributed in more than one file. * */ -void read_ic_serial(char* fileName, const struct UnitSystem* internal_units, +void read_ic_serial(char* fileName, const struct unit_system* internal_units, double dim[3], struct part** parts, struct gpart** gparts, - size_t* Ngas, size_t* Ngparts, int* periodic, - int* flag_entropy, int mpi_rank, int mpi_size, - MPI_Comm comm, MPI_Info info, int dry_run) { + struct spart** sparts, size_t* Ngas, size_t* Ngparts, + size_t* Nstars, int* periodic, int* flag_entropy, + int with_hydro, int with_gravity, int with_stars, + int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info, + int dry_run) { + hid_t h_file = 0, h_grp = 0; /* GADGET has only cubic boxes (in cosmological mode) */ double boxSize[3] = {0.0, -1.0, -1.0}; /* GADGET has 6 particle types. We only keep the type 0 & 1 for now*/ - int numParticles[NUM_PARTICLE_TYPES] = {0}; - int numParticles_highWord[NUM_PARTICLE_TYPES] = {0}; - size_t N[NUM_PARTICLE_TYPES] = {0}; - long long N_total[NUM_PARTICLE_TYPES] = {0}; - long long offset[NUM_PARTICLE_TYPES] = {0}; + long long numParticles[swift_type_count] = {0}; + long long numParticles_highWord[swift_type_count] = {0}; + size_t N[swift_type_count] = {0}; + long long N_total[swift_type_count] = {0}; + long long offset[swift_type_count] = {0}; int dimension = 3; /* Assume 3D if nothing is specified */ - struct UnitSystem* ic_units = malloc(sizeof(struct UnitSystem)); + size_t Ndm = 0; + struct unit_system* ic_units = malloc(sizeof(struct unit_system)); /* First read some information about the content */ if (mpi_rank == 0) { @@ -448,7 +461,7 @@ void read_ic_serial(char* fileName, const struct UnitSystem* internal_units, if (h_grp < 0) error("Error while opening runtime parameters\n"); /* Read the relevant information */ - readAttribute(h_grp, "PeriodicBoundariesOn", INT, periodic); + io_read_attribute(h_grp, "PeriodicBoundariesOn", INT, periodic); /* Close runtime parameters */ H5Gclose(h_grp); @@ -462,22 +475,23 @@ void read_ic_serial(char* fileName, const struct UnitSystem* internal_units, const hid_t hid_dim = H5Aexists(h_grp, "Dimension"); if (hid_dim < 0) error("Error while testing existance of 'Dimension' attribute"); - if (hid_dim > 0) readAttribute(h_grp, "Dimension", INT, &dimension); + if (hid_dim > 0) io_read_attribute(h_grp, "Dimension", INT, &dimension); if (dimension != hydro_dimension) error("ICs dimensionality (%dD) does not match code dimensionality (%dD)", dimension, (int)hydro_dimension); /* Read the relevant information and print status */ int flag_entropy_temp[6]; - readAttribute(h_grp, "Flag_Entropy_ICs", INT, flag_entropy_temp); + io_read_attribute(h_grp, "Flag_Entropy_ICs", INT, flag_entropy_temp); *flag_entropy = flag_entropy_temp[0]; - readAttribute(h_grp, "BoxSize", DOUBLE, boxSize); - readAttribute(h_grp, "NumPart_Total", UINT, numParticles); - readAttribute(h_grp, "NumPart_Total_HighWord", UINT, numParticles_highWord); + io_read_attribute(h_grp, "BoxSize", DOUBLE, boxSize); + io_read_attribute(h_grp, "NumPart_Total", LONGLONG, numParticles); + io_read_attribute(h_grp, "NumPart_Total_HighWord", LONGLONG, + numParticles_highWord); - for (int ptype = 0; ptype < NUM_PARTICLE_TYPES; ++ptype) - N_total[ptype] = ((long long)numParticles[ptype]) + - ((long long)numParticles_highWord[ptype] << 32); + for (int ptype = 0; ptype < swift_type_count; ++ptype) + N_total[ptype] = + (numParticles[ptype]) + (numParticles_highWord[ptype] << 32); dim[0] = boxSize[0]; dim[1] = (boxSize[1] < 0) ? boxSize[0] : boxSize[1]; @@ -494,7 +508,7 @@ void read_ic_serial(char* fileName, const struct UnitSystem* internal_units, /* Read the unit system used in the ICs */ if (ic_units == NULL) error("Unable to allocate memory for IC unit system"); - readUnitSystem(h_file, ic_units); + io_read_unit_system(h_file, ic_units); if (units_are_equal(ic_units, internal_units)) { @@ -536,30 +550,45 @@ void read_ic_serial(char* fileName, const struct UnitSystem* internal_units, /* Now need to broadcast that information to all ranks. */ MPI_Bcast(flag_entropy, 1, MPI_INT, 0, comm); MPI_Bcast(periodic, 1, MPI_INT, 0, comm); - MPI_Bcast(&N_total, NUM_PARTICLE_TYPES, MPI_LONG_LONG_INT, 0, comm); + MPI_Bcast(&N_total, swift_type_count, MPI_LONG_LONG_INT, 0, comm); MPI_Bcast(dim, 3, MPI_DOUBLE, 0, comm); - MPI_Bcast(ic_units, sizeof(struct UnitSystem), MPI_BYTE, 0, comm); + MPI_Bcast(ic_units, sizeof(struct unit_system), MPI_BYTE, 0, comm); /* Divide the particles among the tasks. */ - for (int ptype = 0; ptype < NUM_PARTICLE_TYPES; ++ptype) { + for (int ptype = 0; ptype < swift_type_count; ++ptype) { offset[ptype] = mpi_rank * N_total[ptype] / mpi_size; N[ptype] = (mpi_rank + 1) * N_total[ptype] / mpi_size - offset[ptype]; } /* Allocate memory to store SPH particles */ - *Ngas = N[0]; - if (posix_memalign((void*)parts, part_align, (*Ngas) * sizeof(struct part)) != - 0) - error("Error while allocating memory for particles"); - bzero(*parts, *Ngas * sizeof(struct part)); - - /* Allocate memory to store all particles */ - const size_t Ndm = N[1]; - *Ngparts = N[1] + N[0]; - if (posix_memalign((void*)gparts, gpart_align, - *Ngparts * sizeof(struct gpart)) != 0) - error("Error while allocating memory for gravity particles"); - bzero(*gparts, *Ngparts * sizeof(struct gpart)); + if (with_hydro) { + *Ngas = N[0]; + if (posix_memalign((void*)parts, part_align, *Ngas * sizeof(struct part)) != + 0) + error("Error while allocating memory for SPH particles"); + bzero(*parts, *Ngas * sizeof(struct part)); + } + + /* Allocate memory to store star particles */ + if (with_stars) { + *Nstars = N[swift_type_star]; + if (posix_memalign((void*)sparts, spart_align, + *Nstars * sizeof(struct spart)) != 0) + error("Error while allocating memory for star particles"); + bzero(*sparts, *Nstars * sizeof(struct spart)); + } + + /* Allocate memory to store all gravity particles */ + if (with_gravity) { + Ndm = N[1]; + *Ngparts = (with_hydro ? N[swift_type_gas] : 0) + + N[swift_type_dark_matter] + + (with_stars ? N[swift_type_star] : 0); + if (posix_memalign((void*)gparts, gpart_align, + *Ngparts * sizeof(struct gpart)) != 0) + error("Error while allocating memory for gravity particles"); + bzero(*gparts, *Ngparts * sizeof(struct gpart)); + } /* message("Allocated %8.2f MB for particles.", *N * sizeof(struct part) / */ /* (1024.*1024.)); */ @@ -580,7 +609,7 @@ void read_ic_serial(char* fileName, const struct UnitSystem* internal_units, error("Error while opening file '%s' on rank %d.", fileName, mpi_rank); /* Loop over all particle types */ - for (int ptype = 0; ptype < NUM_PARTICLE_TYPES; ptype++) { + for (int ptype = 0; ptype < swift_type_count; ptype++) { /* Don't do anything if no particle of this kind */ if (N[ptype] == 0) continue; @@ -601,14 +630,25 @@ void read_ic_serial(char* fileName, const struct UnitSystem* internal_units, /* Read particle fields into the particle structure */ switch (ptype) { - case GAS: - Nparticles = *Ngas; - hydro_read_particles(*parts, list, &num_fields); + case swift_type_gas: + if (with_hydro) { + Nparticles = *Ngas; + hydro_read_particles(*parts, list, &num_fields); + } break; - case DM: - Nparticles = Ndm; - darkmatter_read_particles(*gparts, list, &num_fields); + case swift_type_dark_matter: + if (with_gravity) { + Nparticles = Ndm; + darkmatter_read_particles(*gparts, list, &num_fields); + } + break; + + case swift_type_star: + if (with_stars) { + Nparticles = *Nstars; + star_read_particles(*sparts, list, &num_fields); + } break; default: @@ -634,16 +674,21 @@ void read_ic_serial(char* fileName, const struct UnitSystem* internal_units, MPI_Barrier(comm); } - /* Clean up */ - free(ic_units); - /* Prepare the DM particles */ - if (!dry_run) prepare_dm_gparts(*gparts, Ndm); + if (!dry_run && with_gravity) io_prepare_dm_gparts(*gparts, Ndm); + + /* Duplicate the hydro particles into gparts */ + if (!dry_run && with_gravity && with_hydro) + io_duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm); - /* Now duplicate the hydro particle into gparts */ - if (!dry_run) duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm); + /* Duplicate the star particles into gparts */ + if (!dry_run && with_gravity && with_stars) + io_duplicate_star_gparts(*sparts, *gparts, *Nstars, Ndm + *Ngas); /* message("Done Reading particles..."); */ + + /* Clean up */ + free(ic_units); } /** @@ -651,8 +696,8 @@ void read_ic_serial(char* fileName, const struct UnitSystem* internal_units, * * @param e The engine containing all the system. * @param baseName The common part of the snapshot file name. - * @param internal_units The #UnitSystem used internally - * @param snapshot_units The #UnitSystem used in the snapshots + * @param internal_units The #unit_system used internally + * @param snapshot_units The #unit_system used in the snapshots * @param mpi_rank The MPI rank of this node. * @param mpi_size The number of MPI ranks. * @param comm The MPI communicator. @@ -667,23 +712,25 @@ void read_ic_serial(char* fileName, const struct UnitSystem* internal_units, * */ void write_output_serial(struct engine* e, const char* baseName, - const struct UnitSystem* internal_units, - const struct UnitSystem* snapshot_units, int mpi_rank, + const struct unit_system* internal_units, + const struct unit_system* snapshot_units, int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info) { hid_t h_file = 0, h_grp = 0; const size_t Ngas = e->s->nr_parts; + const size_t Nstars = e->s->nr_sparts; const size_t Ntot = e->s->nr_gparts; int periodic = e->s->periodic; int numFiles = 1; struct part* parts = e->s->parts; struct gpart* gparts = e->s->gparts; struct gpart* dmparts = NULL; + struct spart* sparts = e->s->sparts; static int outputCount = 0; FILE* xmfFile = 0; /* Number of unassociated gparts */ - const size_t Ndm = Ntot > 0 ? Ntot - Ngas : 0; + const size_t Ndm = Ntot > 0 ? Ntot - (Ngas + Nstars) : 0; /* File name */ char fileName[FILENAME_BUFFER_SIZE]; @@ -691,11 +738,11 @@ void write_output_serial(struct engine* e, const char* baseName, outputCount); /* Compute offset in the file and total number of particles */ - size_t N[NUM_PARTICLE_TYPES] = {Ngas, Ndm, 0}; - long long N_total[NUM_PARTICLE_TYPES] = {0}; - long long offset[NUM_PARTICLE_TYPES] = {0}; - MPI_Exscan(&N, &offset, NUM_PARTICLE_TYPES, MPI_LONG_LONG_INT, MPI_SUM, comm); - for (int ptype = 0; ptype < NUM_PARTICLE_TYPES; ++ptype) + size_t N[swift_type_count] = {Ngas, Ndm, 0, 0, Nstars, 0}; + long long N_total[swift_type_count] = {0}; + long long offset[swift_type_count] = {0}; + MPI_Exscan(&N, &offset, swift_type_count, MPI_LONG_LONG_INT, MPI_SUM, comm); + for (int ptype = 0; ptype < swift_type_count; ++ptype) N_total[ptype] = offset[ptype] + N[ptype]; /* The last rank now has the correct N_total. Let's broadcast from there */ @@ -708,13 +755,13 @@ void write_output_serial(struct engine* e, const char* baseName, if (mpi_rank == 0) { /* First time, we need to create the XMF file */ - if (outputCount == 0) createXMFfile(baseName); + if (outputCount == 0) xmf_create_file(baseName); /* Prepare the XMF file for the new entry */ - xmfFile = prepareXMFfile(baseName); + xmfFile = xmf_prepare_file(baseName); /* Write the part corresponding to this specific output */ - writeXMFoutputheader(xmfFile, fileName, e->time); + xmf_write_outputheader(xmfFile, fileName, e->time); /* Open file */ /* message("Opening file '%s'.", fileName); */ @@ -730,7 +777,7 @@ void write_output_serial(struct engine* e, const char* baseName, if (h_grp < 0) error("Error while creating runtime parameters group\n"); /* Write the relevant information */ - writeAttribute(h_grp, "PeriodicBoundariesOn", INT, &periodic, 1); + io_write_attribute(h_grp, "PeriodicBoundariesOn", INT, &periodic, 1); /* Close runtime parameters */ H5Gclose(h_grp); @@ -741,47 +788,58 @@ void write_output_serial(struct engine* e, const char* baseName, if (h_grp < 0) error("Error while creating file header\n"); /* Print the relevant information and print status */ - writeAttribute(h_grp, "BoxSize", DOUBLE, e->s->dim, 3); + io_write_attribute(h_grp, "BoxSize", DOUBLE, e->s->dim, 3); double dblTime = e->time; - writeAttribute(h_grp, "Time", DOUBLE, &dblTime, 1); + io_write_attribute(h_grp, "Time", DOUBLE, &dblTime, 1); int dimension = (int)hydro_dimension; - writeAttribute(h_grp, "Dimension", INT, &dimension, 1); + io_write_attribute(h_grp, "Dimension", INT, &dimension, 1); /* GADGET-2 legacy values */ /* Number of particles of each type */ - unsigned int numParticles[NUM_PARTICLE_TYPES] = {0}; - unsigned int numParticlesHighWord[NUM_PARTICLE_TYPES] = {0}; - for (int ptype = 0; ptype < NUM_PARTICLE_TYPES; ++ptype) { + unsigned int numParticles[swift_type_count] = {0}; + unsigned int numParticlesHighWord[swift_type_count] = {0}; + for (int ptype = 0; ptype < swift_type_count; ++ptype) { numParticles[ptype] = (unsigned int)N_total[ptype]; numParticlesHighWord[ptype] = (unsigned int)(N_total[ptype] >> 32); } - writeAttribute(h_grp, "NumPart_ThisFile", LONGLONG, N_total, - NUM_PARTICLE_TYPES); - writeAttribute(h_grp, "NumPart_Total", UINT, numParticles, - NUM_PARTICLE_TYPES); - writeAttribute(h_grp, "NumPart_Total_HighWord", UINT, numParticlesHighWord, - NUM_PARTICLE_TYPES); + io_write_attribute(h_grp, "NumPart_ThisFile", LONGLONG, N_total, + swift_type_count); + io_write_attribute(h_grp, "NumPart_Total", UINT, numParticles, + swift_type_count); + io_write_attribute(h_grp, "NumPart_Total_HighWord", UINT, + numParticlesHighWord, swift_type_count); double MassTable[6] = {0., 0., 0., 0., 0., 0.}; - writeAttribute(h_grp, "MassTable", DOUBLE, MassTable, NUM_PARTICLE_TYPES); - unsigned int flagEntropy[NUM_PARTICLE_TYPES] = {0}; + io_write_attribute(h_grp, "MassTable", DOUBLE, MassTable, swift_type_count); + unsigned int flagEntropy[swift_type_count] = {0}; flagEntropy[0] = writeEntropyFlag(); - writeAttribute(h_grp, "Flag_Entropy_ICs", UINT, flagEntropy, - NUM_PARTICLE_TYPES); - writeAttribute(h_grp, "NumFilesPerSnapshot", INT, &numFiles, 1); + io_write_attribute(h_grp, "Flag_Entropy_ICs", UINT, flagEntropy, + swift_type_count); + io_write_attribute(h_grp, "NumFilesPerSnapshot", INT, &numFiles, 1); /* Close header */ H5Gclose(h_grp); /* Print the code version */ - writeCodeDescription(h_file); + io_write_code_description(h_file); /* Print the SPH parameters */ - h_grp = H5Gcreate(h_file, "/HydroScheme", H5P_DEFAULT, H5P_DEFAULT, - H5P_DEFAULT); - if (h_grp < 0) error("Error while creating SPH group"); - hydro_props_print_snapshot(h_grp, e->hydro_properties); - writeSPHflavour(h_grp); - H5Gclose(h_grp); + if (e->policy & engine_policy_hydro) { + h_grp = H5Gcreate(h_file, "/HydroScheme", H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT); + if (h_grp < 0) error("Error while creating SPH group"); + hydro_props_print_snapshot(h_grp, e->hydro_properties); + writeSPHflavour(h_grp); + H5Gclose(h_grp); + } + + /* Print the gravity parameters */ + if (e->policy & engine_policy_self_gravity) { + h_grp = H5Gcreate(h_file, "/GravityScheme", H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT); + if (h_grp < 0) error("Error while creating gravity group"); + gravity_props_print_snapshot(h_grp, e->gravity_properties); + H5Gclose(h_grp); + } /* Print the runtime parameters */ h_grp = @@ -791,10 +849,10 @@ void write_output_serial(struct engine* e, const char* baseName, H5Gclose(h_grp); /* Print the system of Units used in the spashot */ - writeUnitSystem(h_file, snapshot_units, "Units"); + io_write_unit_system(h_file, snapshot_units, "Units"); /* Print the system of Units used internally */ - writeUnitSystem(h_file, internal_units, "InternalCodeUnits"); + io_write_unit_system(h_file, internal_units, "InternalCodeUnits"); /* Tell the user if a conversion will be needed */ if (e->verbose) { @@ -830,7 +888,7 @@ void write_output_serial(struct engine* e, const char* baseName, } /* Loop over all particle types */ - for (int ptype = 0; ptype < NUM_PARTICLE_TYPES; ptype++) { + for (int ptype = 0; ptype < swift_type_count; ptype++) { /* Don't do anything if no particle of this kind */ if (N_total[ptype] == 0) continue; @@ -864,7 +922,7 @@ void write_output_serial(struct engine* e, const char* baseName, error("Error while opening file '%s' on rank %d.", fileName, mpi_rank); /* Loop over all particle types */ - for (int ptype = 0; ptype < NUM_PARTICLE_TYPES; ptype++) { + for (int ptype = 0; ptype < swift_type_count; ptype++) { /* Don't do anything if no particle of this kind */ if (N_total[ptype] == 0) continue; @@ -872,8 +930,8 @@ void write_output_serial(struct engine* e, const char* baseName, /* Add the global information for that particle type to the XMF * meta-file */ if (mpi_rank == 0) - writeXMFgroupheader(xmfFile, fileName, N_total[ptype], - (enum PARTICLE_TYPE)ptype); + xmf_write_groupheader(xmfFile, fileName, N_total[ptype], + (enum part_type)ptype); /* Open the particle group in the file */ char partTypeGroupName[PARTICLE_GROUP_BUFFER_SIZE]; @@ -891,12 +949,12 @@ void write_output_serial(struct engine* e, const char* baseName, /* Write particle fields from the particle structure */ switch (ptype) { - case GAS: + case swift_type_gas: Nparticles = Ngas; hydro_write_particles(parts, list, &num_fields); break; - case DM: + case swift_type_dark_matter: /* Allocate temporary array */ if (posix_memalign((void*)&dmparts, gpart_align, Ndm * sizeof(struct gpart)) != 0) @@ -904,12 +962,16 @@ void write_output_serial(struct engine* e, const char* baseName, bzero(dmparts, Ndm * sizeof(struct gpart)); /* Collect the DM particles from gpart */ - collect_dm_gparts(gparts, Ntot, dmparts, Ndm); + io_collect_dm_gparts(gparts, Ntot, dmparts, Ndm); /* Write DM particles */ Nparticles = Ndm; darkmatter_write_particles(dmparts, list, &num_fields); + break; + case swift_type_star: + Nparticles = Nstars; + star_write_particles(sparts, list, &num_fields); break; default: @@ -923,14 +985,17 @@ void write_output_serial(struct engine* e, const char* baseName, internal_units, snapshot_units); /* Free temporary array */ - free(dmparts); + if (dmparts) { + free(dmparts); + dmparts = 0; + } /* Close particle group */ H5Gclose(h_grp); /* Close this particle group in the XMF file as well */ if (mpi_rank == 0) - writeXMFgroupfooter(xmfFile, (enum PARTICLE_TYPE)ptype); + xmf_write_groupfooter(xmfFile, (enum part_type)ptype); } /* Close file */ @@ -942,7 +1007,7 @@ void write_output_serial(struct engine* e, const char* baseName, } /* Write footer of LXMF file descriptor */ - if (mpi_rank == 0) writeXMFoutputfooter(xmfFile, outputCount, e->time); + if (mpi_rank == 0) xmf_write_outputfooter(xmfFile, outputCount, e->time); /* message("Done writing particles..."); */ ++outputCount; diff --git a/src/serial_io.h b/src/serial_io.h index a2226e5cd9848ff2515b15111af43ccc67275a28..df8d0f1917a69eb28af32aed6780783b0f1099d8 100644 --- a/src/serial_io.h +++ b/src/serial_io.h @@ -34,15 +34,17 @@ #if defined(HAVE_HDF5) && defined(WITH_MPI) && !defined(HAVE_PARALLEL_HDF5) -void read_ic_serial(char* fileName, const struct UnitSystem* internal_units, +void read_ic_serial(char* fileName, const struct unit_system* internal_units, double dim[3], struct part** parts, struct gpart** gparts, - size_t* Ngas, size_t* Ngparts, int* periodic, - int* flag_entropy, int mpi_rank, int mpi_size, - MPI_Comm comm, MPI_Info info, int dry_run); + struct spart** sparts, size_t* Ngas, size_t* Ngparts, + size_t* Nstars, int* periodic, int* flag_entropy, + int with_hydro, int with_gravity, int with_stars, + int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info, + int dry_run); void write_output_serial(struct engine* e, const char* baseName, - const struct UnitSystem* internal_units, - const struct UnitSystem* snapshot_units, int mpi_rank, + const struct unit_system* internal_units, + const struct unit_system* snapshot_units, int mpi_rank, int mpi_size, MPI_Comm comm, MPI_Info info); #endif diff --git a/src/single_io.c b/src/single_io.c index ceeba4eb80a47c3feed7e898deb5e1fe7e427c0c..85c1286f7f0d3769c30a79af411b03d1523aa292 100644 --- a/src/single_io.c +++ b/src/single_io.c @@ -40,12 +40,15 @@ #include "engine.h" #include "error.h" #include "gravity_io.h" +#include "gravity_properties.h" #include "hydro_io.h" #include "hydro_properties.h" #include "io_properties.h" #include "kernel_hydro.h" #include "part.h" +#include "stars_io.h" #include "units.h" +#include "xmf.h" /*----------------------------------------------------------------------------- * Routines reading an IC file @@ -57,18 +60,18 @@ * @param h_grp The group from which to read. * @param prop The #io_props of the field to read * @param N The number of particles. - * @param internal_units The #UnitSystem used internally - * @param ic_units The #UnitSystem used in the ICs + * @param internal_units The #unit_system used internally + * @param ic_units The #unit_system used in the ICs * * @todo A better version using HDF5 hyper-slabs to read the file directly into *the part array * will be written once the structures have been stabilized. */ void readArray(hid_t h_grp, const struct io_props prop, size_t N, - const struct UnitSystem* internal_units, - const struct UnitSystem* ic_units) { + const struct unit_system* internal_units, + const struct unit_system* ic_units) { - const size_t typeSize = sizeOfType(prop.type); + const size_t typeSize = io_sizeof_type(prop.type); const size_t copySize = typeSize * prop.dimension; const size_t num_elements = N * prop.dimension; @@ -103,7 +106,7 @@ void readArray(hid_t h_grp, const struct io_props prop, size_t N, /* Check data type */ const hid_t h_type = H5Dget_type(h_data); if (h_type < 0) error("Unable to retrieve data type from the file"); - // if (!H5Tequal(h_type, hdf5Type(type))) + // if (!H5Tequal(h_type, hdf5_type(type))) // error("Non-matching types between the code and the file"); /* Allocate temporary buffer */ @@ -113,8 +116,8 @@ void readArray(hid_t h_grp, const struct io_props prop, size_t N, /* Read HDF5 dataspace in temporary buffer */ /* Dirty version that happens to work for vectors but should be improved */ /* Using HDF5 dataspaces would be better */ - const hid_t h_err = - H5Dread(h_data, hdf5Type(prop.type), H5S_ALL, H5S_ALL, H5P_DEFAULT, temp); + const hid_t h_err = H5Dread(h_data, io_hdf5_type(prop.type), H5S_ALL, H5S_ALL, + H5P_DEFAULT, temp); if (h_err < 0) { error("Error while reading data array '%s'.", prop.name); } @@ -126,7 +129,7 @@ void readArray(hid_t h_grp, const struct io_props prop, size_t N, /* message("Converting ! factor=%e", factor); */ - if (isDoublePrecision(prop.type)) { + if (io_is_double_precision(prop.type)) { double* temp_d = temp; for (size_t i = 0; i < num_elements; ++i) temp_d[i] *= factor; } else { @@ -161,25 +164,25 @@ void readArray(hid_t h_grp, const struct io_props prop, size_t N, * the HDF5 file. * @param props The #io_props of the field to read * @param N The number of particles to write. - * @param internal_units The #UnitSystem used internally - * @param snapshot_units The #UnitSystem used in the snapshots + * @param internal_units The #unit_system used internally + * @param snapshot_units The #unit_system used in the snapshots * * @todo A better version using HDF5 hyper-slabs to write the file directly from * the part array will be written once the structures have been stabilized. */ void writeArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile, char* partTypeGroupName, const struct io_props props, size_t N, - const struct UnitSystem* internal_units, - const struct UnitSystem* snapshot_units) { + const struct unit_system* internal_units, + const struct unit_system* snapshot_units) { - const size_t typeSize = sizeOfType(props.type); + const size_t typeSize = io_sizeof_type(props.type); const size_t copySize = typeSize * props.dimension; const size_t num_elements = N * props.dimension; /* message("Writing '%s' array...", props.name); */ /* Allocate temporary buffer */ - void* temp = malloc(num_elements * sizeOfType(props.type)); + void* temp = malloc(num_elements * io_sizeof_type(props.type)); if (temp == NULL) error("Unable to allocate memory for temporary buffer"); /* Copy particle data to temporary buffer */ @@ -210,7 +213,7 @@ void writeArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile, /* message("Converting ! factor=%e", factor); */ - if (isDoublePrecision(props.type)) { + if (io_is_double_precision(props.type)) { double* temp_d = temp; for (size_t i = 0; i < num_elements; ++i) temp_d[i] *= factor; } else { @@ -271,33 +274,34 @@ void writeArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile, } /* Create dataset */ - const hid_t h_data = H5Dcreate(grp, props.name, hdf5Type(props.type), h_space, - H5P_DEFAULT, h_prop, H5P_DEFAULT); + const hid_t h_data = H5Dcreate(grp, props.name, io_hdf5_type(props.type), + h_space, H5P_DEFAULT, h_prop, H5P_DEFAULT); if (h_data < 0) { error("Error while creating dataspace '%s'.", props.name); } /* Write temporary buffer to HDF5 dataspace */ - h_err = H5Dwrite(h_data, hdf5Type(props.type), h_space, H5S_ALL, H5P_DEFAULT, - temp); + h_err = H5Dwrite(h_data, io_hdf5_type(props.type), h_space, H5S_ALL, + H5P_DEFAULT, temp); if (h_err < 0) { error("Error while writing data array '%s'.", props.name); } /* Write XMF description for this data set */ - writeXMFline(xmfFile, fileName, partTypeGroupName, props.name, N, - props.dimension, props.type); + xmf_write_line(xmfFile, fileName, partTypeGroupName, props.name, N, + props.dimension, props.type); /* Write unit conversion factors for this data set */ char buffer[FIELD_BUFFER_SIZE]; units_cgs_conversion_string(buffer, snapshot_units, props.units); - writeAttribute_d(h_data, "CGS conversion factor", - units_cgs_conversion_factor(snapshot_units, props.units)); - writeAttribute_f(h_data, "h-scale exponent", - units_h_factor(snapshot_units, props.units)); - writeAttribute_f(h_data, "a-scale exponent", - units_a_factor(snapshot_units, props.units)); - writeAttribute_s(h_data, "Conversion factor", buffer); + io_write_attribute_d( + h_data, "CGS conversion factor", + units_cgs_conversion_factor(snapshot_units, props.units)); + io_write_attribute_f(h_data, "h-scale exponent", + units_h_factor(snapshot_units, props.units)); + io_write_attribute_f(h_data, "a-scale exponent", + units_a_factor(snapshot_units, props.units)); + io_write_attribute_s(h_data, "Conversion factor", buffer); /* Free and close everything */ free(temp); @@ -312,14 +316,18 @@ void writeArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile, * @param fileName The file to read. * @param internal_units The system units used internally * @param dim (output) The dimension of the volume. - * @param parts (output) Array of Gas particles. + * @param parts (output) Array of #part particles. * @param gparts (output) Array of #gpart particles. + * @param sparts (output) Array of #spart particles. * @param Ngas (output) number of Gas particles read. * @param Ngparts (output) The number of #gpart read. + * @param Nstars (output) The number of #spart read. * @param periodic (output) 1 if the volume is periodic, 0 if not. * @param flag_entropy (output) 1 if the ICs contained Entropy in the - * InternalEnergy - * field + * InternalEnergy field + * @param with_hydro Are we reading gas particles ? + * @param with_gravity Are we reading/creating #gpart arrays ? + * @param with_stars Are we reading star particles ? * @param dry_run If 1, don't read the particle. Only allocates the arrays. * * Opens the HDF5 file fileName and reads the particles contained @@ -330,20 +338,22 @@ void writeArray(struct engine* e, hid_t grp, char* fileName, FILE* xmfFile, * @todo Read snapshots distributed in more than one file. * */ -void read_ic_single(char* fileName, const struct UnitSystem* internal_units, +void read_ic_single(char* fileName, const struct unit_system* internal_units, double dim[3], struct part** parts, struct gpart** gparts, - size_t* Ngas, size_t* Ngparts, int* periodic, - int* flag_entropy, int dry_run) { + struct spart** sparts, size_t* Ngas, size_t* Ngparts, + size_t* Nstars, int* periodic, int* flag_entropy, + int with_hydro, int with_gravity, int with_stars, + int dry_run) { hid_t h_file = 0, h_grp = 0; /* GADGET has only cubic boxes (in cosmological mode) */ double boxSize[3] = {0.0, -1.0, -1.0}; /* GADGET has 6 particle types. We only keep the type 0 & 1 for now...*/ - int numParticles[NUM_PARTICLE_TYPES] = {0}; - int numParticles_highWord[NUM_PARTICLE_TYPES] = {0}; - size_t N[NUM_PARTICLE_TYPES] = {0}; + int numParticles[swift_type_count] = {0}; + int numParticles_highWord[swift_type_count] = {0}; + size_t N[swift_type_count] = {0}; int dimension = 3; /* Assume 3D if nothing is specified */ - size_t Ndm; + size_t Ndm = 0; /* Open file */ /* message("Opening file '%s' as IC.", fileName); */ @@ -358,7 +368,7 @@ void read_ic_single(char* fileName, const struct UnitSystem* internal_units, if (h_grp < 0) error("Error while opening runtime parameters\n"); /* Read the relevant information */ - readAttribute(h_grp, "PeriodicBoundariesOn", INT, periodic); + io_read_attribute(h_grp, "PeriodicBoundariesOn", INT, periodic); /* Close runtime parameters */ H5Gclose(h_grp); @@ -372,20 +382,21 @@ void read_ic_single(char* fileName, const struct UnitSystem* internal_units, const hid_t hid_dim = H5Aexists(h_grp, "Dimension"); if (hid_dim < 0) error("Error while testing existance of 'Dimension' attribute"); - if (hid_dim > 0) readAttribute(h_grp, "Dimension", INT, &dimension); + if (hid_dim > 0) io_read_attribute(h_grp, "Dimension", INT, &dimension); if (dimension != hydro_dimension) error("ICs dimensionality (%dD) does not match code dimensionality (%dD)", dimension, (int)hydro_dimension); /* Read the relevant information and print status */ int flag_entropy_temp[6]; - readAttribute(h_grp, "Flag_Entropy_ICs", INT, flag_entropy_temp); + io_read_attribute(h_grp, "Flag_Entropy_ICs", INT, flag_entropy_temp); *flag_entropy = flag_entropy_temp[0]; - readAttribute(h_grp, "BoxSize", DOUBLE, boxSize); - readAttribute(h_grp, "NumPart_Total", UINT, numParticles); - readAttribute(h_grp, "NumPart_Total_HighWord", UINT, numParticles_highWord); + io_read_attribute(h_grp, "BoxSize", DOUBLE, boxSize); + io_read_attribute(h_grp, "NumPart_Total", UINT, numParticles); + io_read_attribute(h_grp, "NumPart_Total_HighWord", UINT, + numParticles_highWord); - for (int ptype = 0; ptype < NUM_PARTICLE_TYPES; ++ptype) + for (int ptype = 0; ptype < swift_type_count; ++ptype) N[ptype] = ((long long)numParticles[ptype]) + ((long long)numParticles_highWord[ptype] << 32); @@ -400,9 +411,9 @@ void read_ic_single(char* fileName, const struct UnitSystem* internal_units, H5Gclose(h_grp); /* Read the unit system used in the ICs */ - struct UnitSystem* ic_units = malloc(sizeof(struct UnitSystem)); + struct unit_system* ic_units = malloc(sizeof(struct unit_system)); if (ic_units == NULL) error("Unable to allocate memory for IC unit system"); - readUnitSystem(h_file, ic_units); + io_read_unit_system(h_file, ic_units); /* Tell the user if a conversion will be needed */ if (units_are_equal(ic_units, internal_units)) { @@ -439,19 +450,34 @@ void read_ic_single(char* fileName, const struct UnitSystem* internal_units, units_conversion_factor(ic_units, internal_units, UNIT_CONV_LENGTH); /* Allocate memory to store SPH particles */ - *Ngas = N[0]; - if (posix_memalign((void*)parts, part_align, *Ngas * sizeof(struct part)) != - 0) - error("Error while allocating memory for SPH particles"); - bzero(*parts, *Ngas * sizeof(struct part)); - - /* Allocate memory to store all particles */ - Ndm = N[1]; - *Ngparts = N[1] + N[0]; - if (posix_memalign((void*)gparts, gpart_align, - *Ngparts * sizeof(struct gpart)) != 0) - error("Error while allocating memory for gravity particles"); - bzero(*gparts, *Ngparts * sizeof(struct gpart)); + if (with_hydro) { + *Ngas = N[swift_type_gas]; + if (posix_memalign((void*)parts, part_align, *Ngas * sizeof(struct part)) != + 0) + error("Error while allocating memory for SPH particles"); + bzero(*parts, *Ngas * sizeof(struct part)); + } + + /* Allocate memory to store star particles */ + if (with_stars) { + *Nstars = N[swift_type_star]; + if (posix_memalign((void*)sparts, spart_align, + *Nstars * sizeof(struct spart)) != 0) + error("Error while allocating memory for star particles"); + bzero(*sparts, *Nstars * sizeof(struct spart)); + } + + /* Allocate memory to store all gravity particles */ + if (with_gravity) { + Ndm = N[swift_type_dark_matter]; + *Ngparts = (with_hydro ? N[swift_type_gas] : 0) + + N[swift_type_dark_matter] + + (with_stars ? N[swift_type_star] : 0); + if (posix_memalign((void*)gparts, gpart_align, + *Ngparts * sizeof(struct gpart)) != 0) + error("Error while allocating memory for gravity particles"); + bzero(*gparts, *Ngparts * sizeof(struct gpart)); + } /* message("Allocated %8.2f MB for particles.", *N * sizeof(struct part) / * (1024.*1024.)); */ @@ -460,7 +486,7 @@ void read_ic_single(char* fileName, const struct UnitSystem* internal_units, /* message("NumPart = [%zd, %zd] Total = %zd", *Ngas, Ndm, *Ngparts); */ /* Loop over all particle types */ - for (int ptype = 0; ptype < NUM_PARTICLE_TYPES; ptype++) { + for (int ptype = 0; ptype < swift_type_count; ptype++) { /* Don't do anything if no particle of this kind */ if (N[ptype] == 0) continue; @@ -481,14 +507,25 @@ void read_ic_single(char* fileName, const struct UnitSystem* internal_units, /* Read particle fields into the structure */ switch (ptype) { - case GAS: - Nparticles = *Ngas; - hydro_read_particles(*parts, list, &num_fields); + case swift_type_gas: + if (with_hydro) { + Nparticles = *Ngas; + hydro_read_particles(*parts, list, &num_fields); + } break; - case DM: - Nparticles = Ndm; - darkmatter_read_particles(*gparts, list, &num_fields); + case swift_type_dark_matter: + if (with_gravity) { + Nparticles = Ndm; + darkmatter_read_particles(*gparts, list, &num_fields); + } + break; + + case swift_type_star: + if (with_stars) { + Nparticles = *Nstars; + star_read_particles(*sparts, list, &num_fields); + } break; default: @@ -505,10 +542,15 @@ void read_ic_single(char* fileName, const struct UnitSystem* internal_units, } /* Prepare the DM particles */ - if (!dry_run) prepare_dm_gparts(*gparts, Ndm); + if (!dry_run && with_gravity) io_prepare_dm_gparts(*gparts, Ndm); + + /* Duplicate the hydro particles into gparts */ + if (!dry_run && with_gravity && with_hydro) + io_duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm); - /* Now duplicate the hydro particle into gparts */ - if (!dry_run) duplicate_hydro_gparts(*parts, *gparts, *Ngas, Ndm); + /* Duplicate the star particles into gparts */ + if (!dry_run && with_gravity && with_stars) + io_duplicate_star_gparts(*sparts, *gparts, *Nstars, Ndm + *Ngas); /* message("Done Reading particles..."); */ @@ -524,8 +566,8 @@ void read_ic_single(char* fileName, const struct UnitSystem* internal_units, * * @param e The engine containing all the system. * @param baseName The common part of the snapshot file name. - * @param internal_units The #UnitSystem used internally - * @param snapshot_units The #UnitSystem used in the snapshots + * @param internal_units The #unit_system used internally + * @param snapshot_units The #unit_system used in the snapshots * * Creates an HDF5 output file and writes the particles contained * in the engine. If such a file already exists, it is erased and replaced @@ -536,23 +578,25 @@ void read_ic_single(char* fileName, const struct UnitSystem* internal_units, * */ void write_output_single(struct engine* e, const char* baseName, - const struct UnitSystem* internal_units, - const struct UnitSystem* snapshot_units) { + const struct unit_system* internal_units, + const struct unit_system* snapshot_units) { hid_t h_file = 0, h_grp = 0; const size_t Ngas = e->s->nr_parts; + const size_t Nstars = e->s->nr_sparts; const size_t Ntot = e->s->nr_gparts; int periodic = e->s->periodic; int numFiles = 1; struct part* parts = e->s->parts; struct gpart* gparts = e->s->gparts; struct gpart* dmparts = NULL; + struct spart* sparts = e->s->sparts; static int outputCount = 0; /* Number of unassociated gparts */ - const size_t Ndm = Ntot > 0 ? Ntot - Ngas : 0; + const size_t Ndm = Ntot > 0 ? Ntot - (Ngas + Nstars) : 0; - long long N_total[NUM_PARTICLE_TYPES] = {Ngas, Ndm, 0}; + long long N_total[swift_type_count] = {Ngas, Ndm, 0, 0, Nstars, 0}; /* File name */ char fileName[FILENAME_BUFFER_SIZE]; @@ -560,14 +604,14 @@ void write_output_single(struct engine* e, const char* baseName, outputCount); /* First time, we need to create the XMF file */ - if (outputCount == 0) createXMFfile(baseName); + if (outputCount == 0) xmf_create_file(baseName); /* Prepare the XMF file for the new entry */ FILE* xmfFile = 0; - xmfFile = prepareXMFfile(baseName); + xmfFile = xmf_prepare_file(baseName); /* Write the part corresponding to this specific output */ - writeXMFoutputheader(xmfFile, fileName, e->time); + xmf_write_outputheader(xmfFile, fileName, e->time); /* Open file */ /* message("Opening file '%s'.", fileName); */ @@ -583,7 +627,7 @@ void write_output_single(struct engine* e, const char* baseName, if (h_grp < 0) error("Error while creating runtime parameters group\n"); /* Write the relevant information */ - writeAttribute(h_grp, "PeriodicBoundariesOn", INT, &periodic, 1); + io_write_attribute(h_grp, "PeriodicBoundariesOn", INT, &periodic, 1); /* Close runtime parameters */ H5Gclose(h_grp); @@ -594,47 +638,58 @@ void write_output_single(struct engine* e, const char* baseName, if (h_grp < 0) error("Error while creating file header\n"); /* Print the relevant information and print status */ - writeAttribute(h_grp, "BoxSize", DOUBLE, e->s->dim, 3); + io_write_attribute(h_grp, "BoxSize", DOUBLE, e->s->dim, 3); double dblTime = e->time; - writeAttribute(h_grp, "Time", DOUBLE, &dblTime, 1); + io_write_attribute(h_grp, "Time", DOUBLE, &dblTime, 1); int dimension = (int)hydro_dimension; - writeAttribute(h_grp, "Dimension", INT, &dimension, 1); + io_write_attribute(h_grp, "Dimension", INT, &dimension, 1); /* GADGET-2 legacy values */ /* Number of particles of each type */ - unsigned int numParticles[NUM_PARTICLE_TYPES] = {0}; - unsigned int numParticlesHighWord[NUM_PARTICLE_TYPES] = {0}; - for (int ptype = 0; ptype < NUM_PARTICLE_TYPES; ++ptype) { + unsigned int numParticles[swift_type_count] = {0}; + unsigned int numParticlesHighWord[swift_type_count] = {0}; + for (int ptype = 0; ptype < swift_type_count; ++ptype) { numParticles[ptype] = (unsigned int)N_total[ptype]; numParticlesHighWord[ptype] = (unsigned int)(N_total[ptype] >> 32); } - writeAttribute(h_grp, "NumPart_ThisFile", LONGLONG, N_total, - NUM_PARTICLE_TYPES); - writeAttribute(h_grp, "NumPart_Total", UINT, numParticles, - NUM_PARTICLE_TYPES); - writeAttribute(h_grp, "NumPart_Total_HighWord", UINT, numParticlesHighWord, - NUM_PARTICLE_TYPES); - double MassTable[NUM_PARTICLE_TYPES] = {0}; - writeAttribute(h_grp, "MassTable", DOUBLE, MassTable, NUM_PARTICLE_TYPES); - unsigned int flagEntropy[NUM_PARTICLE_TYPES] = {0}; + io_write_attribute(h_grp, "NumPart_ThisFile", LONGLONG, N_total, + swift_type_count); + io_write_attribute(h_grp, "NumPart_Total", UINT, numParticles, + swift_type_count); + io_write_attribute(h_grp, "NumPart_Total_HighWord", UINT, + numParticlesHighWord, swift_type_count); + double MassTable[swift_type_count] = {0}; + io_write_attribute(h_grp, "MassTable", DOUBLE, MassTable, swift_type_count); + unsigned int flagEntropy[swift_type_count] = {0}; flagEntropy[0] = writeEntropyFlag(); - writeAttribute(h_grp, "Flag_Entropy_ICs", UINT, flagEntropy, - NUM_PARTICLE_TYPES); - writeAttribute(h_grp, "NumFilesPerSnapshot", INT, &numFiles, 1); + io_write_attribute(h_grp, "Flag_Entropy_ICs", UINT, flagEntropy, + swift_type_count); + io_write_attribute(h_grp, "NumFilesPerSnapshot", INT, &numFiles, 1); /* Close header */ H5Gclose(h_grp); /* Print the code version */ - writeCodeDescription(h_file); + io_write_code_description(h_file); /* Print the SPH parameters */ - h_grp = - H5Gcreate(h_file, "/HydroScheme", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if (h_grp < 0) error("Error while creating SPH group"); - hydro_props_print_snapshot(h_grp, e->hydro_properties); - writeSPHflavour(h_grp); - H5Gclose(h_grp); + if (e->policy & engine_policy_hydro) { + h_grp = H5Gcreate(h_file, "/HydroScheme", H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT); + if (h_grp < 0) error("Error while creating SPH group"); + hydro_props_print_snapshot(h_grp, e->hydro_properties); + writeSPHflavour(h_grp); + H5Gclose(h_grp); + } + + /* Print the gravity parameters */ + if (e->policy & engine_policy_self_gravity) { + h_grp = H5Gcreate(h_file, "/GravityScheme", H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT); + if (h_grp < 0) error("Error while creating gravity group"); + gravity_props_print_snapshot(h_grp, e->gravity_properties); + H5Gclose(h_grp); + } /* Print the runtime parameters */ h_grp = @@ -644,10 +699,10 @@ void write_output_single(struct engine* e, const char* baseName, H5Gclose(h_grp); /* Print the system of Units used in the spashot */ - writeUnitSystem(h_file, snapshot_units, "Units"); + io_write_unit_system(h_file, snapshot_units, "Units"); /* Print the system of Units used internally */ - writeUnitSystem(h_file, internal_units, "InternalCodeUnits"); + io_write_unit_system(h_file, internal_units, "InternalCodeUnits"); /* Tell the user if a conversion will be needed */ if (e->verbose) { @@ -683,14 +738,14 @@ void write_output_single(struct engine* e, const char* baseName, } /* Loop over all particle types */ - for (int ptype = 0; ptype < NUM_PARTICLE_TYPES; ptype++) { + for (int ptype = 0; ptype < swift_type_count; ptype++) { /* Don't do anything if no particle of this kind */ if (numParticles[ptype] == 0) continue; /* Add the global information for that particle type to the XMF meta-file */ - writeXMFgroupheader(xmfFile, fileName, numParticles[ptype], - (enum PARTICLE_TYPE)ptype); + xmf_write_groupheader(xmfFile, fileName, numParticles[ptype], + (enum part_type)ptype); /* Open the particle group in the file */ char partTypeGroupName[PARTICLE_GROUP_BUFFER_SIZE]; @@ -709,12 +764,12 @@ void write_output_single(struct engine* e, const char* baseName, /* Write particle fields from the particle structure */ switch (ptype) { - case GAS: + case swift_type_gas: N = Ngas; hydro_write_particles(parts, list, &num_fields); break; - case DM: + case swift_type_dark_matter: /* Allocate temporary array */ if (posix_memalign((void*)&dmparts, gpart_align, Ndm * sizeof(struct gpart)) != 0) @@ -722,13 +777,18 @@ void write_output_single(struct engine* e, const char* baseName, bzero(dmparts, Ndm * sizeof(struct gpart)); /* Collect the DM particles from gpart */ - collect_dm_gparts(gparts, Ntot, dmparts, Ndm); + io_collect_dm_gparts(gparts, Ntot, dmparts, Ndm); /* Write DM particles */ N = Ndm; darkmatter_write_particles(dmparts, list, &num_fields); break; + case swift_type_star: + N = Nstars; + star_write_particles(sparts, list, &num_fields); + break; + default: error("Particle Type %d not yet supported. Aborting", ptype); } @@ -739,17 +799,20 @@ void write_output_single(struct engine* e, const char* baseName, internal_units, snapshot_units); /* Free temporary array */ - free(dmparts); + if (dmparts) { + free(dmparts); + dmparts = NULL; + } /* Close particle group */ H5Gclose(h_grp); /* Close this particle group in the XMF file as well */ - writeXMFgroupfooter(xmfFile, (enum PARTICLE_TYPE)ptype); + xmf_write_groupfooter(xmfFile, (enum part_type)ptype); } /* Write LXMF file descriptor */ - writeXMFoutputfooter(xmfFile, outputCount, e->time); + xmf_write_outputfooter(xmfFile, outputCount, e->time); /* message("Done writing particles..."); */ diff --git a/src/single_io.h b/src/single_io.h index 51a30a7bc6af7f3aaf5708a3d2df14982e026e3e..e1f1b3a7b39ff3f0d31078cd34b7a6cd5f0dfc77 100644 --- a/src/single_io.h +++ b/src/single_io.h @@ -29,14 +29,16 @@ #if defined(HAVE_HDF5) && !defined(WITH_MPI) -void read_ic_single(char* fileName, const struct UnitSystem* internal_units, +void read_ic_single(char* fileName, const struct unit_system* internal_units, double dim[3], struct part** parts, struct gpart** gparts, - size_t* Ngas, size_t* Ndm, int* periodic, int* flag_entropy, + struct spart** sparts, size_t* Ngas, size_t* Ndm, + size_t* Nstars, int* periodic, int* flag_entropy, + int with_hydro, int with_gravity, int with_stars, int dry_run); void write_output_single(struct engine* e, const char* baseName, - const struct UnitSystem* internal_units, - const struct UnitSystem* snapshot_units); + const struct unit_system* internal_units, + const struct unit_system* snapshot_units); #endif diff --git a/src/sourceterms.c b/src/sourceterms.c index 2a5f326688b52b0e4abfde8dca823d765e0e8a5e..f12071cf912eae3aa8d0e25f0f3b4c5e139de667 100644 --- a/src/sourceterms.c +++ b/src/sourceterms.c @@ -37,7 +37,7 @@ * @param source the structure that has all the source term properties */ void sourceterms_init(const struct swift_params* parameter_file, - struct UnitSystem* us, struct sourceterms* source) { + struct unit_system* us, struct sourceterms* source) { #ifdef SOURCETERMS_SN_FEEDBACK supernova_init(parameter_file, us, source); #endif /* SOURCETERMS_SN_FEEDBACK */ diff --git a/src/sourceterms.h b/src/sourceterms.h index 95c37e9bba9881702a6cf6caea1004b8cfb52636..1445bcb777ff634d1e3a2312cb0a49ac155e1020 100644 --- a/src/sourceterms.h +++ b/src/sourceterms.h @@ -42,7 +42,7 @@ struct sourceterms { #endif void sourceterms_init(const struct swift_params* parameter_file, - struct UnitSystem* us, struct sourceterms* source); + struct unit_system* us, struct sourceterms* source); void sourceterms_print(struct sourceterms* source); /** diff --git a/src/sourceterms/sn_feedback/sn_feedback.h b/src/sourceterms/sn_feedback/sn_feedback.h index 667c79cb1eef42f7fde79c43059d1baa5c61268e..f2f224ce871ebb768c318aef42a690861dd974df 100644 --- a/src/sourceterms/sn_feedback/sn_feedback.h +++ b/src/sourceterms/sn_feedback/sn_feedback.h @@ -171,7 +171,7 @@ __attribute__((always_inline)) INLINE static void supernova_feedback_apply( */ __attribute__((always_inline)) INLINE static void supernova_init( - const struct swift_params* parameter_file, struct UnitSystem* us, + const struct swift_params* parameter_file, struct unit_system* us, struct sourceterms* source) { source->supernova.time = parser_get_param_double(parameter_file, "SN:time"); source->supernova.energy = diff --git a/src/space.c b/src/space.c index b7b3523e6a4dc345f3acdad3552482ec8738b8f3..7920a3697f629e8d57bbd09660e5e82eacbed0c0 100644 --- a/src/space.c +++ b/src/space.c @@ -51,7 +51,9 @@ #include "lock.h" #include "memswap.h" #include "minmax.h" +#include "multipole.h" #include "runner.h" +#include "stars.h" #include "threadpool.h" #include "tools.h" @@ -107,6 +109,7 @@ struct parallel_sort { struct part *parts; struct gpart *gparts; struct xpart *xparts; + struct spart *sparts; int *ind; struct qstack *stack; unsigned int stack_size; @@ -173,18 +176,38 @@ int space_getsid(struct space *s, struct cell **ci, struct cell **cj, * * @param s The #space. * @param c The #cell to recycle. - * @param rec_begin Pointer to the start of the list of cells to recycle. - * @param rec_end Pointer to the end of the list of cells to recycle. + * @param cell_rec_begin Pointer to the start of the list of cells to recycle. + * @param cell_rec_end Pointer to the end of the list of cells to recycle. + * @param multipole_rec_begin Pointer to the start of the list of multipoles to + * recycle. + * @param multipole_rec_end Pointer to the end of the list of multipoles to + * recycle. */ void space_rebuild_recycle_rec(struct space *s, struct cell *c, - struct cell **rec_begin, struct cell **rec_end) { + struct cell **cell_rec_begin, + struct cell **cell_rec_end, + struct gravity_tensors **multipole_rec_begin, + struct gravity_tensors **multipole_rec_end) { if (c->split) for (int k = 0; k < 8; k++) if (c->progeny[k] != NULL) { - space_rebuild_recycle_rec(s, c->progeny[k], rec_begin, rec_end); - c->progeny[k]->next = *rec_begin; - *rec_begin = c->progeny[k]; - if (*rec_end == NULL) *rec_end = *rec_begin; + space_rebuild_recycle_rec(s, c->progeny[k], cell_rec_begin, + cell_rec_end, multipole_rec_begin, + multipole_rec_end); + + c->progeny[k]->next = *cell_rec_begin; + *cell_rec_begin = c->progeny[k]; + + if (s->gravity) { + c->progeny[k]->multipole->next = *multipole_rec_begin; + *multipole_rec_begin = c->progeny[k]->multipole; + } + + if (*cell_rec_end == NULL) *cell_rec_end = *cell_rec_begin; + if (s->gravity && *multipole_rec_end == NULL) + *multipole_rec_end = *multipole_rec_begin; + + c->progeny[k]->multipole = NULL; c->progeny[k] = NULL; } } @@ -197,9 +220,14 @@ void space_rebuild_recycle_mapper(void *map_data, int num_elements, for (int k = 0; k < num_elements; k++) { struct cell *c = &cells[k]; - struct cell *rec_begin = NULL, *rec_end = NULL; - space_rebuild_recycle_rec(s, c, &rec_begin, &rec_end); - if (rec_begin != NULL) space_recycle_list(s, rec_begin, rec_end); + struct cell *cell_rec_begin = NULL, *cell_rec_end = NULL; + struct gravity_tensors *multipole_rec_begin = NULL, + *multipole_rec_end = NULL; + space_rebuild_recycle_rec(s, c, &cell_rec_begin, &cell_rec_end, + &multipole_rec_begin, &multipole_rec_end); + if (cell_rec_begin != NULL) + space_recycle_list(s, cell_rec_begin, cell_rec_end, multipole_rec_begin, + multipole_rec_end); c->sorts = NULL; c->nr_tasks = 0; c->density = NULL; @@ -211,6 +239,7 @@ void space_rebuild_recycle_mapper(void *map_data, int num_elements, c->sorted = 0; c->count = 0; c->gcount = 0; + c->scount = 0; c->init = NULL; c->extra_ghost = NULL; c->ghost = NULL; @@ -220,6 +249,9 @@ void space_rebuild_recycle_mapper(void *map_data, int num_elements, c->drift = NULL; c->cooling = NULL; c->sourceterms = NULL; + c->grav_top_level = NULL; + c->grav_long_range = NULL; + c->grav_down = NULL; c->super = c; if (c->sort != NULL) { free(c->sort); @@ -249,7 +281,7 @@ void space_regrid(struct space *s, int verbose) { const size_t nr_parts = s->nr_parts; const ticks tic = getticks(); - const integertime_t ti_current = (s->e != NULL) ? s->e->ti_current : 0; + const integertime_t ti_old = (s->e != NULL) ? s->e->ti_old : 0; /* Run through the cells and get the current h_max. */ // tic = getticks(); @@ -360,6 +392,7 @@ void space_regrid(struct space *s, int verbose) { threadpool_map(&s->e->threadpool, space_rebuild_recycle_mapper, s->cells_top, s->nr_cells, sizeof(struct cell), 100, s); free(s->cells_top); + free(s->multipoles_top); s->maxdepth = 0; } @@ -375,19 +408,35 @@ void space_regrid(struct space *s, int verbose) { s->tot_cells = s->nr_cells = cdim[0] * cdim[1] * cdim[2]; if (posix_memalign((void *)&s->cells_top, cell_align, s->nr_cells * sizeof(struct cell)) != 0) - error("Failed to allocate cells."); + error("Failed to allocate top-level cells."); bzero(s->cells_top, s->nr_cells * sizeof(struct cell)); + /* Allocate the multipoles for the top-level cells. */ + if (s->gravity) { + if (posix_memalign((void *)&s->multipoles_top, multipole_align, + s->nr_cells * sizeof(struct gravity_tensors)) != 0) + error("Failed to allocate top-level multipoles."); + bzero(s->multipoles_top, s->nr_cells * sizeof(struct gravity_tensors)); + } + /* Set the cells' locks */ - for (int k = 0; k < s->nr_cells; k++) + for (int k = 0; k < s->nr_cells; k++) { if (lock_init(&s->cells_top[k].lock) != 0) - error("Failed to init spinlock."); + error("Failed to init spinlock for hydro."); + if (lock_init(&s->cells_top[k].glock) != 0) + error("Failed to init spinlock for gravity."); + if (lock_init(&s->cells_top[k].mlock) != 0) + error("Failed to init spinlock for multipoles."); + if (lock_init(&s->cells_top[k].slock) != 0) + error("Failed to init spinlock for stars."); + } /* Set the cell location and sizes. */ for (int i = 0; i < cdim[0]; i++) for (int j = 0; j < cdim[1]; j++) for (int k = 0; k < cdim[2]; k++) { - struct cell *restrict c = &s->cells_top[cell_getid(cdim, i, j, k)]; + const size_t cid = cell_getid(cdim, i, j, k); + struct cell *restrict c = &s->cells_top[cid]; c->loc[0] = i * s->width[0]; c->loc[1] = j * s->width[1]; c->loc[2] = k * s->width[2]; @@ -398,9 +447,11 @@ void space_regrid(struct space *s, int verbose) { c->depth = 0; c->count = 0; c->gcount = 0; + c->scount = 0; c->super = c; - c->ti_old = ti_current; - lock_init(&c->lock); + c->ti_old = ti_old; + c->ti_old_multipole = ti_old; + if (s->gravity) c->multipole = &s->multipoles_top[cid]; } /* Be verbose about the change. */ @@ -482,8 +533,9 @@ void space_rebuild(struct space *s, int verbose) { size_t nr_parts = s->nr_parts; size_t nr_gparts = s->nr_gparts; + size_t nr_sparts = s->nr_sparts; struct cell *restrict cells_top = s->cells_top; - const integertime_t ti_current = (s->e != NULL) ? s->e->ti_current : 0; + const integertime_t ti_old = (s->e != NULL) ? s->e->ti_old : 0; /* Run through the particles and get their cell index. Allocates an index that is larger than the number of particles to avoid @@ -502,25 +554,37 @@ void space_rebuild(struct space *s, int verbose) { if (s->size_gparts > 0) space_gparts_get_cell_index(s, gind, cells_top, verbose); + /* Run through the star particles and get their cell index. */ + const size_t sind_size = s->size_sparts + 100; + int *sind; + if ((sind = (int *)malloc(sizeof(int) * sind_size)) == NULL) + error("Failed to allocate temporary s-particle indices."); + if (s->size_sparts > 0) + space_sparts_get_cell_index(s, sind, cells_top, verbose); + #ifdef WITH_MPI + const int local_nodeID = s->e->nodeID; /* Move non-local parts to the end of the list. */ - const int local_nodeID = s->e->nodeID; for (size_t k = 0; k < nr_parts;) { if (cells_top[ind[k]].nodeID != local_nodeID) { nr_parts -= 1; + /* Swap the particle */ const struct part tp = s->parts[k]; s->parts[k] = s->parts[nr_parts]; s->parts[nr_parts] = tp; + /* Swap the link with the gpart */ if (s->parts[k].gpart != NULL) { s->parts[k].gpart->id_or_neg_offset = -k; } if (s->parts[nr_parts].gpart != NULL) { s->parts[nr_parts].gpart->id_or_neg_offset = -nr_parts; } + /* Swap the xpart */ const struct xpart txp = s->xparts[k]; s->xparts[k] = s->xparts[nr_parts]; s->xparts[nr_parts] = txp; + /* Swap the index */ const int t = ind[k]; ind[k] = ind[nr_parts]; ind[nr_parts] = t; @@ -544,20 +608,67 @@ void space_rebuild(struct space *s, int verbose) { } #endif + /* Move non-local sparts to the end of the list. */ + for (size_t k = 0; k < nr_sparts;) { + if (cells_top[sind[k]].nodeID != local_nodeID) { + nr_sparts -= 1; + /* Swap the particle */ + const struct spart tp = s->sparts[k]; + s->sparts[k] = s->sparts[nr_sparts]; + s->sparts[nr_sparts] = tp; + /* Swap the link with the gpart */ + if (s->sparts[k].gpart != NULL) { + s->sparts[k].gpart->id_or_neg_offset = -k; + } + if (s->sparts[nr_sparts].gpart != NULL) { + s->sparts[nr_sparts].gpart->id_or_neg_offset = -nr_sparts; + } + /* Swap the index */ + const int t = sind[k]; + sind[k] = sind[nr_sparts]; + sind[nr_sparts] = t; + } else { + /* Increment when not exchanging otherwise we need to retest "k".*/ + k++; + } + } + +#ifdef SWIFT_DEBUG_CHECKS + /* Check that all sparts are in the correct place (untested). */ + for (size_t k = 0; k < nr_sparts; k++) { + if (cells_top[sind[k]].nodeID != local_nodeID) { + error("Failed to move all non-local sparts to send list"); + } + } + for (size_t k = nr_sparts; k < s->nr_sparts; k++) { + if (cells_top[sind[k]].nodeID == local_nodeID) { + error("Failed to remove local sparts from send list"); + } + } +#endif + /* Move non-local gparts to the end of the list. */ for (size_t k = 0; k < nr_gparts;) { if (cells_top[gind[k]].nodeID != local_nodeID) { nr_gparts -= 1; + /* Swap the particle */ const struct gpart tp = s->gparts[k]; s->gparts[k] = s->gparts[nr_gparts]; s->gparts[nr_gparts] = tp; - if (s->gparts[k].id_or_neg_offset <= 0) { + /* Swap the link with part/spart */ + if (s->gparts[k].type == swift_type_gas) { s->parts[-s->gparts[k].id_or_neg_offset].gpart = &s->gparts[k]; + } else if (s->gparts[k].type == swift_type_star) { + s->sparts[-s->gparts[k].id_or_neg_offset].gpart = &s->gparts[k]; } - if (s->gparts[nr_gparts].id_or_neg_offset <= 0) { + if (s->gparts[nr_gparts].type == swift_type_gas) { s->parts[-s->gparts[nr_gparts].id_or_neg_offset].gpart = &s->gparts[nr_gparts]; + } else if (s->gparts[nr_gparts].type == swift_type_star) { + s->sparts[-s->gparts[nr_gparts].id_or_neg_offset].gpart = + &s->gparts[nr_gparts]; } + /* Swap the index */ const int t = gind[k]; gind[k] = gind[nr_gparts]; gind[nr_gparts] = t; @@ -585,14 +696,17 @@ void space_rebuild(struct space *s, int verbose) { the parts arrays. */ size_t nr_parts_exchanged = s->nr_parts - nr_parts; size_t nr_gparts_exchanged = s->nr_gparts - nr_gparts; + size_t nr_sparts_exchanged = s->nr_sparts - nr_sparts; engine_exchange_strays(s->e, nr_parts, &ind[nr_parts], &nr_parts_exchanged, - nr_gparts, &gind[nr_gparts], &nr_gparts_exchanged); + nr_gparts, &gind[nr_gparts], &nr_gparts_exchanged, + nr_sparts, &sind[nr_sparts], &nr_sparts_exchanged); /* Set the new particle counts. */ s->nr_parts = nr_parts + nr_parts_exchanged; s->nr_gparts = nr_gparts + nr_gparts_exchanged; + s->nr_sparts = nr_sparts + nr_sparts_exchanged; - /* Re-allocate the index array if needed.. */ + /* Re-allocate the index array for the parts if needed.. */ if (s->nr_parts + 1 > ind_size) { int *ind_new; if ((ind_new = (int *)malloc(sizeof(int) * (s->nr_parts + 1))) == NULL) @@ -602,10 +716,20 @@ void space_rebuild(struct space *s, int verbose) { ind = ind_new; } + /* Re-allocate the index array for the sparts if needed.. */ + if (s->nr_sparts + 1 > sind_size) { + int *sind_new; + if ((sind_new = (int *)malloc(sizeof(int) * (s->nr_sparts + 1))) == NULL) + error("Failed to allocate temporary s-particle indices."); + memcpy(sind_new, sind, sizeof(int) * nr_sparts); + free(sind); + sind = sind_new; + } + const int cdim[3] = {s->cdim[0], s->cdim[1], s->cdim[2]}; const double ih[3] = {s->iwidth[0], s->iwidth[1], s->iwidth[2]}; - /* Assign each particle to its cell. */ + /* Assign each received part to its cell. */ for (size_t k = nr_parts; k < s->nr_parts; k++) { const struct part *const p = &s->parts[k]; ind[k] = @@ -618,28 +742,81 @@ void space_rebuild(struct space *s, int verbose) { } nr_parts = s->nr_parts; + /* Assign each received spart to its cell. */ + for (size_t k = nr_sparts; k < s->nr_sparts; k++) { + const struct spart *const sp = &s->sparts[k]; + sind[k] = + cell_getid(cdim, sp->x[0] * ih[0], sp->x[1] * ih[1], sp->x[2] * ih[2]); +#ifdef SWIFT_DEBUG_CHECKS + if (cells_top[sind[k]].nodeID != local_nodeID) + error("Received s-part that does not belong to me (nodeID=%i).", + cells_top[sind[k]].nodeID); +#endif + } + nr_sparts = s->nr_sparts; + #endif /* WITH_MPI */ /* Sort the parts according to their cells. */ if (nr_parts > 0) space_parts_sort(s, ind, nr_parts, 0, s->nr_cells - 1, verbose); - /* Re-link the gparts. */ - if (nr_parts > 0 && nr_gparts > 0) part_relink_gparts(s->parts, nr_parts, 0); +#ifdef SWIFT_DEBUG_CHECKS + /* Verify that the part have been sorted correctly. */ + for (size_t k = 0; k < nr_parts; k++) { + const struct part *p = &s->parts[k]; + + /* New cell index */ + const int new_ind = + cell_getid(s->cdim, p->x[0] * s->iwidth[0], p->x[1] * s->iwidth[1], + p->x[2] * s->iwidth[2]); + + /* New cell of this part */ + const struct cell *c = &s->cells_top[new_ind]; + + if (ind[k] != new_ind) + error("part's new cell index not matching sorted index."); + + if (p->x[0] < c->loc[0] || p->x[0] > c->loc[0] + c->width[0] || + p->x[1] < c->loc[1] || p->x[1] > c->loc[1] + c->width[1] || + p->x[2] < c->loc[2] || p->x[2] > c->loc[2] + c->width[2]) + error("part not sorted into the right top-level cell!"); + } +#endif + + /* Sort the sparts according to their cells. */ + if (nr_sparts > 0) + space_sparts_sort(s, sind, nr_sparts, 0, s->nr_cells - 1, verbose); #ifdef SWIFT_DEBUG_CHECKS - /* Verify space_sort_struct. */ - for (size_t k = 1; k < nr_parts; k++) { - if (ind[k - 1] > ind[k]) { - error("Sort failed!"); - } else if (ind[k] != cell_getid(s->cdim, s->parts[k].x[0] * s->iwidth[0], - s->parts[k].x[1] * s->iwidth[1], - s->parts[k].x[2] * s->iwidth[2])) { - error("Incorrect indices!"); - } + /* Verify that the spart have been sorted correctly. */ + for (size_t k = 0; k < nr_sparts; k++) { + const struct spart *sp = &s->sparts[k]; + + /* New cell index */ + const int new_sind = + cell_getid(s->cdim, sp->x[0] * s->iwidth[0], sp->x[1] * s->iwidth[1], + sp->x[2] * s->iwidth[2]); + + /* New cell of this spart */ + const struct cell *c = &s->cells_top[new_sind]; + + if (sind[k] != new_sind) + error("spart's new cell index not matching sorted index."); + + if (sp->x[0] < c->loc[0] || sp->x[0] > c->loc[0] + c->width[0] || + sp->x[1] < c->loc[1] || sp->x[1] > c->loc[1] + c->width[1] || + sp->x[2] < c->loc[2] || sp->x[2] > c->loc[2] + c->width[2]) + error("spart not sorted into the right top-level cell!"); } #endif + /* Re-link the gparts to their (s-)particles. */ + if (nr_parts > 0 && nr_gparts > 0) + part_relink_gparts_to_parts(s->parts, nr_parts, 0); + if (nr_sparts > 0 && nr_gparts > 0) + part_relink_gparts_to_sparts(s->sparts, nr_sparts, 0); + /* Extract the cell counts from the sorted indices. */ size_t last_index = 0; ind[nr_parts] = s->nr_cells; // sentinel. @@ -650,12 +827,23 @@ void space_rebuild(struct space *s, int verbose) { } } + /* Extract the cell counts from the sorted indices. */ + size_t last_sindex = 0; + sind[nr_sparts] = s->nr_cells; // sentinel. + for (size_t k = 0; k < nr_sparts; k++) { + if (sind[k] < sind[k + 1]) { + cells_top[sind[k]].scount = k - last_sindex + 1; + last_sindex = k + 1; + } + } + /* We no longer need the indices as of here. */ free(ind); + free(sind); #ifdef WITH_MPI - /* Re-allocate the index array if needed.. */ + /* Re-allocate the index array for the gparts if needed.. */ if (s->nr_gparts + 1 > gind_size) { int *gind_new; if ((gind_new = (int *)malloc(sizeof(int) * (s->nr_gparts + 1))) == NULL) @@ -665,7 +853,7 @@ void space_rebuild(struct space *s, int verbose) { gind = gind_new; } - /* Assign each particle to its cell. */ + /* Assign each received gpart to its cell. */ for (size_t k = nr_gparts; k < s->nr_gparts; k++) { const struct gpart *const p = &s->gparts[k]; gind[k] = @@ -673,21 +861,48 @@ void space_rebuild(struct space *s, int verbose) { #ifdef SWIFT_DEBUG_CHECKS if (cells_top[gind[k]].nodeID != s->e->nodeID) - error("Received part that does not belong to me (nodeID=%i).", + error("Received g-part that does not belong to me (nodeID=%i).", cells_top[gind[k]].nodeID); #endif } nr_gparts = s->nr_gparts; -#endif +#endif /* WITH_MPI */ /* Sort the gparts according to their cells. */ if (nr_gparts > 0) space_gparts_sort(s, gind, nr_gparts, 0, s->nr_cells - 1, verbose); +#ifdef SWIFT_DEBUG_CHECKS + /* Verify that the gpart have been sorted correctly. */ + for (size_t k = 0; k < nr_gparts; k++) { + const struct gpart *gp = &s->gparts[k]; + + /* New cell index */ + const int new_gind = + cell_getid(s->cdim, gp->x[0] * s->iwidth[0], gp->x[1] * s->iwidth[1], + gp->x[2] * s->iwidth[2]); + + /* New cell of this gpart */ + const struct cell *c = &s->cells_top[new_gind]; + + if (gind[k] != new_gind) + error("gpart's new cell index not matching sorted index."); + + if (gp->x[0] < c->loc[0] || gp->x[0] > c->loc[0] + c->width[0] || + gp->x[1] < c->loc[1] || gp->x[1] > c->loc[1] + c->width[1] || + gp->x[2] < c->loc[2] || gp->x[2] > c->loc[2] + c->width[2]) + error("gpart not sorted into the right top-level cell!"); + } +#endif + /* Re-link the parts. */ if (nr_parts > 0 && nr_gparts > 0) - part_relink_parts(s->gparts, nr_gparts, s->parts); + part_relink_parts_to_gparts(s->gparts, nr_gparts, s->parts); + + /* Re-link the sparts. */ + if (nr_sparts > 0 && nr_gparts > 0) + part_relink_sparts_to_gparts(s->gparts, nr_gparts, s->sparts); /* Extract the cell counts from the sorted indices. */ size_t last_gindex = 0; @@ -704,26 +919,8 @@ void space_rebuild(struct space *s, int verbose) { #ifdef SWIFT_DEBUG_CHECKS /* Verify that the links are correct */ - for (size_t k = 0; k < nr_gparts; ++k) { - - if (s->gparts[k].id_or_neg_offset < 0) { - - const struct part *part = &s->parts[-s->gparts[k].id_or_neg_offset]; - - if (part->gpart != &s->gparts[k]) error("Linking problem !"); - - if (s->gparts[k].x[0] != part->x[0] || s->gparts[k].x[1] != part->x[1] || - s->gparts[k].x[2] != part->x[2]) - error("Linked particles are not at the same position !"); - } - } - for (size_t k = 0; k < nr_parts; ++k) { - - if (s->parts[k].gpart != NULL && - s->parts[k].gpart->id_or_neg_offset != -(ptrdiff_t)k) { - error("Linking problem !"); - } - } + part_verify_links(s->parts, s->gparts, s->sparts, nr_parts, nr_gparts, + nr_sparts, verbose); #endif /* Hook the cells up to the parts. */ @@ -731,15 +928,19 @@ void space_rebuild(struct space *s, int verbose) { struct part *finger = s->parts; struct xpart *xfinger = s->xparts; struct gpart *gfinger = s->gparts; + struct spart *sfinger = s->sparts; for (int k = 0; k < s->nr_cells; k++) { struct cell *restrict c = &cells_top[k]; - c->ti_old = ti_current; + c->ti_old = ti_old; + c->ti_old_multipole = ti_old; c->parts = finger; c->xparts = xfinger; c->gparts = gfinger; + c->sparts = sfinger; finger = &finger[c->count]; xfinger = &xfinger[c->count]; gfinger = &gfinger[c->gcount]; + sfinger = &sfinger[c->scount]; } // message( "hooking up cells took %.3f %s." , // clocks_from_ticks(getticks() - tic), clocks_getunit()); @@ -748,6 +949,13 @@ void space_rebuild(struct space *s, int verbose) { sure that the parts in each cell are ok. */ space_split(s, cells_top, s->nr_cells, verbose); +#ifdef SWIFT_DEBUG_CHECKS + /* Check that the multipole construction went OK */ + if (s->gravity) + for (int k = 0; k < s->nr_cells; k++) + cell_check_multipole(&s->cells_top[k], NULL); +#endif + if (verbose) message("took %.3f %s.", clocks_from_ticks(getticks() - tic), clocks_getunit()); @@ -911,8 +1119,58 @@ void space_gparts_get_cell_index_mapper(void *map_data, int nr_gparts, } /** - * @brief Computes the cell index of all the particles and update the cell - * count. + * @brief #threadpool mapper function to compute the s-particle cell indices. + * + * @param map_data Pointer towards the s-particles. + * @param nr_sparts The number of s-particles to treat. + * @param extra_data Pointers to the space and index list + */ +void space_sparts_get_cell_index_mapper(void *map_data, int nr_sparts, + void *extra_data) { + + /* Unpack the data */ + struct spart *restrict sparts = (struct spart *)map_data; + struct index_data *data = (struct index_data *)extra_data; + struct space *s = data->s; + int *const ind = data->ind + (ptrdiff_t)(sparts - s->sparts); + + /* Get some constants */ + const double dim_x = s->dim[0]; + const double dim_y = s->dim[1]; + const double dim_z = s->dim[2]; + const int cdim[3] = {s->cdim[0], s->cdim[1], s->cdim[2]}; + const double ih_x = s->iwidth[0]; + const double ih_y = s->iwidth[1]; + const double ih_z = s->iwidth[2]; + + for (int k = 0; k < nr_sparts; k++) { + + /* Get the particle */ + struct spart *restrict sp = &sparts[k]; + + const double old_pos_x = sp->x[0]; + const double old_pos_y = sp->x[1]; + const double old_pos_z = sp->x[2]; + + /* Put it back into the simulation volume */ + const double pos_x = box_wrap(old_pos_x, 0.0, dim_x); + const double pos_y = box_wrap(old_pos_y, 0.0, dim_y); + const double pos_z = box_wrap(old_pos_z, 0.0, dim_z); + + /* Get its cell index */ + const int index = + cell_getid(cdim, pos_x * ih_x, pos_y * ih_y, pos_z * ih_z); + ind[k] = index; + + /* Update the position */ + sp->x[0] = pos_x; + sp->x[1] = pos_y; + sp->x[2] = pos_z; + } +} + +/** + * @brief Computes the cell index of all the particles. * * @param s The #space. * @param ind The array of indices to fill. @@ -939,8 +1197,7 @@ void space_parts_get_cell_index(struct space *s, int *ind, struct cell *cells, } /** - * @brief Computes the cell index of all the g-particles and update the cell - * gcount. + * @brief Computes the cell index of all the g-particles. * * @param s The #space. * @param gind The array of indices to fill. @@ -966,6 +1223,33 @@ void space_gparts_get_cell_index(struct space *s, int *gind, struct cell *cells, clocks_getunit()); } +/** + * @brief Computes the cell index of all the s-particles. + * + * @param s The #space. + * @param sind The array of indices to fill. + * @param cells The array of #cell to update. + * @param verbose Are we talkative ? + */ +void space_sparts_get_cell_index(struct space *s, int *sind, struct cell *cells, + int verbose) { + + const ticks tic = getticks(); + + /* Pack the extra information */ + struct index_data data; + data.s = s; + data.cells = cells; + data.ind = sind; + + threadpool_map(&s->e->threadpool, space_sparts_get_cell_index_mapper, + s->sparts, s->nr_sparts, sizeof(struct spart), 1000, &data); + + if (verbose) + message("took %.3f %s.", clocks_from_ticks(getticks() - tic), + clocks_getunit()); +} + /** * @brief Sort the particles and condensed particles according to the given * indices. @@ -1077,18 +1361,207 @@ void space_parts_sort_mapper(void *map_data, int num_elements, #ifdef SWIFT_DEBUG_CHECKS /* Verify space_sort_struct. */ - for (int k = i; k <= jj; k++) - if (ind[k] > pivot) { - message("sorting failed at k=%i, ind[k]=%i, pivot=%i, i=%li, j=%li.", - k, ind[k], pivot, i, j); - error("Partition failed (<=pivot)."); + if (i != j) { + for (int k = i; k <= jj; k++) { + if (ind[k] > pivot) { + message( + "sorting failed at k=%i, ind[k]=%i, pivot=%i, i=%li, j=%li.", k, + ind[k], pivot, i, j); + error("Partition failed (<=pivot)."); + } + } + for (int k = jj + 1; k <= j; k++) { + if (ind[k] <= pivot) { + message( + "sorting failed at k=%i, ind[k]=%i, pivot=%i, i=%li, j=%li.", k, + ind[k], pivot, i, j); + error("Partition failed (>pivot)."); + } + } + } +#endif + + /* Split-off largest interval. */ + if (jj - i > j - jj + 1) { + + /* Recurse on the left? */ + if (jj > i && pivot > min) { + qid = atomic_inc(&sort_struct->last) % sort_struct->stack_size; + while (sort_struct->stack[qid].ready) + ; + sort_struct->stack[qid].i = i; + sort_struct->stack[qid].j = jj; + sort_struct->stack[qid].min = min; + sort_struct->stack[qid].max = pivot; + if (atomic_inc(&sort_struct->waiting) >= sort_struct->stack_size) + error("Qstack overflow."); + sort_struct->stack[qid].ready = 1; + } + + /* Recurse on the right? */ + if (jj + 1 < j && pivot + 1 < max) { + i = jj + 1; + min = pivot + 1; + } else + break; + + } else { + + /* Recurse on the right? */ + if (pivot + 1 < max) { + qid = atomic_inc(&sort_struct->last) % sort_struct->stack_size; + while (sort_struct->stack[qid].ready) + ; + sort_struct->stack[qid].i = jj + 1; + sort_struct->stack[qid].j = j; + sort_struct->stack[qid].min = pivot + 1; + sort_struct->stack[qid].max = max; + if (atomic_inc(&sort_struct->waiting) >= sort_struct->stack_size) + error("Qstack overflow."); + sort_struct->stack[qid].ready = 1; + } + + /* Recurse on the left? */ + if (jj > i && pivot > min) { + j = jj; + max = pivot; + } else + break; + } + + } /* loop over sub-intervals. */ + + atomic_dec(&sort_struct->waiting); + + } /* main loop. */ +} + +/** + * @brief Sort the s-particles according to the given indices. + * + * @param s The #space. + * @param ind The indices with respect to which the #spart are sorted. + * @param N The number of parts + * @param min Lowest index. + * @param max highest index. + * @param verbose Are we talkative ? + */ +void space_sparts_sort(struct space *s, int *ind, size_t N, int min, int max, + int verbose) { + + const ticks tic = getticks(); + + /* Populate a parallel_sort structure with the input data */ + struct parallel_sort sort_struct; + sort_struct.sparts = s->sparts; + sort_struct.ind = ind; + sort_struct.stack_size = 2 * (max - min + 1) + 10 + s->e->nr_threads; + if ((sort_struct.stack = + malloc(sizeof(struct qstack) * sort_struct.stack_size)) == NULL) + error("Failed to allocate sorting stack."); + for (unsigned int i = 0; i < sort_struct.stack_size; i++) + sort_struct.stack[i].ready = 0; + + /* Add the first interval. */ + sort_struct.stack[0].i = 0; + sort_struct.stack[0].j = N - 1; + sort_struct.stack[0].min = min; + sort_struct.stack[0].max = max; + sort_struct.stack[0].ready = 1; + sort_struct.first = 0; + sort_struct.last = 1; + sort_struct.waiting = 1; + + /* Launch the sorting tasks with a stride of zero such that the same + map data is passed to each thread. */ + threadpool_map(&s->e->threadpool, space_sparts_sort_mapper, &sort_struct, + s->e->threadpool.num_threads, 0, 1, NULL); + +#ifdef SWIFT_DEBUG_CHECKS + /* Verify space_sort_struct. */ + for (size_t i = 1; i < N; i++) + if (ind[i - 1] > ind[i]) + error("Sorting failed (ind[%zu]=%i,ind[%zu]=%i), min=%i, max=%i.", i - 1, + ind[i - 1], i, ind[i], min, max); + if (s->e->nodeID == 0 || verbose) message("Sorting succeeded."); +#endif + + /* Clean up. */ + free(sort_struct.stack); + + if (verbose) + message("took %.3f %s.", clocks_from_ticks(getticks() - tic), + clocks_getunit()); +} + +void space_sparts_sort_mapper(void *map_data, int num_elements, + void *extra_data) { + + /* Unpack the mapping data. */ + struct parallel_sort *sort_struct = (struct parallel_sort *)map_data; + + /* Pointers to the sorting data. */ + int *ind = sort_struct->ind; + struct spart *sparts = sort_struct->sparts; + + /* Main loop. */ + while (sort_struct->waiting) { + + /* Grab an interval off the queue. */ + int qid = atomic_inc(&sort_struct->first) % sort_struct->stack_size; + + /* Wait for the entry to be ready, or for the sorting do be done. */ + while (!sort_struct->stack[qid].ready) + if (!sort_struct->waiting) return; + + /* Get the stack entry. */ + ptrdiff_t i = sort_struct->stack[qid].i; + ptrdiff_t j = sort_struct->stack[qid].j; + int min = sort_struct->stack[qid].min; + int max = sort_struct->stack[qid].max; + sort_struct->stack[qid].ready = 0; + + /* Loop over sub-intervals. */ + while (1) { + + /* Bring beer. */ + const int pivot = (min + max) / 2; + /* message("Working on interval [%i,%i] with min=%i, max=%i, pivot=%i.", + i, j, min, max, pivot); */ + + /* One pass of QuickSort's partitioning. */ + ptrdiff_t ii = i; + ptrdiff_t jj = j; + while (ii < jj) { + while (ii <= j && ind[ii] <= pivot) ii++; + while (jj >= i && ind[jj] > pivot) jj--; + if (ii < jj) { + memswap(&ind[ii], &ind[jj], sizeof(int)); + memswap(&sparts[ii], &sparts[jj], sizeof(struct spart)); + } + } + +#ifdef SWIFT_DEBUG_CHECKS + /* Verify space_sort_struct. */ + if (i != j) { + for (int k = i; k <= jj; k++) { + if (ind[k] > pivot) { + message( + "sorting failed at k=%i, ind[k]=%i, pivot=%i, i=%li, j=%li " + "min=%i max=%i.", + k, ind[k], pivot, i, j, min, max); + error("Partition failed (<=pivot)."); + } } - for (int k = jj + 1; k <= j; k++) - if (ind[k] <= pivot) { - message("sorting failed at k=%i, ind[k]=%i, pivot=%i, i=%li, j=%li.", - k, ind[k], pivot, i, j); - error("Partition failed (>pivot)."); + for (int k = jj + 1; k <= j; k++) { + if (ind[k] <= pivot) { + message( + "sorting failed at k=%i, ind[k]=%i, pivot=%i, i=%li, j=%li.", k, + ind[k], pivot, i, j); + error("Partition failed (>pivot)."); + } } + } #endif /* Split-off largest interval. */ @@ -1253,18 +1726,24 @@ void space_gparts_sort_mapper(void *map_data, int num_elements, #ifdef SWIFT_DEBUG_CHECKS /* Verify space_sort_struct. */ - for (int k = i; k <= jj; k++) - if (ind[k] > pivot) { - message("sorting failed at k=%i, ind[k]=%i, pivot=%i, i=%li, j=%li.", - k, ind[k], pivot, i, j); - error("Partition failed (<=pivot)."); + if (i != j) { + for (int k = i; k <= jj; k++) { + if (ind[k] > pivot) { + message( + "sorting failed at k=%i, ind[k]=%i, pivot=%i, i=%li, j=%li.", k, + ind[k], pivot, i, j); + error("Partition failed (<=pivot)."); + } } - for (int k = jj + 1; k <= j; k++) - if (ind[k] <= pivot) { - message("sorting failed at k=%i, ind[k]=%i, pivot=%i, i=%li, j=%li.", - k, ind[k], pivot, i, j); - error("Partition failed (>pivot)."); + for (int k = jj + 1; k <= j; k++) { + if (ind[k] <= pivot) { + message( + "sorting failed at k=%i, ind[k]=%i, pivot=%i, i=%li, j=%li.", k, + ind[k], pivot, i, j); + error("Partition failed (>pivot)."); + } } + } #endif /* Split-off largest interval. */ @@ -1479,25 +1958,30 @@ void space_map_cells_pre(struct space *s, int full, * @param c The #cell to split recursively. * @param buff A buffer for particle sorting, should be of size at least * c->count or @c NULL. + * @param sbuff A buffer for particle sorting, should be of size at least + * c->scount or @c NULL. * @param gbuff A buffer for particle sorting, should be of size at least * c->gcount or @c NULL. */ void space_split_recursive(struct space *s, struct cell *c, - struct cell_buff *buff, struct cell_buff *gbuff) { + struct cell_buff *buff, struct cell_buff *sbuff, + struct cell_buff *gbuff) { const int count = c->count; const int gcount = c->gcount; + const int scount = c->scount; const int depth = c->depth; int maxdepth = 0; float h_max = 0.0f; - integertime_t ti_end_min = max_nr_timesteps, ti_end_max = 0; + integertime_t ti_end_min = max_nr_timesteps, ti_end_max = 0, ti_beg_max = 0; struct part *parts = c->parts; struct gpart *gparts = c->gparts; + struct spart *sparts = c->sparts; struct xpart *xparts = c->xparts; struct engine *e = s->e; /* If the buff is NULL, allocate it, and remember to free it. */ - const int allocate_buffer = (buff == NULL && gbuff == NULL); + const int allocate_buffer = (buff == NULL && gbuff == NULL && sbuff == NULL); if (allocate_buffer) { if (count > 0) { if (posix_memalign((void *)&buff, SWIFT_STRUCT_ALIGNMENT, @@ -1519,6 +2003,16 @@ void space_split_recursive(struct space *s, struct cell *c, gbuff[k].x[2] = gparts[k].x[2]; } } + if (scount > 0) { + if (posix_memalign((void *)&sbuff, SWIFT_STRUCT_ALIGNMENT, + sizeof(struct cell_buff) * scount) != 0) + error("Failed to allocate temporary indices."); + for (int k = 0; k < scount; k++) { + sbuff[k].x[0] = sparts[k].x[0]; + sbuff[k].x[1] = sparts[k].x[1]; + sbuff[k].x[2] = sparts[k].x[2]; + } + } } /* Check the depth. */ @@ -1533,7 +2027,8 @@ void space_split_recursive(struct space *s, struct cell *c, } /* Split or let it be? */ - if (count > space_splitsize || gcount > space_splitsize) { + if (count > space_splitsize || gcount > space_splitsize || + scount > space_splitsize) { /* No longer just a leaf. */ c->split = 1; @@ -1544,7 +2039,9 @@ void space_split_recursive(struct space *s, struct cell *c, struct cell *cp = c->progeny[k]; cp->count = 0; cp->gcount = 0; + cp->scount = 0; cp->ti_old = c->ti_old; + cp->ti_old_multipole = c->ti_old_multipole; cp->loc[0] = c->loc[0]; cp->loc[1] = c->loc[1]; cp->loc[2] = c->loc[2]; @@ -1566,25 +2063,67 @@ void space_split_recursive(struct space *s, struct cell *c, } /* Split the cell data. */ - cell_split(c, c->parts - s->parts, buff, gbuff); + cell_split(c, c->parts - s->parts, c->sparts - s->sparts, buff, sbuff, + gbuff); /* Remove any progeny with zero parts. */ - struct cell_buff *progeny_buff = buff, *progeny_gbuff = gbuff; - for (int k = 0; k < 8; k++) - if (c->progeny[k]->count == 0 && c->progeny[k]->gcount == 0) { + struct cell_buff *progeny_buff = buff, *progeny_gbuff = gbuff, + *progeny_sbuff = sbuff; + for (int k = 0; k < 8; k++) { + if (c->progeny[k]->count == 0 && c->progeny[k]->gcount == 0 && + c->progeny[k]->scount == 0) { space_recycle(s, c->progeny[k]); c->progeny[k] = NULL; } else { - space_split_recursive(s, c->progeny[k], progeny_buff, progeny_gbuff); + space_split_recursive(s, c->progeny[k], progeny_buff, progeny_sbuff, + progeny_gbuff); progeny_buff += c->progeny[k]->count; progeny_gbuff += c->progeny[k]->gcount; + progeny_sbuff += c->progeny[k]->scount; h_max = max(h_max, c->progeny[k]->h_max); ti_end_min = min(ti_end_min, c->progeny[k]->ti_end_min); ti_end_max = max(ti_end_max, c->progeny[k]->ti_end_max); + ti_beg_max = max(ti_beg_max, c->progeny[k]->ti_beg_max); if (c->progeny[k]->maxdepth > maxdepth) maxdepth = c->progeny[k]->maxdepth; } + } + + /* Deal with multipole */ + if (s->gravity) { + + /* Reset everything */ + gravity_reset(c->multipole); + /* Compute CoM of all progenies */ + double CoM[3] = {0., 0., 0.}; + double mass = 0.; + + for (int k = 0; k < 8; ++k) { + if (c->progeny[k] != NULL) { + const struct gravity_tensors *m = c->progeny[k]->multipole; + CoM[0] += m->CoM[0] * m->m_pole.mass; + CoM[1] += m->CoM[1] * m->m_pole.mass; + CoM[2] += m->CoM[2] * m->m_pole.mass; + mass += m->m_pole.mass; + } + } + c->multipole->CoM[0] = CoM[0] / mass; + c->multipole->CoM[1] = CoM[1] / mass; + c->multipole->CoM[2] = CoM[2] / mass; + + /* Now shift progeny multipoles and add them up */ + struct multipole temp; + for (int k = 0; k < 8; ++k) { + if (c->progeny[k] != NULL) { + const struct cell *cp = c->progeny[k]; + const struct multipole *m = &cp->multipole->m_pole; + gravity_M2M(&temp, m, c->multipole->CoM, cp->multipole->CoM, + s->periodic); + gravity_multipole_add(&c->multipole->m_pole, &temp); + } + } + } } /* Otherwise, collect the data for this cell. */ @@ -1595,42 +2134,70 @@ void space_split_recursive(struct space *s, struct cell *c, c->split = 0; maxdepth = c->depth; - /* Get dt_min/dt_max. */ + timebin_t time_bin_min = num_time_bins, time_bin_max = 0; + + /* parts: Get dt_min/dt_max and h_max. */ for (int k = 0; k < count; k++) { - struct part *p = &parts[k]; - struct xpart *xp = &xparts[k]; - const float h = p->h; - const integertime_t ti_end = - get_integer_time_end(e->ti_current, p->time_bin); - xp->x_diff[0] = 0.f; - xp->x_diff[1] = 0.f; - xp->x_diff[2] = 0.f; - if (h > h_max) h_max = h; - if (ti_end < ti_end_min) ti_end_min = ti_end; - if (ti_end > ti_end_max) ti_end_max = ti_end; +#ifdef SWIFT_DEBUG_CHECKS + if (parts[k].time_bin == time_bin_inhibited) + error("Inhibited particle present in space_split()"); +#endif + time_bin_min = min(time_bin_min, parts[k].time_bin); + time_bin_max = max(time_bin_max, parts[k].time_bin); + h_max = max(h_max, parts[k].h); } + /* parts: Reset x_diff */ + for (int k = 0; k < count; k++) { + xparts[k].x_diff[0] = 0.f; + xparts[k].x_diff[1] = 0.f; + xparts[k].x_diff[2] = 0.f; + } + /* gparts: Get dt_min/dt_max, reset x_diff. */ for (int k = 0; k < gcount; k++) { - struct gpart *gp = &gparts[k]; - const integertime_t ti_end = - get_integer_time_end(e->ti_current, gp->time_bin); - gp->x_diff[0] = 0.f; - gp->x_diff[1] = 0.f; - gp->x_diff[2] = 0.f; - if (ti_end < ti_end_min) ti_end_min = ti_end; - if (ti_end > ti_end_max) ti_end_max = ti_end; +#ifdef SWIFT_DEBUG_CHECKS + if (gparts[k].time_bin == time_bin_inhibited) + error("Inhibited s-particle present in space_split()"); +#endif + time_bin_min = min(time_bin_min, gparts[k].time_bin); + time_bin_max = max(time_bin_max, gparts[k].time_bin); + + gparts[k].x_diff[0] = 0.f; + gparts[k].x_diff[1] = 0.f; + gparts[k].x_diff[2] = 0.f; + } + /* sparts: Get dt_min/dt_max */ + for (int k = 0; k < scount; k++) { +#ifdef SWIFT_DEBUG_CHECKS + if (sparts[k].time_bin == time_bin_inhibited) + error("Inhibited g-particle present in space_split()"); +#endif + time_bin_min = min(time_bin_min, sparts[k].time_bin); + time_bin_max = max(time_bin_max, sparts[k].time_bin); } + + /* Convert into integer times */ + ti_end_min = get_integer_time_end(e->ti_current, time_bin_min); + ti_end_max = get_integer_time_end(e->ti_current, time_bin_max); + ti_beg_max = get_integer_time_begin(e->ti_current + 1, time_bin_max); + + /* Construct the multipole and the centre of mass*/ + if (s->gravity) gravity_P2M(c->multipole, c->gparts, c->gcount); } /* Set the values for this cell. */ c->h_max = h_max; c->ti_end_min = ti_end_min; c->ti_end_max = ti_end_max; + c->ti_beg_max = ti_beg_max; c->maxdepth = maxdepth; /* Set ownership according to the start of the parts array. */ if (s->nr_parts > 0) c->owner = ((c->parts - s->parts) % s->nr_parts) * s->nr_queues / s->nr_parts; + else if (s->nr_sparts > 0) + c->owner = + ((c->sparts - s->sparts) % s->nr_sparts) * s->nr_queues / s->nr_sparts; else if (s->nr_gparts > 0) c->owner = ((c->gparts - s->gparts) % s->nr_gparts) * s->nr_queues / s->nr_gparts; @@ -1641,6 +2208,7 @@ void space_split_recursive(struct space *s, struct cell *c, if (allocate_buffer) { if (buff != NULL) free(buff); if (gbuff != NULL) free(gbuff); + if (sbuff != NULL) free(sbuff); } } @@ -1660,7 +2228,7 @@ void space_split_mapper(void *map_data, int num_cells, void *extra_data) { for (int ind = 0; ind < num_cells; ind++) { struct cell *c = &cells_top[ind]; - space_split_recursive(s, c, NULL, NULL); + space_split_recursive(s, c, NULL, NULL, NULL); } #ifdef SWIFT_DEBUG_CHECKS @@ -1668,7 +2236,7 @@ void space_split_mapper(void *map_data, int num_cells, void *extra_data) { for (int ind = 0; ind < num_cells; ind++) { int depth = 0; if (!checkCellhdxmax(&cells_top[ind], &depth)) - error(" at cell depth %d", depth); + message(" at cell depth %d", depth); } #endif } @@ -1682,8 +2250,9 @@ void space_split_mapper(void *map_data, int num_cells, void *extra_data) { void space_recycle(struct space *s, struct cell *c) { /* Clear the cell. */ - if (lock_destroy(&c->lock) != 0 || lock_destroy(&c->glock) != 0) - error("Failed to destroy spinlock."); + if (lock_destroy(&c->lock) != 0 || lock_destroy(&c->glock) != 0 || + lock_destroy(&c->mlock) != 0 || lock_destroy(&c->slock) != 0) + error("Failed to destroy spinlocks."); /* Clear this cell's sort arrays. */ if (c->sort != NULL) free(c->sort); @@ -1691,6 +2260,12 @@ void space_recycle(struct space *s, struct cell *c) { /* Lock the space. */ lock_lock(&s->lock); + /* Hook the multipole back in the buffer */ + if (s->gravity) { + c->multipole->next = s->multipoles_sub; + s->multipoles_sub = c->multipole; + } + /* Hook this cell into the buffer. */ c->next = s->cells_sub; s->cells_sub = c; @@ -1704,22 +2279,32 @@ void space_recycle(struct space *s, struct cell *c) { * @brief Return a list of used cells to the buffer of unused sub-cells. * * @param s The #space. - * @param list_begin Pointer to the first #cell in the linked list of + * @param cell_list_begin Pointer to the first #cell in the linked list of * cells joined by their @c next pointers. - * @param list_end Pointer to the last #cell in the linked list of + * @param cell_list_end Pointer to the last #cell in the linked list of * cells joined by their @c next pointers. It is assumed that this * cell's @c next pointer is @c NULL. + * @param multipole_list_begin Pointer to the first #multipole in the linked + * list of + * multipoles joined by their @c next pointers. + * @param multipole_list_end Pointer to the last #multipole in the linked list + * of + * multipoles joined by their @c next pointers. It is assumed that this + * multipole's @c next pointer is @c NULL. */ -void space_recycle_list(struct space *s, struct cell *list_begin, - struct cell *list_end) { +void space_recycle_list(struct space *s, struct cell *cell_list_begin, + struct cell *cell_list_end, + struct gravity_tensors *multipole_list_begin, + struct gravity_tensors *multipole_list_end) { int count = 0; /* Clean up the list of cells. */ - for (struct cell *c = list_begin; c != NULL; c = c->next) { + for (struct cell *c = cell_list_begin; c != NULL; c = c->next) { /* Clear the cell. */ - if (lock_destroy(&c->lock) != 0 || lock_destroy(&c->glock) != 0) - error("Failed to destroy spinlock."); + if (lock_destroy(&c->lock) != 0 || lock_destroy(&c->glock) != 0 || + lock_destroy(&c->mlock) != 0 || lock_destroy(&c->slock) != 0) + error("Failed to destroy spinlocks."); /* Clear this cell's sort arrays. */ if (c->sort != NULL) free(c->sort); @@ -1731,11 +2316,17 @@ void space_recycle_list(struct space *s, struct cell *list_begin, /* Lock the space. */ lock_lock(&s->lock); - /* Hook this cell into the buffer. */ - list_end->next = s->cells_sub; - s->cells_sub = list_begin; + /* Hook the cells into the buffer. */ + cell_list_end->next = s->cells_sub; + s->cells_sub = cell_list_begin; s->tot_cells -= count; + /* Hook the multipoles into the buffer. */ + if (s->gravity) { + multipole_list_end->next = s->multipoles_sub; + s->multipoles_sub = multipole_list_begin; + } + /* Unlock the space. */ lock_unlock_blind(&s->lock); } @@ -1759,7 +2350,7 @@ void space_getcells(struct space *s, int nr_cells, struct cell **cells) { /* For each requested cell... */ for (int j = 0; j < nr_cells; j++) { - /* Is the buffer empty? */ + /* Is the cell buffer empty? */ if (s->cells_sub == NULL) { if (posix_memalign((void *)&s->cells_sub, cell_align, space_cellallocchunk * sizeof(struct cell)) != 0) @@ -1771,10 +2362,29 @@ void space_getcells(struct space *s, int nr_cells, struct cell **cells) { s->cells_sub[space_cellallocchunk - 1].next = NULL; } + /* Is the multipole buffer empty? */ + if (s->gravity && s->multipoles_sub == NULL) { + if (posix_memalign( + (void *)&s->multipoles_sub, multipole_align, + space_cellallocchunk * sizeof(struct gravity_tensors)) != 0) + error("Failed to allocate more multipoles."); + + /* Constructed a linked list */ + for (int k = 0; k < space_cellallocchunk - 1; k++) + s->multipoles_sub[k].next = &s->multipoles_sub[k + 1]; + s->multipoles_sub[space_cellallocchunk - 1].next = NULL; + } + /* Pick off the next cell. */ cells[j] = s->cells_sub; s->cells_sub = cells[j]->next; s->tot_cells += 1; + + /* Hook the multipole */ + if (s->gravity) { + cells[j]->multipole = s->multipoles_sub; + s->multipoles_sub = cells[j]->multipole->next; + } } /* Unlock the space. */ @@ -1782,9 +2392,12 @@ void space_getcells(struct space *s, int nr_cells, struct cell **cells) { /* Init some things in the cell we just got. */ for (int j = 0; j < nr_cells; j++) { + struct gravity_tensors *temp = cells[j]->multipole; bzero(cells[j], sizeof(struct cell)); + cells[j]->multipole = temp; cells[j]->nodeID = -1; - if (lock_init(&cells[j]->lock) != 0 || lock_init(&cells[j]->glock) != 0) + if (lock_init(&cells[j]->lock) != 0 || lock_init(&cells[j]->glock) != 0 || + lock_init(&cells[j]->mlock) != 0 || lock_init(&cells[j]->slock) != 0) error("Failed to initialize cell spinlocks."); } } @@ -1861,6 +2474,42 @@ void space_init_gparts(struct space *s) { #endif gravity_first_init_gpart(&gp[i]); + +#ifdef SWIFT_DEBUG_CHECKS + gp->ti_drift = 0; + gp->ti_kick = 0; +#endif + } +} + +/** + * @brief Initialises all the s-particles by setting them into a valid state + * + * Calls star_first_init_spart() on all the particles + */ +void space_init_sparts(struct space *s) { + + const size_t nr_sparts = s->nr_sparts; + struct spart *restrict sp = s->sparts; + + for (size_t i = 0; i < nr_sparts; ++i) { + +#ifdef HYDRO_DIMENSION_2D + sp[i].x[2] = 0.f; + sp[i].v[2] = 0.f; +#endif + +#ifdef HYDRO_DIMENSION_1D + sp[i].x[1] = sp[i].x[2] = 0.f; + sp[i].v[1] = sp[i].v[2] = 0.f; +#endif + + star_first_init_spart(&sp[i]); + +#ifdef SWIFT_DEBUG_CHECKS + sp->ti_drift = 0; + sp->ti_kick = 0; +#endif } } @@ -1872,9 +2521,12 @@ void space_init_gparts(struct space *s) { * @param dim Spatial dimensions of the domain. * @param parts Array of Gas particles. * @param gparts Array of Gravity particles. + * @param sparts Array of star particles. * @param Npart The number of Gas particles in the space. * @param Ngpart The number of Gravity particles in the space. + * @param Nspart The number of star particles in the space. * @param periodic flag whether the domain is periodic or not. + * @param replicate How many replications along each direction do we want ? * @param gravity flag whether we are doing gravity or not. * @param verbose Print messages to stdout or not. * @param dry_run If 1, just initialise stuff, don't do anything with the parts. @@ -1886,7 +2538,8 @@ void space_init_gparts(struct space *s) { */ void space_init(struct space *s, const struct swift_params *params, double dim[3], struct part *parts, struct gpart *gparts, - size_t Npart, size_t Ngpart, int periodic, int gravity, + struct spart *sparts, size_t Npart, size_t Ngpart, + size_t Nspart, int periodic, int replicate, int gravity, int verbose, int dry_run) { /* Clean-up everything */ @@ -1905,17 +2558,34 @@ void space_init(struct space *s, const struct swift_params *params, s->nr_gparts = Ngpart; s->size_gparts = Ngpart; s->gparts = gparts; + s->nr_sparts = Nspart; + s->size_sparts = Nspart; + s->sparts = sparts; s->nr_queues = 1; /* Temporary value until engine construction */ + /* Are we replicating the space ? */ + if (replicate < 1) + error("Value of 'InitialConditions:replicate' (%d) is too small", + replicate); + if (replicate > 1) { + space_replicate(s, replicate, verbose); + parts = s->parts; + gparts = s->gparts; + sparts = s->sparts; + Npart = s->nr_parts; + Ngpart = s->nr_gparts; + Nspart = s->nr_sparts; + } + /* Decide on the minimal top-level cell size */ - const double dmax = max(max(dim[0], dim[1]), dim[2]); + const double dmax = max(max(s->dim[0], s->dim[1]), s->dim[2]); int maxtcells = parser_get_opt_param_int(params, "Scheduler:max_top_level_cells", space_max_top_level_cells_default); s->cell_min = 0.99 * dmax / maxtcells; /* Check that it is big enough. */ - const double dmin = min(min(dim[0], dim[1]), dim[2]); + const double dmin = min(min(s->dim[0], s->dim[1]), s->dim[2]); int needtcells = 3 * dmax / dmin; if (maxtcells < needtcells) error( @@ -1964,6 +2634,11 @@ void space_init(struct space *s, const struct swift_params *params, gparts[k].x[1] += shift[1]; gparts[k].x[2] += shift[2]; } + for (size_t k = 0; k < Nspart; k++) { + sparts[k].x[0] += shift[0]; + sparts[k].x[1] += shift[1]; + sparts[k].x[2] += shift[2]; + } } if (!dry_run) { @@ -1972,13 +2647,13 @@ void space_init(struct space *s, const struct swift_params *params, if (periodic) { for (size_t k = 0; k < Npart; k++) for (int j = 0; j < 3; j++) { - while (parts[k].x[j] < 0) parts[k].x[j] += dim[j]; - while (parts[k].x[j] >= dim[j]) parts[k].x[j] -= dim[j]; + while (parts[k].x[j] < 0) parts[k].x[j] += s->dim[j]; + while (parts[k].x[j] >= s->dim[j]) parts[k].x[j] -= s->dim[j]; } } else { for (size_t k = 0; k < Npart; k++) for (int j = 0; j < 3; j++) - if (parts[k].x[j] < 0 || parts[k].x[j] >= dim[j]) + if (parts[k].x[j] < 0 || parts[k].x[j] >= s->dim[j]) error("Not all particles are within the specified domain."); } @@ -1986,18 +2661,32 @@ void space_init(struct space *s, const struct swift_params *params, if (periodic) { for (size_t k = 0; k < Ngpart; k++) for (int j = 0; j < 3; j++) { - while (gparts[k].x[j] < 0) gparts[k].x[j] += dim[j]; - while (gparts[k].x[j] >= dim[j]) gparts[k].x[j] -= dim[j]; + while (gparts[k].x[j] < 0) gparts[k].x[j] += s->dim[j]; + while (gparts[k].x[j] >= s->dim[j]) gparts[k].x[j] -= s->dim[j]; } } else { for (size_t k = 0; k < Ngpart; k++) for (int j = 0; j < 3; j++) - if (gparts[k].x[j] < 0 || gparts[k].x[j] >= dim[j]) + if (gparts[k].x[j] < 0 || gparts[k].x[j] >= s->dim[j]) error("Not all g-particles are within the specified domain."); } + + /* Same for the sparts */ + if (periodic) { + for (size_t k = 0; k < Nspart; k++) + for (int j = 0; j < 3; j++) { + while (sparts[k].x[j] < 0) sparts[k].x[j] += s->dim[j]; + while (sparts[k].x[j] >= s->dim[j]) sparts[k].x[j] -= s->dim[j]; + } + } else { + for (size_t k = 0; k < Nspart; k++) + for (int j = 0; j < 3; j++) + if (sparts[k].x[j] < 0 || sparts[k].x[j] >= s->dim[j]) + error("Not all s-particles are within the specified domain."); + } } - /* Allocate the extra parts array. */ + /* Allocate the extra parts array for the gas particles. */ if (Npart > 0) { if (posix_memalign((void *)&s->xparts, xpart_align, Npart * sizeof(struct xpart)) != 0) @@ -2009,14 +2698,136 @@ void space_init(struct space *s, const struct swift_params *params, space_init_parts(s); space_init_xparts(s); space_init_gparts(s); + space_init_sparts(s); /* Init the space lock. */ if (lock_init(&s->lock) != 0) error("Failed to create space spin-lock."); - /* Build the cells and the tasks. */ + /* Build the cells recursively. */ if (!dry_run) space_regrid(s, verbose); } +/** + * @brief Replicate the content of a space along each axis. + * + * Should only be called during initialisation. + * + * @param s The #space to replicate. + * @param replicate The number of copies along each axis. + * @param verbose Are we talkative ? + */ +void space_replicate(struct space *s, int replicate, int verbose) { + + if (replicate < 1) error("Invalid replicate value: %d", replicate); + + message("Replicating space %d times along each axis.", replicate); + + const int factor = replicate * replicate * replicate; + + /* Store the current values */ + const size_t nr_parts = s->nr_parts; + const size_t nr_gparts = s->nr_gparts; + const size_t nr_sparts = s->nr_sparts; + const size_t nr_dm = nr_gparts - nr_parts - nr_sparts; + + s->size_parts = s->nr_parts = nr_parts * factor; + s->size_gparts = s->nr_gparts = nr_gparts * factor; + s->size_sparts = s->nr_sparts = nr_sparts * factor; + + /* Allocate space for new particles */ + struct part *parts = NULL; + struct gpart *gparts = NULL; + struct spart *sparts = NULL; + + if (posix_memalign((void *)&parts, part_align, + s->nr_parts * sizeof(struct part)) != 0) + error("Failed to allocate new part array."); + + if (posix_memalign((void *)&gparts, gpart_align, + s->nr_gparts * sizeof(struct gpart)) != 0) + error("Failed to allocate new gpart array."); + + if (posix_memalign((void *)&sparts, spart_align, + s->nr_sparts * sizeof(struct spart)) != 0) + error("Failed to allocate new spart array."); + + /* Replicate everything */ + for (int i = 0; i < replicate; ++i) { + for (int j = 0; j < replicate; ++j) { + for (int k = 0; k < replicate; ++k) { + const size_t offset = i * replicate * replicate + j * replicate + k; + + /* First copy the data */ + memcpy(parts + offset * nr_parts, s->parts, + nr_parts * sizeof(struct part)); + memcpy(sparts + offset * nr_sparts, s->sparts, + nr_sparts * sizeof(struct spart)); + memcpy(gparts + offset * nr_gparts, s->gparts, + nr_gparts * sizeof(struct gpart)); + + /* Shift the positions */ + const double shift[3] = {i * s->dim[0], j * s->dim[1], k * s->dim[2]}; + + for (size_t n = offset * nr_parts; n < (offset + 1) * nr_parts; ++n) { + parts[n].x[0] += shift[0]; + parts[n].x[1] += shift[1]; + parts[n].x[2] += shift[2]; + } + for (size_t n = offset * nr_gparts; n < (offset + 1) * nr_gparts; ++n) { + gparts[n].x[0] += shift[0]; + gparts[n].x[1] += shift[1]; + gparts[n].x[2] += shift[2]; + } + for (size_t n = offset * nr_sparts; n < (offset + 1) * nr_sparts; ++n) { + sparts[n].x[0] += shift[0]; + sparts[n].x[1] += shift[1]; + sparts[n].x[2] += shift[2]; + } + + /* Set the correct links (recall gpart are sorted by type at start-up): + first DM (unassociated gpart), then gas, then stars */ + if (nr_parts > 0 && nr_gparts > 0) { + const size_t offset_part = offset * nr_parts; + const size_t offset_gpart = offset * nr_gparts + nr_dm; + + for (size_t n = 0; n < nr_parts; ++n) { + parts[offset_part + n].gpart = &gparts[offset_gpart + n]; + gparts[offset_gpart + n].id_or_neg_offset = -(offset_part + n); + } + } + if (nr_sparts > 0 && nr_gparts > 0) { + const size_t offset_spart = offset * nr_sparts; + const size_t offset_gpart = offset * nr_gparts + nr_dm + nr_parts; + + for (size_t n = 0; n < nr_sparts; ++n) { + sparts[offset_spart + n].gpart = &gparts[offset_gpart + n]; + gparts[offset_gpart + n].id_or_neg_offset = -(offset_spart + n); + } + } + } + } + } + + /* Replace the content of the space */ + free(s->parts); + free(s->gparts); + free(s->sparts); + s->parts = parts; + s->gparts = gparts; + s->sparts = sparts; + + /* Finally, update the domain size */ + s->dim[0] *= replicate; + s->dim[1] *= replicate; + s->dim[2] *= replicate; + +#ifdef SWIFT_DEBUG_CHECKS + /* Verify that everything is correct */ + part_verify_links(s->parts, s->gparts, s->sparts, s->nr_parts, s->nr_gparts, + s->nr_sparts, verbose); +#endif +} + /** * @brief Cleans-up all the cell links in the space * @@ -2031,29 +2842,53 @@ void space_link_cleanup(struct space *s) { } /** - * @brief Checks that all cells have been drifted to the current point in time + * @brief Checks that all cells have been drifted to a given point in time * - * Expensive function. Should only be used for debugging purposes. + * Should only be used for debugging purposes. * * @param s The #space to check. - * @param ti_current The (integer) time. + * @param ti_drift The (integer) time. */ -void space_check_drift_point(struct space *s, integertime_t ti_current) { - +void space_check_drift_point(struct space *s, integertime_t ti_drift) { +#ifdef SWIFT_DEBUG_CHECKS /* Recursively check all cells */ - space_map_cells_pre(s, 1, cell_check_drift_point, &ti_current); + space_map_cells_pre(s, 1, cell_check_drift_point, &ti_drift); +#else + error("Calling debugging code without debugging flag activated."); +#endif } /** * @brief Checks that all particles and local cells have a non-zero time-step. + * + * Should only be used for debugging purposes. + * + * @param s The #space to check. */ void space_check_timesteps(struct space *s) { #ifdef SWIFT_DEBUG_CHECKS - for (int i = 0; i < s->nr_cells; ++i) { cell_check_timesteps(&s->cells_top[i]); } +#else + error("Calling debugging code without debugging flag activated."); +#endif +} +/** + * @brief Resets all the individual cell task counters to 0. + * + * Should only be used for debugging purposes. + * + * @param s The #space to reset. + */ +void space_reset_task_counters(struct space *s) { +#ifdef SWIFT_DEBUG_CHECKS + for (int i = 0; i < s->nr_cells; ++i) { + cell_reset_task_counters(&s->cells_top[i]); + } +#else + error("Calling debugging code without debugging flag activated."); #endif } @@ -2064,7 +2899,9 @@ void space_clean(struct space *s) { for (int i = 0; i < s->nr_cells; ++i) cell_clean(&s->cells_top[i]); free(s->cells_top); + free(s->multipoles_top); free(s->parts); free(s->xparts); free(s->gparts); + free(s->sparts); } diff --git a/src/space.h b/src/space.h index 3513c39e624570e8f1ad31ee711548bbec1a036d..d6972533b74a935ccc630e287c2d3119ce857e27 100644 --- a/src/space.h +++ b/src/space.h @@ -72,6 +72,9 @@ struct space { /*! Are we doing gravity? */ int gravity; + /*! Total mass in the system */ + double total_mass; + /*! Width of the top-level cells. */ double width[3]; @@ -102,12 +105,21 @@ struct space { /*! Buffer of unused cells for the sub-cells. */ struct cell *cells_sub; + /*! The multipoles associated with the top-level (level 0) cells */ + struct gravity_tensors *multipoles_top; + + /*! Buffer of unused multipoles for the sub-cells. */ + struct gravity_tensors *multipoles_sub; + /*! The total number of parts in the space. */ size_t nr_parts, size_parts; /*! The total number of g-parts in the space. */ size_t nr_gparts, size_gparts; + /*! The total number of g-parts in the space. */ + size_t nr_sparts, size_sparts; + /*! The particle data (cells have pointers to this). */ struct part *parts; @@ -117,6 +129,9 @@ struct space { /*! The g-particle data (cells have pointers to this). */ struct gpart *gparts; + /*! The s-particle data (cells have pointers to this). */ + struct spart *sparts; + /*! General-purpose lock for this space. */ swift_lock_type lock; @@ -139,6 +154,10 @@ struct space { struct gpart *gparts_foreign; size_t nr_gparts_foreign, size_gparts_foreign; + /*! Buffers for g-parts that we will receive from foreign cells. */ + struct spart *sparts_foreign; + size_t nr_sparts_foreign, size_sparts_foreign; + #endif }; @@ -147,12 +166,15 @@ void space_parts_sort(struct space *s, int *ind, size_t N, int min, int max, int verbose); void space_gparts_sort(struct space *s, int *ind, size_t N, int min, int max, int verbose); +void space_sparts_sort(struct space *s, int *ind, size_t N, int min, int max, + int verbose); void space_getcells(struct space *s, int nr_cells, struct cell **cells); int space_getsid(struct space *s, struct cell **ci, struct cell **cj, double *shift); void space_init(struct space *s, const struct swift_params *params, double dim[3], struct part *parts, struct gpart *gparts, - size_t Npart, size_t Ngpart, int periodic, int gravity, + struct spart *sparts, size_t Npart, size_t Ngpart, + size_t Nspart, int periodic, int replicate, int gravity, int verbose, int dry_run); void space_sanitize(struct space *s); void space_map_cells_pre(struct space *s, int full, @@ -169,10 +191,14 @@ void space_parts_sort_mapper(void *map_data, int num_elements, void *extra_data); void space_gparts_sort_mapper(void *map_data, int num_elements, void *extra_data); +void space_sparts_sort_mapper(void *map_data, int num_elements, + void *extra_data); void space_rebuild(struct space *s, int verbose); void space_recycle(struct space *s, struct cell *c); -void space_recycle_list(struct space *s, struct cell *list_begin, - struct cell *list_end); +void space_recycle_list(struct space *s, struct cell *cell_list_begin, + struct cell *cell_list_end, + struct gravity_tensors *multipole_list_begin, + struct gravity_tensors *multipole_list_end); void space_split(struct space *s, struct cell *cells, int nr_cells, int verbose); void space_split_mapper(void *map_data, int num_elements, void *extra_data); @@ -180,13 +206,19 @@ void space_parts_get_cell_index(struct space *s, int *ind, struct cell *cells, int verbose); void space_gparts_get_cell_index(struct space *s, int *gind, struct cell *cells, int verbose); +void space_sparts_get_cell_index(struct space *s, int *sind, struct cell *cells, + int verbose); void space_do_parts_sort(); void space_do_gparts_sort(); +void space_do_sparts_sort(); void space_init_parts(struct space *s); void space_init_gparts(struct space *s); +void space_init_sparts(struct space *s); void space_link_cleanup(struct space *s); -void space_check_drift_point(struct space *s, integertime_t ti_current); +void space_check_drift_point(struct space *s, integertime_t ti_drift); void space_check_timesteps(struct space *s); +void space_replicate(struct space *s, int replicate, int verbose); +void space_reset_task_counters(struct space *s); void space_clean(struct space *s); #endif /* SWIFT_SPACE_H */ diff --git a/src/stars.h b/src/stars.h new file mode 100644 index 0000000000000000000000000000000000000000..ade47ff57298c13bf205e991548945576a802293 --- /dev/null +++ b/src/stars.h @@ -0,0 +1,30 @@ +/******************************************************************************* + * This file is part of SWIFT. + * Coypright (c) 2016 Matthieu Schaller (matthieu.schaller@durham.ac.uk) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + ******************************************************************************/ +#ifndef SWIFT_STAR_H +#define SWIFT_STAR_H + +/* Config parameters. */ +#include "../config.h" + +/* So far only one model here */ +/* Straight-forward import */ +#include "./stars/Default/star.h" +#include "./stars/Default/star_iact.h" + +#endif diff --git a/src/stars/Default/star.h b/src/stars/Default/star.h new file mode 100644 index 0000000000000000000000000000000000000000..61ae4aeb5c51e18e39c3f4c6855d7c6ddfe05abb --- /dev/null +++ b/src/stars/Default/star.h @@ -0,0 +1,86 @@ +/******************************************************************************* + * This file is part of SWIFT. + * Coypright (c) 2016 Matthieu Schaller (matthieu.schaller@durham.ac.uk) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + ******************************************************************************/ +#ifndef SWIFT_DEFAULT_STAR_H +#define SWIFT_DEFAULT_STAR_H + +#include <float.h> +#include "minmax.h" + +/** + * @brief Computes the gravity time-step of a given star particle. + * + * @param sp Pointer to the s-particle data. + */ +__attribute__((always_inline)) INLINE static float star_compute_timestep( + const struct spart* const sp) { + + return FLT_MAX; +} + +/** + * @brief Initialises the s-particles for the first time + * + * This function is called only once just after the ICs have been + * read in to do some conversions. + * + * @param sp The particle to act upon + */ +__attribute__((always_inline)) INLINE static void star_first_init_spart( + struct spart* sp) { + + sp->time_bin = 0; +} + +/** + * @brief Prepares a s-particle for its interactions + * + * @param sp The particle to act upon + */ +__attribute__((always_inline)) INLINE static void star_init_spart( + struct spart* sp) {} + +/** + * @brief Sets the values to be predicted in the drifts to their values at a + * kick time + * + * @param sp The particle. + */ +__attribute__((always_inline)) INLINE static void star_reset_predicted_values( + struct spart* restrict sp) {} + +/** + * @brief Finishes the calculation of (non-gravity) forces acting on stars + * + * Multiplies the forces and accelerations by the appropiate constants + * + * @param sp The particle to act upon + */ +__attribute__((always_inline)) INLINE static void star_end_force( + struct spart* sp) {} + +/** + * @brief Kick the additional variables + * + * @param sp The particle to act upon + * @param dt The time-step for this kick + */ +__attribute__((always_inline)) INLINE static void star_kick_extra( + struct spart* sp, float dt) {} + +#endif /* SWIFT_DEFAULT_STAR_H */ diff --git a/src/stars/Default/star_debug.h b/src/stars/Default/star_debug.h new file mode 100644 index 0000000000000000000000000000000000000000..d940afac2eb67c97481f48a4bda6fa56085166d5 --- /dev/null +++ b/src/stars/Default/star_debug.h @@ -0,0 +1,31 @@ +/******************************************************************************* + * This file is part of SWIFT. + * Coypright (c) 2016 Matthieu Schaller (matthieu.schaller@durham.ac.uk) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + ******************************************************************************/ +#ifndef SWIFT_DEFAULT_STAR_DEBUG_H +#define SWIFT_DEFAULT_STAR_DEBUG_H + +__attribute__((always_inline)) INLINE static void star_debug_particle( + const struct spart* p) { + printf( + "x=[%.3e,%.3e,%.3e], " + "v_full=[%.3e,%.3e,%.3e] p->mass=%.3e \n t_begin=%d, t_end=%d\n", + p->x[0], p->x[1], p->x[2], p->v_full[0], p->v_full[1], p->v_full[2], + p->mass, p->ti_begin, p->ti_end); +} + +#endif /* SWIFT_DEFAULT_STAR_DEBUG_H */ diff --git a/src/stars/Default/star_iact.h b/src/stars/Default/star_iact.h new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/stars/Default/star_io.h b/src/stars/Default/star_io.h new file mode 100644 index 0000000000000000000000000000000000000000..96bbdce6d83dc241d05e7dd1754f476dc0b8e5f9 --- /dev/null +++ b/src/stars/Default/star_io.h @@ -0,0 +1,72 @@ +/******************************************************************************* + * This file is part of SWIFT. + * Coypright (c) 2016 Matthieu Schaller (matthieu.schaller@durham.ac.uk) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + ******************************************************************************/ +#ifndef SWIFT_DEFAULT_STAR_IO_H +#define SWIFT_DEFAULT_STAR_IO_H + +#include "io_properties.h" + +/** + * @brief Specifies which s-particle fields to read from a dataset + * + * @param sparts The s-particle array. + * @param list The list of i/o properties to read. + * @param num_fields The number of i/o fields to read. + */ +void star_read_particles(struct spart* sparts, struct io_props* list, + int* num_fields) { + + /* Say how much we want to read */ + *num_fields = 4; + + /* List what we want to read */ + list[0] = io_make_input_field("Coordinates", DOUBLE, 3, COMPULSORY, + UNIT_CONV_LENGTH, sparts, x); + list[1] = io_make_input_field("Velocities", FLOAT, 3, COMPULSORY, + UNIT_CONV_SPEED, sparts, v); + list[2] = io_make_input_field("Masses", FLOAT, 1, COMPULSORY, UNIT_CONV_MASS, + sparts, mass); + list[3] = io_make_input_field("ParticleIDs", LONGLONG, 1, COMPULSORY, + UNIT_CONV_NO_UNITS, sparts, id); +} + +/** + * @brief Specifies which s-particle fields to write to a dataset + * + * @param sparts The s-particle array. + * @param list The list of i/o properties to write. + * @param num_fields The number of i/o fields to write. + */ +void star_write_particles(struct spart* sparts, struct io_props* list, + int* num_fields) { + + /* Say how much we want to read */ + *num_fields = 4; + + /* List what we want to read */ + list[0] = io_make_output_field("Coordinates", DOUBLE, 3, UNIT_CONV_LENGTH, + sparts, x); + list[1] = + io_make_output_field("Velocities", FLOAT, 3, UNIT_CONV_SPEED, sparts, v); + list[2] = + io_make_output_field("Masses", FLOAT, 1, UNIT_CONV_MASS, sparts, mass); + list[3] = io_make_output_field("ParticleIDs", LONGLONG, 1, UNIT_CONV_NO_UNITS, + sparts, id); +} + +#endif /* SWIFT_DEFAULT_STAR_IO_H */ diff --git a/src/stars/Default/star_part.h b/src/stars/Default/star_part.h new file mode 100644 index 0000000000000000000000000000000000000000..68dd4869c257e35b3be7dc21f36e6dcdb725dc17 --- /dev/null +++ b/src/stars/Default/star_part.h @@ -0,0 +1,62 @@ +/******************************************************************************* + * This file is part of SWIFT. + * Copyright (c) 2016 Matthieu Schaller (matthieu.schaller@durham.ac.uk) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + ******************************************************************************/ +#ifndef SWIFT_DEFAULT_STAR_PART_H +#define SWIFT_DEFAULT_STAR_PART_H + +/* Some standard headers. */ +#include <stdlib.h> + +/** + * @brief Particle fields for the star particles. + * + * All quantities related to gravity are stored in the associate #gpart. + */ +struct spart { + + /*! Particle ID. */ + long long id; + + /*! Pointer to corresponding gravity part. */ + struct gpart* gpart; + + /*! Particle position. */ + double x[3]; + + /*! Particle velocity. */ + float v[3]; + + /*! Star mass */ + float mass; + + /*! Particle time bin */ + timebin_t time_bin; + +#ifdef SWIFT_DEBUG_CHECKS + + /* Time of the last drift */ + integertime_t ti_drift; + + /* Time of the last kick */ + integertime_t ti_kick; + +#endif + +} SWIFT_STRUCT_ALIGN; + +#endif /* SWIFT_DEFAULT_STAR_PART_H */ diff --git a/src/stars_io.h b/src/stars_io.h new file mode 100644 index 0000000000000000000000000000000000000000..18a13ec19163008f1c8e9f64cf544ddf812db655 --- /dev/null +++ b/src/stars_io.h @@ -0,0 +1,26 @@ +/******************************************************************************* + * This file is part of SWIFT. + * Coypright (c) 2016 Matthieu Schaller (matthieu.schaller@durham.ac.uk) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + ******************************************************************************/ +#ifndef SWIFT_STAR_IO_H +#define SWIFT_STAR_IO_H + +#include "./const.h" + +#include "./stars/Default/star_io.h" + +#endif /* SWIFT_STAR_IO_H */ diff --git a/src/swift.h b/src/swift.h index c08a4f3209d9eea0fe02ad9112179a0ed7ccae1e..7f1b19b6066c2d55df1cb9101172ae94c9085583 100644 --- a/src/swift.h +++ b/src/swift.h @@ -35,6 +35,7 @@ #include "engine.h" #include "error.h" #include "gravity.h" +#include "gravity_properties.h" #include "hydro.h" #include "hydro_properties.h" #include "lock.h" diff --git a/src/task.c b/src/task.c index 0a078b40c3914419cef294b63a527b65d9bea077..dfc3dc538c75297cbe7e79b7d64c1b7e13a015dc 100644 --- a/src/task.c +++ b/src/task.c @@ -47,16 +47,32 @@ #include "lock.h" /* Task type names. */ -const char *taskID_names[task_type_count] = { - "none", "sort", "self", "pair", "sub_self", - "sub_pair", "init", "ghost", "extra_ghost", "drift", - "kick1", "kick2", "timestep", "send", "recv", - "grav_gather_m", "grav_fft", "grav_mm", "grav_up", "cooling", - "sourceterms"}; - +const char *taskID_names[task_type_count] = {"none", + "sort", + "self", + "pair", + "sub_self", + "sub_pair", + "init", + "ghost", + "extra_ghost", + "drift", + "kick1", + "kick2", + "timestep", + "send", + "recv", + "grav_top_level", + "grav_long_range", + "grav_mm", + "grav_down", + "cooling", + "sourceterms"}; + +/* Sub-task type names. */ const char *subtaskID_names[task_subtype_count] = { - "none", "density", "gradient", "force", "grav", - "external_grav", "tend", "xv", "rho", "gpart"}; + "none", "density", "gradient", "force", "grav", "external_grav", + "tend", "xv", "rho", "gpart", "multipole", "spart"}; /** * @brief Computes the overlap between the parts array of two given cells. @@ -165,20 +181,22 @@ __attribute__((always_inline)) INLINE static enum task_actions task_acts_on( error("Task without particles"); break; - case task_type_grav_gather_m: - case task_type_grav_fft: + case task_type_grav_top_level: + case task_type_grav_long_range: case task_type_grav_mm: - case task_type_grav_up: return task_action_multipole; break; + case task_type_grav_down: + return task_action_gpart; + default: error("Unknown task_action for task"); return task_action_none; break; } - /* Silence compile warnings */ + /* Silence compiler warnings */ error("Unknown task_action for task"); return task_action_none; } @@ -265,6 +283,10 @@ void task_unlock(struct task *t) { /* Act based on task type. */ switch (type) { + case task_type_init: + case task_type_kick1: + case task_type_kick2: + case task_type_timestep: case task_type_drift: cell_unlocktree(ci); cell_gunlocktree(ci); @@ -337,6 +359,10 @@ int task_lock(struct task *t) { #endif break; + case task_type_init: + case task_type_kick1: + case task_type_kick2: + case task_type_timestep: case task_type_drift: if (ci->hold || ci->ghold) return 0; if (cell_locktree(ci) != 0) return 0; diff --git a/src/task.h b/src/task.h index 57fe4dcf506ba9dee47d8bc60931b49ab05e8a38..024ff1ec36a9d49acd6a8fe7b55f8c4811f9b780 100644 --- a/src/task.h +++ b/src/task.h @@ -51,10 +51,10 @@ enum task_types { task_type_timestep, task_type_send, task_type_recv, - task_type_grav_gather_m, - task_type_grav_fft, + task_type_grav_top_level, + task_type_grav_long_range, task_type_grav_mm, - task_type_grav_up, + task_type_grav_down, task_type_cooling, task_type_sourceterms, task_type_count @@ -74,6 +74,8 @@ enum task_subtypes { task_subtype_xv, task_subtype_rho, task_subtype_gpart, + task_subtype_multipole, + task_subtype_spart, task_subtype_count } __attribute__((packed)); diff --git a/src/timeline.h b/src/timeline.h index c73b2432b219a8ab0254d21c59102841557a57b9..0f38ff3e9421c122b61caf74290c170b62b2dd92 100644 --- a/src/timeline.h +++ b/src/timeline.h @@ -37,6 +37,9 @@ typedef char timebin_t; /*! The maximal number of timesteps in a simulation */ #define max_nr_timesteps (1LL << (num_time_bins + 1)) +/*! Fictious time-bin to hold inhibited particles */ +#define time_bin_inhibited (num_time_bins + 2) + /** * @brief Returns the integer time interval corresponding to a time bin * diff --git a/src/timers.h b/src/timers.h index 1e7965f3fdda5a7d7aa9213a3c36abdc783efccb..4cb4d7e0ba60003ba4caefffe257c929c59a8d9e 100644 --- a/src/timers.h +++ b/src/timers.h @@ -46,6 +46,7 @@ enum { timer_dopair_gradient, timer_dopair_force, timer_dopair_grav_pm, + timer_dopair_grav_mm, timer_dopair_grav_pp, timer_dograv_external, timer_dosource, @@ -62,6 +63,7 @@ enum { timer_do_extra_ghost, timer_dorecv_part, timer_dorecv_gpart, + timer_dorecv_spart, timer_gettask, timer_qget, timer_qsteal, diff --git a/src/timestep.h b/src/timestep.h index b3ef96493772e7e93853ad28a0536e4f8448dceb..99ca1b4f90cc7b8894d4dfe0e0d2670da8f01d65 100644 --- a/src/timestep.h +++ b/src/timestep.h @@ -78,7 +78,8 @@ __attribute__((always_inline)) INLINE static integertime_t get_gpart_timestep( e->physical_constants, gp)); if (e->policy & engine_policy_self_gravity) - new_dt = min(new_dt, gravity_compute_timestep_self(gp)); + new_dt = + min(new_dt, gravity_compute_timestep_self(gp, e->gravity_properties)); /* Limit timestep within the allowed range */ new_dt = min(new_dt, e->dt_max); @@ -109,7 +110,7 @@ __attribute__((always_inline)) INLINE static integertime_t get_part_timestep( float new_dt_cooling = FLT_MAX; if (e->policy & engine_policy_cooling) new_dt_cooling = cooling_timestep(e->cooling_func, e->physical_constants, - e->internalUnits, p); + e->internal_units, p); /* Compute the next timestep (gravity condition) */ float new_dt_grav = FLT_MAX; @@ -121,7 +122,8 @@ __attribute__((always_inline)) INLINE static integertime_t get_part_timestep( e->physical_constants, p->gpart)); if (e->policy & engine_policy_self_gravity) - new_dt_grav = min(new_dt_grav, gravity_compute_timestep_self(p->gpart)); + new_dt_grav = min(new_dt_grav, gravity_compute_timestep_self( + p->gpart, e->gravity_properties)); } /* Final time-step is minimum of hydro and gravity */ @@ -146,4 +148,35 @@ __attribute__((always_inline)) INLINE static integertime_t get_part_timestep( return new_dti; } +/** + * @brief Compute the new (integer) time-step of a given #spart + * + * @param sp The #spart. + * @param e The #engine (used to get some constants). + */ +__attribute__((always_inline)) INLINE static integertime_t get_spart_timestep( + const struct spart *restrict sp, const struct engine *restrict e) { + + float new_dt = star_compute_timestep(sp); + + if (e->policy & engine_policy_external_gravity) + new_dt = min(new_dt, + external_gravity_timestep(e->time, e->external_potential, + e->physical_constants, sp->gpart)); + + if (e->policy & engine_policy_self_gravity) + new_dt = min(new_dt, gravity_compute_timestep_self(sp->gpart, + e->gravity_properties)); + + /* Limit timestep within the allowed range */ + new_dt = min(new_dt, e->dt_max); + new_dt = max(new_dt, e->dt_min); + + /* Convert to integer time */ + const integertime_t new_dti = make_integer_timestep( + new_dt, sp->time_bin, e->ti_current, e->timeBase_inv); + + return new_dti; +} + #endif /* SWIFT_TIMESTEP_H */ diff --git a/src/tools.c b/src/tools.c index ab11d1f5930cf5319aaf6424f1559f144718e154..89ac286fb435c01b361bdea66e62dd2d7f41ee24 100644 --- a/src/tools.c +++ b/src/tools.c @@ -732,19 +732,24 @@ int compare_particles(struct part a, struct part b, double threshold) { #endif } -/** @brief Computes the forces between all g-particles using the N^2 algorithm +/** + * @brief Computes the forces between all g-particles using the N^2 algorithm * * Overwrites the accelerations of the gparts with the values. * Do not use for actual runs. * * @brief gparts The array of particles. * @brief gcount The number of particles. + * @brief constants Physical constants in internal units. + * @brief gravity_properties Constants governing the gravity scheme. */ void gravity_n2(struct gpart *gparts, const int gcount, - const struct phys_const *constants, float rlr) { + const struct phys_const *constants, + const struct gravity_props *gravity_properties, float rlr) { const float rlr_inv = 1. / rlr; - const float max_d = const_gravity_r_cut * rlr; + const float r_cut = gravity_properties->r_cut; + const float max_d = r_cut * rlr; const float max_d2 = max_d * max_d; message("rlr_inv= %f", rlr_inv); diff --git a/src/tools.h b/src/tools.h index ece3078dce7cc8ab4b15538a1e5d9a990d81b36d..4d9e8d3ef86f9ad2661118acf008797893ea5bd7 100644 --- a/src/tools.h +++ b/src/tools.h @@ -23,6 +23,7 @@ #define SWIFT_TOOL_H #include "cell.h" +#include "gravity_properties.h" #include "part.h" #include "physical_constants.h" #include "runner.h" @@ -45,8 +46,8 @@ void pairs_n2(double *dim, struct part *restrict parts, int N, int periodic); double random_uniform(double a, double b); void shuffle_particles(struct part *parts, const int count); void gravity_n2(struct gpart *gparts, const int gcount, - const struct phys_const *constants, float rlr); - + const struct phys_const *constants, + const struct gravity_props *gravity_properties, float rlr); int compare_values(double a, double b, double threshold, double *absDiff, double *absSum, double *relDiff); int compare_particles(struct part a, struct part b, double threshold); diff --git a/src/units.c b/src/units.c index d50186637f5cc14d8b981d1331c6d656b4575592..13d3e5dd35b14c325527ff35703712258e273ef3 100644 --- a/src/units.c +++ b/src/units.c @@ -41,11 +41,11 @@ #include "error.h" /** - * @brief Initialises the UnitSystem structure with CGS system + * @brief Initialises the unit_system structure with CGS system * - * @param us The UnitSystem to initialize + * @param us The unit_system to initialize */ -void units_init_cgs(struct UnitSystem* us) { +void units_init_cgs(struct unit_system* us) { us->UnitMass_in_cgs = 1.; us->UnitLength_in_cgs = 1.; @@ -55,14 +55,14 @@ void units_init_cgs(struct UnitSystem* us) { } /** - * @brief Initialises the UnitSystem structure with the constants given in + * @brief Initialises the unit_system structure with the constants given in * rhe parameter file. * - * @param us The UnitSystem to initialize. + * @param us The unit_system to initialize. * @param params The parsed parameter file. * @param category The section of the parameter file to read from. */ -void units_init(struct UnitSystem* us, const struct swift_params* params, +void units_init(struct unit_system* us, const struct swift_params* params, const char* category) { char buffer[200]; @@ -80,19 +80,19 @@ void units_init(struct UnitSystem* us, const struct swift_params* params, } /** - * @brief Initialises the UnitSystem structure with the constants given in + * @brief Initialises the unit_system structure with the constants given in * rhe parameter file. Uses a default if the values are not present in the file. * - * @param us The UnitSystem to initialize. + * @param us The unit_system to initialize. * @param params The parsed parameter file. * @param category The section of the parameter file to read from. * @param def The default unit system to copy from if required. */ -void units_init_default(struct UnitSystem* us, +void units_init_default(struct unit_system* us, const struct swift_params* params, const char* category, - const struct UnitSystem* def) { + const struct unit_system* def) { - if (!def) error("Default UnitSystem not allocated"); + if (!def) error("Default unit_system not allocated"); char buffer[200]; sprintf(buffer, "%s:UnitMass_in_cgs", category); @@ -116,11 +116,11 @@ void units_init_default(struct UnitSystem* us, /** * @brief Returns the base unit conversion factor for a given unit system - * @param us The UnitSystem used + * @param us The unit_system used * @param baseUnit The base unit */ -double units_get_base_unit(const struct UnitSystem* us, - enum BaseUnits baseUnit) { +double units_get_base_unit(const struct unit_system* us, + enum base_units baseUnit) { switch (baseUnit) { case UNIT_MASS: return us->UnitMass_in_cgs; @@ -142,7 +142,7 @@ double units_get_base_unit(const struct UnitSystem* us, * @brief Returns the base unit symbol used internally * @param baseUnit The base unit */ -const char* units_get_base_unit_internal_symbol(enum BaseUnits baseUnit) { +const char* units_get_base_unit_internal_symbol(enum base_units baseUnit) { switch (baseUnit) { case UNIT_MASS: return "U_M"; @@ -164,7 +164,7 @@ const char* units_get_base_unit_internal_symbol(enum BaseUnits baseUnit) { * @brief Returns the base unit symbol in the cgs system * @param baseUnit The base unit */ -const char* units_get_base_unit_cgs_symbol(enum BaseUnits baseUnit) { +const char* units_get_base_unit_cgs_symbol(enum base_units baseUnit) { switch (baseUnit) { case UNIT_MASS: return "g"; @@ -183,7 +183,7 @@ const char* units_get_base_unit_cgs_symbol(enum BaseUnits baseUnit) { } void units_get_base_unit_exponants_array(float baseUnitsExp[5], - enum UnitConversionFactor unit) { + enum unit_conversion_factor unit) { switch (unit) { case UNIT_CONV_NO_UNITS: break; @@ -337,8 +337,8 @@ void units_get_base_unit_exponants_array(float baseUnitsExp[5], * @param us The system of units in use * @param unit The unit to convert */ -double units_cgs_conversion_factor(const struct UnitSystem* us, - enum UnitConversionFactor unit) { +double units_cgs_conversion_factor(const struct unit_system* us, + enum unit_conversion_factor unit) { float baseUnitsExp[5] = {0.f}; units_get_base_unit_exponants_array(baseUnitsExp, unit); @@ -351,8 +351,8 @@ double units_cgs_conversion_factor(const struct UnitSystem* us, * @param us The system of units in use * @param unit The unit to convert */ -float units_h_factor(const struct UnitSystem* us, - enum UnitConversionFactor unit) { +float units_h_factor(const struct unit_system* us, + enum unit_conversion_factor unit) { float baseUnitsExp[5] = {0.f}; units_get_base_unit_exponants_array(baseUnitsExp, unit); @@ -365,8 +365,8 @@ float units_h_factor(const struct UnitSystem* us, * @param us The system of units in use * @param unit The unit to convert */ -float units_a_factor(const struct UnitSystem* us, - enum UnitConversionFactor unit) { +float units_a_factor(const struct unit_system* us, + enum unit_conversion_factor unit) { float baseUnitsExp[5] = {0.f}; units_get_base_unit_exponants_array(baseUnitsExp, unit); @@ -378,8 +378,8 @@ float units_a_factor(const struct UnitSystem* us, * @brief Returns a string containing the exponents of the base units making up * the conversion factors */ -void units_cgs_conversion_string(char* buffer, const struct UnitSystem* us, - enum UnitConversionFactor unit) { +void units_cgs_conversion_string(char* buffer, const struct unit_system* us, + enum unit_conversion_factor unit) { float baseUnitsExp[5] = {0.f}; units_get_base_unit_exponants_array(baseUnitsExp, unit); @@ -394,13 +394,13 @@ void units_cgs_conversion_string(char* buffer, const struct UnitSystem* us, * @param baseUnitsExponants The exponent of each base units required to form * the desired quantity. See conversionFactor() for a working example */ -double units_general_cgs_conversion_factor(const struct UnitSystem* us, +double units_general_cgs_conversion_factor(const struct unit_system* us, const float baseUnitsExponants[5]) { double factor = 1.; for (int i = 0; i < 5; ++i) if (baseUnitsExponants[i] != 0) - factor *= pow(units_get_base_unit(us, (enum BaseUnits)i), + factor *= pow(units_get_base_unit(us, (enum base_units)i), baseUnitsExponants[i]); return factor; } @@ -412,7 +412,7 @@ double units_general_cgs_conversion_factor(const struct UnitSystem* us, * @param baseUnitsExponants The exponent of each base units required to form * the desired quantity. See conversionFactor() for a working example */ -float units_general_h_factor(const struct UnitSystem* us, +float units_general_h_factor(const struct unit_system* us, const float baseUnitsExponants[5]) { float factor_exp = 0.f; @@ -430,7 +430,7 @@ float units_general_h_factor(const struct UnitSystem* us, * @param baseUnitsExponants The exponent of each base units required to form * the desired quantity. See conversionFactor() for a working example */ -float units_general_a_factor(const struct UnitSystem* us, +float units_general_a_factor(const struct unit_system* us, const float baseUnitsExponants[5]) { float factor_exp = 0.f; @@ -449,7 +449,7 @@ float units_general_a_factor(const struct UnitSystem* us, * the desired quantity. See conversionFactor() for a working example */ void units_general_cgs_conversion_string(char* buffer, - const struct UnitSystem* us, + const struct unit_system* us, const float baseUnitsExponants[5]) { char temp[14]; const double a_exp = units_general_a_factor(us, baseUnitsExponants); @@ -493,14 +493,14 @@ void units_general_cgs_conversion_string(char* buffer, sprintf(temp, " "); else if (baseUnitsExponants[i] == 1.) sprintf(temp, "%s ", - units_get_base_unit_internal_symbol((enum BaseUnits)i)); + units_get_base_unit_internal_symbol((enum base_units)i)); else if (remainder(baseUnitsExponants[i], 1.) == 0) sprintf(temp, "%s^%d ", - units_get_base_unit_internal_symbol((enum BaseUnits)i), + units_get_base_unit_internal_symbol((enum base_units)i), (int)baseUnitsExponants[i]); else sprintf(temp, "%s^%7.4f ", - units_get_base_unit_internal_symbol((enum BaseUnits)i), + units_get_base_unit_internal_symbol((enum base_units)i), baseUnitsExponants[i]); strncat(buffer, temp, 12); } @@ -513,14 +513,15 @@ void units_general_cgs_conversion_string(char* buffer, if (baseUnitsExponants[i] == 0.) continue; else if (baseUnitsExponants[i] == 1.) - sprintf(temp, "%s ", units_get_base_unit_cgs_symbol((enum BaseUnits)i)); + sprintf(temp, "%s ", + units_get_base_unit_cgs_symbol((enum base_units)i)); else if (remainder(baseUnitsExponants[i], 1.) == 0) sprintf(temp, "%s^%d ", - units_get_base_unit_cgs_symbol((enum BaseUnits)i), + units_get_base_unit_cgs_symbol((enum base_units)i), (int)baseUnitsExponants[i]); else sprintf(temp, "%s^%7.4f ", - units_get_base_unit_cgs_symbol((enum BaseUnits)i), + units_get_base_unit_cgs_symbol((enum base_units)i), baseUnitsExponants[i]); strncat(buffer, temp, 12); } @@ -532,11 +533,11 @@ void units_general_cgs_conversion_string(char* buffer, /** * @brief Are the two unit systems equal ? * - * @param a The First #UnitSystem - * @param b The second #UnitSystem + * @param a The First #unit_system + * @param b The second #unit_system * @return 1 if the systems are the same, 0 otherwise */ -int units_are_equal(const struct UnitSystem* a, const struct UnitSystem* b) { +int units_are_equal(const struct unit_system* a, const struct unit_system* b) { if (a->UnitMass_in_cgs != b->UnitMass_in_cgs) return 0; if (a->UnitLength_in_cgs != b->UnitLength_in_cgs) return 0; @@ -550,13 +551,13 @@ int units_are_equal(const struct UnitSystem* a, const struct UnitSystem* b) { /** * @brief Return the unit conversion factor between two systems * - * @param from The #UnitSystem we are converting from - * @param to The #UnitSystem we are converting to + * @param from The #unit_system we are converting from + * @param to The #unit_system we are converting to * @param baseUnitsExponants The exponent of each base units required to form * the desired quantity. See conversionFactor() for a working example */ -double units_general_conversion_factor(const struct UnitSystem* from, - const struct UnitSystem* to, +double units_general_conversion_factor(const struct unit_system* from, + const struct unit_system* to, const float baseUnitsExponants[5]) { const double from_cgs = @@ -570,15 +571,15 @@ double units_general_conversion_factor(const struct UnitSystem* from, /** * @brief Return the unit conversion factor between two systems * - * @param from The #UnitSystem we are converting from - * @param to The #UnitSystem we are converting to + * @param from The #unit_system we are converting from + * @param to The #unit_system we are converting to * @param unit The unit we are converting * * @return The conversion factor */ -double units_conversion_factor(const struct UnitSystem* from, - const struct UnitSystem* to, - enum UnitConversionFactor unit) { +double units_conversion_factor(const struct unit_system* from, + const struct unit_system* to, + enum unit_conversion_factor unit) { float baseUnitsExp[5] = {0.f}; diff --git a/src/units.h b/src/units.h index 78fdf1c23c3c276607d5353ee3437d8eb1e96537..a5765495f9f52159ab70a1072c1f8571ddcdf14b 100644 --- a/src/units.h +++ b/src/units.h @@ -32,7 +32,7 @@ * internal units. It is used everytime a conversion is performed or an i/o * function is called. **/ -struct UnitSystem { +struct unit_system { /*! Conversion factor from grams to internal mass units */ double UnitMass_in_cgs; @@ -54,7 +54,7 @@ struct UnitSystem { * @brief The base units used in the cgs (and internal) system. All units are * derived from those. */ -enum BaseUnits { +enum base_units { UNIT_MASS = 0, UNIT_LENGTH = 1, UNIT_TIME = 2, @@ -65,7 +65,7 @@ enum BaseUnits { /** * @brief The different conversion factors supported by default */ -enum UnitConversionFactor { +enum unit_conversion_factor { UNIT_CONV_NO_UNITS, UNIT_CONV_MASS, UNIT_CONV_LENGTH, @@ -94,47 +94,47 @@ enum UnitConversionFactor { UNIT_CONV_INV_VOLUME }; -void units_init_cgs(struct UnitSystem*); -void units_init(struct UnitSystem*, const struct swift_params*, +void units_init_cgs(struct unit_system*); +void units_init(struct unit_system*, const struct swift_params*, const char* category); -void units_init_default(struct UnitSystem* us, +void units_init_default(struct unit_system* us, const struct swift_params* params, const char* category, - const struct UnitSystem* def); + const struct unit_system* def); -int units_are_equal(const struct UnitSystem* a, const struct UnitSystem* b); +int units_are_equal(const struct unit_system* a, const struct unit_system* b); /* Base units */ -double units_get_base_unit(const struct UnitSystem*, enum BaseUnits); -const char* units_get_base_unit_internal_symbol(enum BaseUnits); -const char* units_get_base_unit_cgs_symbol(enum BaseUnits); +double units_get_base_unit(const struct unit_system*, enum base_units); +const char* units_get_base_unit_internal_symbol(enum base_units); +const char* units_get_base_unit_cgs_symbol(enum base_units); /* Cosmology factors */ -float units_general_h_factor(const struct UnitSystem* us, +float units_general_h_factor(const struct unit_system* us, const float baseUnitsExponants[5]); -float units_h_factor(const struct UnitSystem* us, - enum UnitConversionFactor unit); -float units_general_a_factor(const struct UnitSystem* us, +float units_h_factor(const struct unit_system* us, + enum unit_conversion_factor unit); +float units_general_a_factor(const struct unit_system* us, const float baseUnitsExponants[5]); -float units_a_factor(const struct UnitSystem* us, - enum UnitConversionFactor unit); +float units_a_factor(const struct unit_system* us, + enum unit_conversion_factor unit); /* Conversion to CGS */ -double units_general_cgs_conversion_factor(const struct UnitSystem* us, +double units_general_cgs_conversion_factor(const struct unit_system* us, const float baseUnitsExponants[5]); -double units_cgs_conversion_factor(const struct UnitSystem* us, - enum UnitConversionFactor unit); +double units_cgs_conversion_factor(const struct unit_system* us, + enum unit_conversion_factor unit); void units_general_cgs_conversion_string(char* buffer, - const struct UnitSystem* us, + const struct unit_system* us, const float baseUnitsExponants[5]); -void units_cgs_conversion_string(char* buffer, const struct UnitSystem* us, - enum UnitConversionFactor unit); +void units_cgs_conversion_string(char* buffer, const struct unit_system* us, + enum unit_conversion_factor unit); /* Conversion between systems */ -double units_general_conversion_factor(const struct UnitSystem* from, - const struct UnitSystem* to, +double units_general_conversion_factor(const struct unit_system* from, + const struct unit_system* to, const float baseUnitsExponants[5]); -double units_conversion_factor(const struct UnitSystem* from, - const struct UnitSystem* to, - enum UnitConversionFactor unit); +double units_conversion_factor(const struct unit_system* from, + const struct unit_system* to, + enum unit_conversion_factor unit); #endif /* SWIFT_UNITS_H */ diff --git a/src/version.c b/src/version.c index c5b9255f6cab1fc716a035d3ef739b969a3ab4d4..54a416f6b0745a523382f338fa838018e5254b1e 100644 --- a/src/version.c +++ b/src/version.c @@ -111,6 +111,27 @@ const char *git_branch(void) { return buf; } +/** + * @brief Return the date of the commit in the git repository + * + * The date of the commit of the code we are running. + * + * @result git branch + */ +const char *git_date(void) { + static char buf[256]; + static int initialised = 0; + static const char *date = GIT_DATE; + if (!initialised) { + if (strlen(date) == 0) + sprintf(buf, "%s", "unknown"); + else + sprintf(buf, "%s", date); + initialised = 1; + } + return buf; +} + /** * @brief Return the options passed to the 'configure' script * @@ -332,7 +353,8 @@ void greetings(void) { printf(" SPH With Inter-dependent Fine-grained Tasking\n\n"); printf(" Version : %s\n", package_version()); - printf(" Revision: %s, Branch: %s\n", git_revision(), git_branch()); + printf(" Revision: %s, Branch: %s, Date: %s\n", git_revision(), git_branch(), + git_date()); printf(" Webpage : %s\n\n", PACKAGE_URL); printf(" Config. options: %s\n\n", configuration_options()); printf(" Compiler: %s, Version: %s\n", compiler_name(), compiler_version()); diff --git a/src/version.h b/src/version.h index 5fa057ec8675e9310364eaec3f5098404976e895..60998958098c1a37198cdbb3729982835f6e4f62 100644 --- a/src/version.h +++ b/src/version.h @@ -25,6 +25,7 @@ const char* package_version(void); const char* hostname(void); const char* git_revision(void); const char* git_branch(void); +const char* git_date(void); const char* configuration_options(void); const char* compilation_cflags(void); const char* compiler_name(void); diff --git a/src/version_string.h.in b/src/version_string.h.in index 2be9a84fd52bfe089917d4c0da874fa7ef2dce6b..cea9f189ea4c68a483b7715cc43acd9a8cc26037 100644 --- a/src/version_string.h.in +++ b/src/version_string.h.in @@ -28,6 +28,7 @@ #define PACKAGE_VERSION "@PACKAGE_VERSION@" #define GIT_REVISION "@GIT_REVISION@" #define GIT_BRANCH "@GIT_BRANCH@" +#define GIT_DATE "@GIT_DATE@" #define SWIFT_CFLAGS "@SWIFT_CFLAGS@" #endif /* SWIFT_VERSION_STRING_H */ diff --git a/src/xmf.c b/src/xmf.c new file mode 100644 index 0000000000000000000000000000000000000000..7292606c9f013601db1e9e9b35ee843dea63f785 --- /dev/null +++ b/src/xmf.c @@ -0,0 +1,215 @@ +/******************************************************************************* + * This file is part of SWIFT. + * Copyright (c) 2017 Matthieu Schaller (matthieu.schaller@durham.ac.uk). + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + ******************************************************************************/ + +/* Config parameters. */ +#include "../config.h" + +/* Some standard headers. */ +#include <stdio.h> + +/* This object's header. */ +#include "xmf.h" + +/* Local headers. */ +#include "common_io.h" +#include "error.h" + +/** + * @brief Prepare the XMF file corresponding to a snapshot. + * + * @param baseName The common part of the file name. + */ +FILE* xmf_prepare_file(const char* baseName) { + char buffer[1024]; + + char fileName[FILENAME_BUFFER_SIZE]; + char tempFileName[FILENAME_BUFFER_SIZE]; + snprintf(fileName, FILENAME_BUFFER_SIZE, "%s.xmf", baseName); + snprintf(tempFileName, FILENAME_BUFFER_SIZE, "%s_temp.xmf", baseName); + FILE* xmfFile = fopen(fileName, "r"); + FILE* tempFile = fopen(tempFileName, "w"); + + if (xmfFile == NULL) error("Unable to open current XMF file."); + + if (tempFile == NULL) error("Unable to open temporary file."); + + /* First we make a temporary copy of the XMF file and count the lines */ + int counter = 0; + while (fgets(buffer, 1024, xmfFile) != NULL) { + counter++; + fprintf(tempFile, "%s", buffer); + } + fclose(tempFile); + fclose(xmfFile); + + /* We then copy the XMF file back up to the closing lines */ + xmfFile = fopen(fileName, "w"); + tempFile = fopen(tempFileName, "r"); + + if (xmfFile == NULL) error("Unable to open current XMF file."); + + if (tempFile == NULL) error("Unable to open temporary file."); + + int i = 0; + while (fgets(buffer, 1024, tempFile) != NULL && i < counter - 3) { + i++; + fprintf(xmfFile, "%s", buffer); + } + fprintf(xmfFile, "\n"); + fclose(tempFile); + remove(tempFileName); + + return xmfFile; +} + +/** + * @brief Writes the begin of the XMF file + * + * @todo Exploit the XML nature of the XMF format to write a proper XML writer + * and simplify all the XMF-related stuff. + */ +void xmf_create_file(const char* baseName) { + + char fileName[FILENAME_BUFFER_SIZE]; + snprintf(fileName, FILENAME_BUFFER_SIZE, "%s.xmf", baseName); + FILE* xmfFile = fopen(fileName, "w"); + + fprintf(xmfFile, "<?xml version=\"1.0\" ?> \n"); + fprintf(xmfFile, "<!DOCTYPE Xdmf SYSTEM \"Xdmf.dtd\" []> \n"); + fprintf( + xmfFile, + "<Xdmf xmlns:xi=\"http://www.w3.org/2003/XInclude\" Version=\"2.1\">\n"); + fprintf(xmfFile, "<Domain>\n"); + fprintf(xmfFile, + "<Grid Name=\"TimeSeries\" GridType=\"Collection\" " + "CollectionType=\"Temporal\">\n\n"); + + fprintf(xmfFile, "</Grid>\n"); + fprintf(xmfFile, "</Domain>\n"); + fprintf(xmfFile, "</Xdmf>\n"); + + fclose(xmfFile); +} + +/** + * @brief Writes the part of the XMF entry presenting the geometry of the + * snapshot + * + * @param xmfFile The file to write in. + * @param hdfFileName The name of the HDF5 file corresponding to this output. + * @param time The current simulation time. + */ +void xmf_write_outputheader(FILE* xmfFile, char* hdfFileName, float time) { + /* Write end of file */ + + fprintf(xmfFile, "<!-- XMF description for file: %s -->\n", hdfFileName); + fprintf(xmfFile, + "<Grid GridType=\"Collection\" CollectionType=\"Spatial\">\n"); + fprintf(xmfFile, "<Time Type=\"Single\" Value=\"%f\"/>\n", time); +} + +/** + * @brief Writes the end of the XMF file (closes all open markups) + * + * @param xmfFile The file to write in. + * @param output The number of this output. + * @param time The current simulation time. + */ +void xmf_write_outputfooter(FILE* xmfFile, int output, float time) { + /* Write end of the section of this time step */ + + fprintf(xmfFile, + "\n</Grid> <!-- End of meta-data for output=%03i, time=%f -->\n", + output, time); + fprintf(xmfFile, "\n</Grid> <!-- timeSeries -->\n"); + fprintf(xmfFile, "</Domain>\n"); + fprintf(xmfFile, "</Xdmf>\n"); + + fclose(xmfFile); +} + +/** + * @brief Writes the header of an XMF group for a given particle type. + * + * @param xmfFile The file to write to. + * @param hdfFileName The name of the corresponding HDF5 file. + * @param N The number of particles to write. + * @param ptype The particle type we are writing. + */ +void xmf_write_groupheader(FILE* xmfFile, char* hdfFileName, size_t N, + enum part_type ptype) { + fprintf(xmfFile, "\n<Grid Name=\"%s\" GridType=\"Uniform\">\n", + part_type_names[ptype]); + fprintf(xmfFile, + "<Topology TopologyType=\"Polyvertex\" Dimensions=\"%zu\"/>\n", N); + fprintf(xmfFile, "<Geometry GeometryType=\"XYZ\">\n"); + fprintf(xmfFile, + "<DataItem Dimensions=\"%zu 3\" NumberType=\"Double\" " + "Precision=\"8\" " + "Format=\"HDF\">%s:/PartType%d/Coordinates</DataItem>\n", + N, hdfFileName, (int)ptype); + fprintf(xmfFile, + "</Geometry>\n <!-- Done geometry for %s, start of particle fields " + "list -->\n", + part_type_names[ptype]); +} + +/** + * @brief Writes the footer of an XMF group for a given particle type. + * + * @param xmfFile The file to write to. + * @param ptype The particle type we are writing. + */ +void xmf_write_groupfooter(FILE* xmfFile, enum part_type ptype) { + fprintf(xmfFile, "</Grid> <!-- End of meta-data for parttype=%s -->\n", + part_type_names[ptype]); +} + +/** + * @brief Writes the lines corresponding to an array of the HDF5 output + * + * @param xmfFile The file in which to write + * @param fileName The name of the HDF5 file associated to this XMF descriptor. + * @param partTypeGroupName The name of the group containing the particles in + * the HDF5 file. + * @param name The name of the array in the HDF5 file. + * @param N The number of particles. + * @param dim The dimension of the quantity (1 for scalars, 3 for vectors). + * @param type The type of the data to write. + * + * @todo Treat the types in a better way. + */ +void xmf_write_line(FILE* xmfFile, const char* fileName, + const char* partTypeGroupName, const char* name, size_t N, + int dim, enum IO_DATA_TYPE type) { + fprintf(xmfFile, + "<Attribute Name=\"%s\" AttributeType=\"%s\" Center=\"Node\">\n", + name, dim == 1 ? "Scalar" : "Vector"); + if (dim == 1) + fprintf(xmfFile, + "<DataItem Dimensions=\"%zu\" NumberType=\"Double\" " + "Precision=\"%d\" Format=\"HDF\">%s:%s/%s</DataItem>\n", + N, type == FLOAT ? 4 : 8, fileName, partTypeGroupName, name); + else + fprintf(xmfFile, + "<DataItem Dimensions=\"%zu %d\" NumberType=\"Double\" " + "Precision=\"%d\" Format=\"HDF\">%s:%s/%s</DataItem>\n", + N, dim, type == FLOAT ? 4 : 8, fileName, partTypeGroupName, name); + fprintf(xmfFile, "</Attribute>\n"); +} diff --git a/src/xmf.h b/src/xmf.h new file mode 100644 index 0000000000000000000000000000000000000000..bd2781685f8d1f96daf6e5bfeb45a2bf645fca6e --- /dev/null +++ b/src/xmf.h @@ -0,0 +1,40 @@ +/******************************************************************************* + * This file is part of SWIFT. + * Copyright (c) 2017 Matthieu Schaller (matthieu.schaller@durham.ac.uk). + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + ******************************************************************************/ +#ifndef SWIFT_XMF_H +#define SWIFT_XMF_H + +/* Config parameters. */ +#include "../config.h" + +/* Local headers. */ +#include "common_io.h" +#include "part_type.h" + +void xmf_create_file(const char* baseName); +FILE* xmf_prepare_file(const char* baseName); +void xmf_write_outputheader(FILE* xmfFile, char* hdfFileName, float time); +void xmf_write_outputfooter(FILE* xmfFile, int outputCount, float time); +void xmf_write_groupheader(FILE* xmfFile, char* hdfFileName, size_t N, + enum part_type ptype); +void xmf_write_groupfooter(FILE* xmfFile, enum part_type ptype); +void xmf_write_line(FILE* xmfFile, const char* fileName, + const char* partTypeGroupName, const char* name, size_t N, + int dim, enum IO_DATA_TYPE type); + +#endif /* SWIFT_XMF_H */ diff --git a/tests/benchmarkInteractions.c b/tests/benchmarkInteractions.c index be23fe0d8d00aa35a37747b049750d6f3b31fa92..e3f558f88dffbab252bf7c06f9e943ff568b6fff 100644 --- a/tests/benchmarkInteractions.c +++ b/tests/benchmarkInteractions.c @@ -91,7 +91,10 @@ struct part *make_particles(size_t count, double *offset, double spacing, p->h = h; p->id = ++(*partId); + +#if !defined(GIZMO_SPH) p->mass = 1.0f; +#endif /* Place rest of particles around the test particle * with random position within a unit sphere. */ @@ -110,7 +113,9 @@ struct part *make_particles(size_t count, double *offset, double spacing, p->h = h; p->id = ++(*partId); +#if !defined(GIZMO_SPH) p->mass = 1.0f; +#endif } return particles; } @@ -120,6 +125,7 @@ struct part *make_particles(size_t count, double *offset, double spacing, */ void prepare_force(struct part *parts, size_t count) { +#if !defined(GIZMO_SPH) struct part *p; for (size_t i = 0; i < count; ++i) { p = &parts[i]; @@ -130,6 +136,7 @@ void prepare_force(struct part *parts, size_t count) { p->force.v_sig = 0.0f; p->force.h_dt = 0.0f; } +#endif } /** @@ -144,10 +151,19 @@ void dump_indv_particle_fields(char *fileName, struct part *p) { "%13e %13e %13e %13e " "%13e %13e %13e\n", p->id, p->x[0], p->x[1], p->x[2], p->v[0], p->v[1], p->v[2], - p->a_hydro[0], p->a_hydro[1], p->a_hydro[2], p->rho, - p->density.rho_dh, p->density.wcount, p->density.wcount_dh, - p->force.h_dt, p->force.v_sig, -#if defined(MINIMAL_SPH) + p->a_hydro[0], p->a_hydro[1], p->a_hydro[2], +#if defined(GIZMO_SPH) + 0., 0., +#else + p->rho, p->density.rho_dh, +#endif + p->density.wcount, p->density.wcount_dh, p->force.h_dt, +#if defined(GIZMO_SPH) + 0., +#else + p->force.v_sig, +#endif +#if defined(MINIMAL_SPH) || defined(GIZMO_SPH) 0., 0., 0., 0. #else p->density.div_v, p->density.rot_v[0], p->density.rot_v[1], @@ -237,13 +253,13 @@ void test_interactions(struct part test_part, struct part *parts, size_t count, float r2[count] __attribute__((aligned(array_align))); float dx[3 * count] __attribute__((aligned(array_align))); +#ifdef WITH_VECTORIZATION struct part *piq[count], *pjq[count]; for (size_t k = 0; k < count; k++) { piq[k] = NULL; pjq[k] = NULL; } -#ifdef WITH_VECTORIZATION float r2q[count] __attribute__((aligned(array_align))); float hiq[count] __attribute__((aligned(array_align))); float dxq[count] __attribute__((aligned(array_align))); @@ -342,10 +358,12 @@ void test_interactions(struct part test_part, struct part *parts, size_t count, /* Only dump data on first run. */ if (k == 0) { +#ifdef WITH_VECTORIZATION /* Dump state of particles before vector interaction. */ dump_indv_particle_fields(vec_filename, piq[0]); for (size_t i = 0; i < count; i++) dump_indv_particle_fields(vec_filename, pjq[i]); +#endif } /* Perform vector interaction. */ @@ -411,10 +429,12 @@ void test_interactions(struct part test_part, struct part *parts, size_t count, fprintf(file, "\n# PARTICLES AFTER INTERACTION:\n"); fclose(file); +#ifdef WITH_VECTORIZATION /* Dump result of serial interaction. */ dump_indv_particle_fields(vec_filename, piq[0]); for (size_t i = 0; i < count; i++) dump_indv_particle_fields(vec_filename, pjq[i]); +#endif #ifdef WITH_VECTORIZATION /* Check serial results against the vectorised results. */ diff --git a/tests/test125cells.c b/tests/test125cells.c index 91b1cf6dc3b321643aae1f4eec6bd3d7abb48350..b3895ffa9fc0e4c147bfbf58dc1b5a6301b02763 100644 --- a/tests/test125cells.c +++ b/tests/test125cells.c @@ -532,6 +532,7 @@ int main(int argc, char *argv[]) { engine.s = &space; engine.time = 0.1f; engine.ti_current = 8; + engine.max_active_bin = num_time_bins; struct runner runner; runner.e = &engine; @@ -580,7 +581,7 @@ int main(int argc, char *argv[]) { // runner_do_kick1(&runner, main_cell, 0); /* And a gentle drift */ - // runner_do_drift(&runner, main_cell, 0); + // runner_do_drift_particles(&runner, main_cell, 0); /* First, sort stuff */ for (int j = 0; j < 125; ++j) runner_do_sort(&runner, cells[j], 0x1FFF, 0); @@ -678,7 +679,7 @@ int main(int argc, char *argv[]) { // runner_do_kick1(&runner, main_cell, 0); /* And drift it */ - runner_do_drift(&runner, main_cell, 0); + runner_do_drift_particles(&runner, main_cell, 0); /* Initialise the particles */ for (int j = 0; j < 125; ++j) runner_do_init(&runner, cells[j], 0); diff --git a/tests/test27cells.c b/tests/test27cells.c index 929a148d1f5730b63de79e9a1ab7e25f1ca7311e..599c8713a4b2077444b10f85ed37ee76f5219c5a 100644 --- a/tests/test27cells.c +++ b/tests/test27cells.c @@ -396,6 +396,7 @@ int main(int argc, char *argv[]) { engine.s = &space; engine.time = 0.1f; engine.ti_current = 8; + engine.max_active_bin = num_time_bins; struct runner runner; runner.e = &engine; diff --git a/tests/testDump.c b/tests/testDump.c index ab74a1b1f022761efedf5258a20c525fcef47bd6..7343af49f654582de444681ec291311d41251dca 100644 --- a/tests/testDump.c +++ b/tests/testDump.c @@ -79,6 +79,9 @@ int main(int argc, char *argv[]) { /* Finalize the dump. */ dump_close(&d); + /* Clean the threads */ + threadpool_clean(&t); + /* Return a happy number. */ return 0; } diff --git a/tests/testKernelGrav.c b/tests/testKernelGrav.c index 2733a4a4d041149499d354ef217ae85b1cd35b7f..41e085945693efaf658c219d0e5f992ddf023d74 100644 --- a/tests/testKernelGrav.c +++ b/tests/testKernelGrav.c @@ -87,7 +87,7 @@ int main() { printf("\nAll values are consistent\n"); /* Now test the long range function */ - const float a_smooth = const_gravity_a_smooth; + const float a_smooth = 4.5f; for (int k = 1; k < numPoints; ++k) { diff --git a/tests/testPair.c b/tests/testPair.c index 8b23cc419a661f4d50ea53948302729784a129f9..c734424bca58b7a8ce05bba2c3392ca5c600fa9e 100644 --- a/tests/testPair.c +++ b/tests/testPair.c @@ -252,6 +252,7 @@ int main(int argc, char *argv[]) { engine.s = &space; engine.time = 0.1f; engine.ti_current = 8; + engine.max_active_bin = num_time_bins; runner.e = &engine; volume = particles * particles * particles; diff --git a/tests/testReading.c b/tests/testReading.c index 2ef32a5ef11c7e24a379ce5131df9cbea153fa7c..930434e7118ca175185cc2c8f13e83f486e91d2c 100644 --- a/tests/testReading.c +++ b/tests/testReading.c @@ -25,16 +25,17 @@ int main() { - size_t Ngas = 0, Ngpart = 0; + size_t Ngas = 0, Ngpart = 0, Nspart = 0; int periodic = -1; int flag_entropy_ICs = -1; int i, j, k; double dim[3]; struct part *parts = NULL; struct gpart *gparts = NULL; + struct spart *sparts = NULL; /* Default unit system */ - struct UnitSystem us; + struct unit_system us; units_init_cgs(&us); /* Properties of the ICs */ @@ -43,8 +44,8 @@ int main() { const double rho = 2.; /* Read data */ - read_ic_single("input.hdf5", &us, dim, &parts, &gparts, &Ngas, &Ngpart, - &periodic, &flag_entropy_ICs, 0); + read_ic_single("input.hdf5", &us, dim, &parts, &gparts, &sparts, &Ngas, + &Ngpart, &Nspart, &periodic, &flag_entropy_ICs, 1, 1, 0, 0); /* Check global properties read are correct */ assert(dim[0] == boxSize);