Commit 69fafd9b authored by Peter W. Draper's avatar Peter W. Draper
Browse files

Merge branch 'master' into GravityParticles

Conflicts:
	examples/main.c
	src/cell.c
	src/common_io.h
	src/debug.c
	src/engine.c
	src/gravity/Default/gravity_io.h
	src/map.c
	src/single_io.c
	src/single_io.h
	src/space.c
	src/space.h
parents 9f82310c b88e04a2
......@@ -216,6 +216,32 @@ if test "$enable_san" = "yes"; then
fi
fi
# Disable vectorisation for known compilers. This switches off optimizations
# that could be enabled above, so in general should be appended. Slightly odd
# implementation as want to describe as --disable-vec, but macro is enable
# (there is no enable action).
AC_ARG_ENABLE([vec],
[AS_HELP_STRING([--disable-vec],
[Disable vectorization]
)],
[enable_vec="$enableval"],
[enable_vec="yes"]
)
if test "$enable_vec" = "no"; then
if test "$ax_cv_c_compiler_vendor" = "intel"; then
CFLAGS="$CFLAGS -no-vec -no-simd"
AC_MSG_RESULT([disabled Intel vectorization])
elif test "$ax_cv_c_compiler_vendor" = "gnu"; then
CFLAGS="$CFLAGS -fno-tree-vectorize"
AC_MSG_RESULT([disabled GCC vectorization])
elif test "$ax_cv_c_compiler_vendor" = "clang"; then
CFLAGS="$CFLAGS -fno-vectorize -fno-slp-vectorize"
AC_MSG_RESULT([disabled clang vectorization])
else
AC_MSG_WARN([Do not know how to disable vectorization for this compiler])
fi
fi
# Autoconf stuff.
AC_PROG_INSTALL
AC_PROG_MAKE_SET
......
......@@ -64,7 +64,7 @@ v = inputFile["/PartType0/Velocities"][:,:]
m = inputFile["/PartType0/Masses"][:]
h = inputFile["/PartType0/SmoothingLength"][:]
u = inputFile["/PartType0/InternalEnergy"][:]
ids = np.array(range(np.size(u)), dtype='L')
ids = np.array(range(np.size(u)), dtype='L') + 1
# Downsample
print "Downsampling..."
......
......@@ -111,6 +111,6 @@ ds[()] = h
ds = grp.create_dataset('InternalEnergy', (numPart,1), 'f')
ds[()] = u
ds = grp.create_dataset('ParticleIDs', (numPart, 1), 'L')
ds[()] = ids
ds[()] = ids + 1
file.close()
......@@ -82,7 +82,7 @@ for i in range(L):
else:
P = P + 3. + 4.*log(2.)
u[index] = P / ((gamma - 1.)*rho)
ids[index] = partId
ids[index] = partId + 1
partId = partId + 1
......
......@@ -20,7 +20,7 @@
MYFLAGS = -DTIMER
# Add the source directory and debug to CFLAGS
AM_CFLAGS = -I../src -DCPU_TPS=2.67e9 $(HDF5_CPPFLAGS)
AM_CFLAGS = -I../src $(HDF5_CPPFLAGS)
AM_LDFLAGS =
......@@ -63,13 +63,15 @@ swift_fixdt_mpi_LDADD = ../src/.libs/libswiftsim_mpi.a $(HDF5_LDFLAGS) $(HDF5_L
# Scripts to generate ICs
EXTRA_DIST = UniformBox/makeIC.py \
PerturbedBox/makeIC.py \
UniformDMBox/makeIC.py \
PerturbedBox/makeIC.py \
SedovBlast/makeIC.py SedovBlast/makeIC_fcc.py SedovBlast/solution.py \
SodShock/makeIC.py SodShock/solution.py SodShock/glass_001.hdf5 SodShock/glass_002.hdf5 SodShock/rhox.py \
CosmoVolume/getIC.sh \
BigCosmoVolume/makeIC.py \
BigPerturbedBox/makeIC_fcc.py \
GreshoVortex/makeIC.py GreshoVortex/solution.py
GreshoVortex/makeIC.py GreshoVortex/solution.py \
MultiTypes/makeIC.py
# Scripts to plot task graphs
......
###############################################################################
# This file is part of SWIFT.
# Copyright (c) 2013 Pedro Gonnet (pedro.gonnet@durham.ac.uk),
# Matthieu Schaller (matthieu.schaller@durham.ac.uk)
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import h5py
import sys
from numpy import *
# Generates a swift IC file containing a cartesian distribution of DM particles
# with a density of 1
# Parameters
periodic= 1 # 1 For periodic box
boxSize = 1.
Lgas = int(sys.argv[1]) # Number of particles along one axis
rhoGas = 2. # Density
P = 1. # Pressure
gamma = 5./3. # Gas adiabatic index
rhoDM = 1.
Ldm = int(sys.argv[2]) # Number of particles along one axis
fileName = "multiTypes.hdf5"
#---------------------------------------------------
numGas = Lgas**3
massGas = boxSize**3 * rhoGas / numGas
internalEnergy = P / ((gamma - 1.)*rhoGas)
numDM = Ldm**3
massDM = boxSize**3 * rhoDM / numDM
#--------------------------------------------------
#File
file = h5py.File(fileName, 'w')
# Header
grp = file.create_group("/Header")
grp.attrs["BoxSize"] = boxSize
grp.attrs["NumPart_Total"] = [numGas, numDM, 0, 0, 0, 0]
grp.attrs["NumPart_Total_HighWord"] = [0, 0, 0, 0, 0, 0]
grp.attrs["NumPart_ThisFile"] = [numGas, numDM, 0, 0, 0, 0]
grp.attrs["Time"] = 0.0
grp.attrs["NumFilesPerSnapshot"] = 1
grp.attrs["MassTable"] = [0.0, massDM, 0.0, 0.0, 0.0, 0.0]
grp.attrs["Flag_Entropy_ICs"] = 0
#Runtime parameters
grp = file.create_group("/RuntimePars")
grp.attrs["PeriodicBoundariesOn"] = periodic
# Gas Particle group
grp = file.create_group("/PartType0")
v = zeros((numGas, 3))
ds = grp.create_dataset('Velocities', (numGas, 3), 'f')
ds[()] = v
v = zeros(1)
m = full((numGas, 1), massGas)
ds = grp.create_dataset('Masses', (numGas,1), 'f')
ds[()] = m
m = zeros(1)
h = full((numGas, 1), 1.1255 * boxSize / Lgas)
ds = grp.create_dataset('SmoothingLength', (numGas,1), 'f')
ds[()] = h
h = zeros(1)
u = full((numGas, 1), internalEnergy)
ds = grp.create_dataset('InternalEnergy', (numGas,1), 'f')
ds[()] = u
u = zeros(1)
ids = linspace(0, numGas, numGas, endpoint=False).reshape((numGas,1))
ds = grp.create_dataset('ParticleIDs', (numGas, 1), 'L')
ds[()] = ids + 1
x = ids % Lgas;
y = ((ids - x) / Lgas) % Lgas;
z = (ids - x - Lgas * y) / Lgas**2;
coords = zeros((numGas, 3))
coords[:,0] = z[:,0] * boxSize / Lgas + boxSize / (2*Lgas)
coords[:,1] = y[:,0] * boxSize / Lgas + boxSize / (2*Lgas)
coords[:,2] = x[:,0] * boxSize / Lgas + boxSize / (2*Lgas)
ds = grp.create_dataset('Coordinates', (numGas, 3), 'd')
ds[()] = coords
# DM Particle group
grp = file.create_group("/PartType1")
v = zeros((numDM, 3))
ds = grp.create_dataset('Velocities', (numDM, 3), 'f')
ds[()] = v
v = zeros(1)
m = full((numDM, 1), massDM)
ds = grp.create_dataset('Masses', (numDM,1), 'f')
ds[()] = m
m = zeros(1)
ids = linspace(0, numDM, numDM, endpoint=False).reshape((numDM,1))
ds = grp.create_dataset('ParticleIDs', (numDM, 1), 'L')
ds[()] = ids + Lgas**3 + 1
x = ids % Ldm;
y = ((ids - x) / Ldm) % Ldm;
z = (ids - x - Ldm * y) / Ldm**2;
coords = zeros((numDM, 3))
coords[:,0] = z[:,0] * boxSize / Ldm + boxSize / (2*Ldm)
coords[:,1] = y[:,0] * boxSize / Ldm + boxSize / (2*Ldm)
coords[:,2] = x[:,0] * boxSize / Ldm + boxSize / (2*Ldm)
ds = grp.create_dataset('Coordinates', (numDM, 3), 'd')
ds[()] = coords
file.close()
......@@ -103,6 +103,6 @@ ds[()] = h
ds = grp.create_dataset('InternalEnergy', (numPart,1), 'f')
ds[()] = u
ds = grp.create_dataset('ParticleIDs', (numPart, 1), 'L')
ds[()] = ids
ds[()] = ids + 1
file.close()
......@@ -69,7 +69,7 @@ for i in range(L):
m[index] = mass
h[index] = 1.1255 * boxSize / L
u[index] = internalEnergy
ids[index] = index
ids[index] = index + 1
if sqrt((x - boxSize/2.)**2 + (y - boxSize/2.)**2 + (z - boxSize/2.)**2) < 2.01 * boxSize/L:
u[index] = u[index] + E0 / (33. * mass)
print "Particle " , index , " set to detonate."
......
......@@ -72,7 +72,7 @@ for i in range(L):
m[index] = mass
h[index] = 1.1255 * hbox
u[index] = internalEnergy
ids[index] = index
ids[index] = index + 1
if sqrt((x - boxSize/2.)**2 + (y - boxSize/2.)**2 + (z - boxSize/2.)**2) < 1.2 * hbox:
u[index] = u[index] + E0 / (28. * mass)
print "Particle " , index , " set to detonate."
......
......@@ -86,8 +86,8 @@ h = append(h1, h2,0)
u = append(u1, u2,0)
m = append(m1, m2,0)
ids = zeros(numPart, dtype='L')
for i in range(numPart):
ids[i] = i
for i in range(1, numPart+1):
ids[i-1] = i
#Final operations
h /= 2
......
......@@ -86,7 +86,7 @@ u = zeros(1)
ids = linspace(0, numPart, numPart, endpoint=False).reshape((numPart,1))
ds = grp.create_dataset('ParticleIDs', (numPart, 1), 'L')
ds[()] = ids
ds[()] = ids + 1
x = ids % L;
y = ((ids - x) / L) % L;
z = (ids - x - L * y) / L**2;
......
......@@ -98,7 +98,7 @@ for n in range(n_iterations):
u = zeros(1)
ids = linspace(offset, offset+N, N, endpoint=False).reshape((N,1))
ds_id[offset:offset+N] = ids
ds_id[offset:offset+N] = ids + 1
x = ids % L;
y = ((ids - x) / L) % L;
z = (ids - x - L * y) / L**2;
......@@ -130,10 +130,8 @@ u = full((remainder, 1), internalEnergy)
ds_u[offset:offset+remainder] = u
u = zeros(1)
print "Done", offset+remainder,"/", numPart
ids = linspace(offset, offset+remainder, remainder, endpoint=False).reshape((remainder,1))
ds_id[offset:offset+remainder] = ids
ds_id[offset:offset+remainder] = ids + 1
x = ids % L;
y = ((ids - x) / L) % L;
z = (ids - x - L * y) / L**2;
......@@ -143,6 +141,7 @@ coords[:,1] = y[:,0] * boxSize / L + boxSize / (2*L)
coords[:,2] = x[:,0] * boxSize / L + boxSize / (2*L)
ds_x[offset:offset+remainder,:] = coords
print "Done", offset+remainder,"/", numPart
......
###############################################################################
# This file is part of SWIFT.
# Copyright (c) 2013 Pedro Gonnet (pedro.gonnet@durham.ac.uk),
# Matthieu Schaller (matthieu.schaller@durham.ac.uk)
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import h5py
import sys
from numpy import *
# Generates a swift IC file containing a cartesian distribution of DM particles
# with a density of 1
# Parameters
periodic= 1 # 1 For periodic box
boxSize = 1.
rho = 1.
L = int(sys.argv[1]) # Number of particles along one axis
fileName = "uniformDMBox_%d.hdf5"%L
#---------------------------------------------------
numPart = L**3
mass = boxSize**3 * rho / numPart
#--------------------------------------------------
#File
file = h5py.File(fileName, 'w')
# Header
grp = file.create_group("/Header")
grp.attrs["BoxSize"] = boxSize
grp.attrs["NumPart_Total"] = [0, numPart, 0, 0, 0, 0]
grp.attrs["NumPart_Total_HighWord"] = [0, 0, 0, 0, 0, 0]
grp.attrs["NumPart_ThisFile"] = [0, numPart, 0, 0, 0, 0]
grp.attrs["Time"] = 0.0
grp.attrs["NumFilesPerSnapshot"] = 1
grp.attrs["MassTable"] = [0.0, mass, 0.0, 0.0, 0.0, 0.0]
grp.attrs["Flag_Entropy_ICs"] = 0
#Runtime parameters
grp = file.create_group("/RuntimePars")
grp.attrs["PeriodicBoundariesOn"] = periodic
#Particle group
grp = file.create_group("/PartType1")
v = zeros((numPart, 3))
ds = grp.create_dataset('Velocities', (numPart, 3), 'f')
ds[()] = v
v = zeros(1)
m = full((numPart, 1), mass)
ds = grp.create_dataset('Masses', (numPart,1), 'f')
ds[()] = m
m = zeros(1)
ids = linspace(0, numPart, numPart, endpoint=False).reshape((numPart,1))
ds = grp.create_dataset('ParticleIDs', (numPart, 1), 'L')
ds[()] = ids + 1
x = ids % L;
y = ((ids - x) / L) % L;
z = (ids - x - L * y) / L**2;
coords = zeros((numPart, 3))
coords[:,0] = z[:,0] * boxSize / L + boxSize / (2*L)
coords[:,1] = y[:,0] * boxSize / L + boxSize / (2*L)
coords[:,2] = x[:,0] * boxSize / L + boxSize / (2*L)
ds = grp.create_dataset('Coordinates', (numPart, 3), 'd')
ds[()] = coords
file.close()
......@@ -46,11 +46,6 @@
/* Local headers. */
#include "swift.h"
/* Ticks per second on this machine. */
#ifndef CPU_TPS
#define CPU_TPS 2.40e9
#endif
/* Engine policy flags. */
#ifndef ENGINE_POLICY
#define ENGINE_POLICY engine_policy_none
......@@ -75,8 +70,9 @@
int main(int argc, char *argv[]) {
int c, icount, j, k, Ngas = 0, Ndm = 0, periodic = 1;
long long N_total = -1;
int c, icount, periodic = 1;
size_t Ngas = 0, Ngpart = 0;
long long N_total[2] = {0, 0};
int nr_threads = 1, nr_queues = -1;
int dump_tasks = 0;
int data[2];
......@@ -88,13 +84,15 @@ int main(int argc, char *argv[]) {
struct space s;
struct engine e;
struct UnitSystem us;
struct clocks_time tic, toc;
char ICfileName[200] = "";
char dumpfile[30];
float dt_max = 0.0f, dt_min = 0.0f;
ticks tic;
int nr_nodes = 1, myrank = 0;
FILE *file_thread;
int with_outputs = 1;
int verbose = 0, talking;
unsigned long long cpufreq = 0;
#ifdef WITH_MPI
struct partition initial_partition;
......@@ -142,6 +140,9 @@ int main(int argc, char *argv[]) {
&initial_partition.grid[1], &initial_partition.grid[0]);
#endif
/* Initialize CPU frequency, this also starts time. */
clocks_set_cpufreq(cpufreq);
/* Greeting message */
if (myrank == 0) greetings();
......@@ -167,7 +168,7 @@ int main(int argc, char *argv[]) {
bzero(&s, sizeof(struct space));
/* Parse the options */
while ((c = getopt(argc, argv, "a:c:d:e:f:m:oP:q:R:s:t:w:y:z:")) != -1)
while ((c = getopt(argc, argv, "a:c:d:e:f:h:m:oP:q:R:s:t:v:w:y:z:")) != -1)
switch (c) {
case 'a':
if (sscanf(optarg, "%lf", &scaling) != 1)
......@@ -184,7 +185,7 @@ int main(int argc, char *argv[]) {
case 'd':
if (sscanf(optarg, "%f", &dt_min) != 1)
error("Error parsing minimal timestep.");
if (myrank == 0) message("dt_min set to %e.", dt_max);
if (myrank == 0) message("dt_min set to %e.", dt_min);
fflush(stdout);
break;
case 'e':
......@@ -196,6 +197,12 @@ int main(int argc, char *argv[]) {
case 'f':
if (!strcpy(ICfileName, optarg)) error("Error parsing IC file name.");
break;
case 'h':
if (sscanf(optarg, "%llu", &cpufreq) != 1)
error("Error parsing CPU frequency.");
if (myrank == 0) message("CPU frequency set to %llu.", cpufreq);
fflush(stdout);
break;
case 'm':
if (sscanf(optarg, "%lf", &h_max) != 1) error("Error parsing h_max.");
if (myrank == 0) message("maximum h set to %e.", h_max);
......@@ -205,8 +212,8 @@ int main(int argc, char *argv[]) {
with_outputs = 0;
break;
case 'P':
/* Partition type is one of "g", "m", "w", or "v"; "g" can be
* followed by three numbers defining the grid. */
/* Partition type is one of "g", "m", "w", or "v"; "g" can be
* followed by three numbers defining the grid. */
#ifdef WITH_MPI
switch (optarg[0]) {
case 'g':
......@@ -237,8 +244,8 @@ int main(int argc, char *argv[]) {
error("Error parsing number of queues.");
break;
case 'R':
/* Repartition type "n", "b", "v", "e" or "x".
* Note only none is available without METIS. */
/* Repartition type "n", "b", "v", "e" or "x".
* Note only none is available without METIS. */
#ifdef WITH_MPI
switch (optarg[0]) {
case 'n':
......@@ -272,6 +279,12 @@ int main(int argc, char *argv[]) {
if (sscanf(optarg, "%d", &nr_threads) != 1)
error("Error parsing number of threads.");
break;
case 'v':
/* verbose = 1: MPI rank 0 writes
verbose = 2: all MPI ranks write */
if (sscanf(optarg, "%d", &verbose) != 1)
error("Error parsing verbosity level.");
break;
case 'w':
if (sscanf(optarg, "%d", &space_subsize) != 1)
error("Error parsing sub size.");
......@@ -336,6 +349,12 @@ int main(int argc, char *argv[]) {
aFactor(&us, UNIT_CONV_ENTROPY), hFactor(&us, UNIT_CONV_ENTROPY));
}
/* Report CPU frequency. */
if (myrank == 0) {
cpufreq = clocks_get_cpufreq();
message("CPU frequency used for tick conversion: %llu Hz", cpufreq);
}
/* Check we have sensible time step bounds */
if (dt_min > dt_max)
error("Minimal time step size must be large than maximal time step size ");
......@@ -345,51 +364,52 @@ int main(int argc, char *argv[]) {
error("An IC file name must be provided via the option -f");
/* Read particles and space information from (GADGET) IC */
tic = getticks();
if (myrank == 0) clocks_gettime(&tic);
#if defined(WITH_MPI)
#if defined(HAVE_PARALLEL_HDF5)
read_ic_parallel(ICfileName, dim, &gparts, &Ndm, &periodic, myrank, nr_nodes,
read_ic_parallel(ICfileName, dim, &parts, &Ngas, &periodic, myrank, nr_nodes,
MPI_COMM_WORLD, MPI_INFO_NULL);
#else
read_ic_serial(ICfileName, dim, &gparts, &Ndm, &periodic, myrank, nr_nodes,
read_ic_serial(ICfileName, dim, &parts, &Ngas, &periodic, myrank, nr_nodes,
MPI_COMM_WORLD, MPI_INFO_NULL);
#endif
#else
read_ic_single(ICfileName, dim, &parts, &gparts, &Ngas, &Ndm, &periodic);
read_ic_single(ICfileName, dim, &parts, &gparts, &Ngas, &Ngpart, &periodic);
#endif
if (myrank == 0)
message("reading particle properties took %.3f ms.",
((double)(getticks() - tic)) / CPU_TPS * 1000);
fflush(stdout);
if (myrank == 0) {
clocks_gettime(&toc);
message("reading particle properties took %.3f %s.",
clocks_diff(&tic, &toc), clocks_getunit());
fflush(stdout);
}
#if defined(WITH_MPI)
long long tmp;
long long N_long = Ngas + Ndm;
MPI_Reduce(&N_long, &tmp, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
long long N_long = Ndm;
MPI_Reduce(&N_long, &N_total, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
N_total += tmp;
long long N_long[2] = {Ngas, Ngpart};
MPI_Reduce(&N_long, &N_total, 2, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
if (myrank == 0)
message("Read %lld gas particles and %lld DM particles from the ICs",
N_total[0], N_total[1]);
#else
N_total = Ngas + Ndm;
N_total[0] = Ngas;
N_total[1] = Ngpart - Ngas;
message("Read %lld gas particles and %lld DM particles from the ICs",
N_total[0], N_total[1]);
#endif
if (myrank == 0) message("Read %lld particles from the ICs", N_total);
j = 0;
/* Apply h scaling */
if (scaling != 1.0) {
for (k = 0; k < Ngas; k++)
parts[k].h *= scaling;
}
if (scaling != 1.0)
for (size_t k = 0; k < Ngas; k++) parts[k].h *= scaling;
/* Apply shift */
if (shift[0] != 0 || shift[1] != 0 || shift[2] != 0) {
for (k = 0; k < Ngas; k++) {
for (size_t k = 0; k < Ngas; k++) {
parts[k].x[0] += shift[0];
parts[k].x[1] += shift[1];
parts[k].x[2] += shift[2];
}
for (k = 0; k < Ndm; k++) {
for (size_t k = 0; k < Ngpart; k++) {
gparts[k].x[0] += shift[0];
gparts[k].x[1] += shift[1];
gparts[k].x[2] += shift[2];
......@@ -399,13 +419,19 @@ int main(int argc, char *argv[]) {
/* Set default number of queues. */
if (nr_queues < 0) nr_queues = nr_threads;
/* How vocal are we ? */
talking = (verbose == 1 && myrank == 0) || (verbose == 2);
/* Initialize the space with this data. */
tic = getticks();
space_init(&s, dim, parts, gparts, Ngas, Ndm, periodic, h_max, myrank == 0);
if (myrank == 0)