Commit 2e9a2c28 authored by Pedro Gonnet's avatar Pedro Gonnet
Browse files

Merge branch 'master' into size_t

Conflicts:
	src/engine.c
	src/space.c
parents a86eb4e2 8a357b65
......@@ -25,9 +25,16 @@ examples/swift_mindt
examples/swift_mindt_mpi
examples/swift_mpi
tests/testVectorize
tests/brute_force.dat
tests/swift_dopair.dat
tests/testPair
tests/brute_force_standard.dat
tests/swift_dopair_standard.dat
tests/brute_force_perturbed.dat
tests/swift_dopair_perturbed.dat
tests/test27cells
tests/brute_force_27_standard.dat
tests/swift_dopair_27_standard.dat
tests/brute_force_27_perturbed.dat
tests/swift_dopair_27_perturbed.dat
tests/testGreetings
tests/testReading
tests/input.hdf5
......
......@@ -64,7 +64,7 @@ v = inputFile["/PartType0/Velocities"][:,:]
m = inputFile["/PartType0/Masses"][:]
h = inputFile["/PartType0/SmoothingLength"][:]
u = inputFile["/PartType0/InternalEnergy"][:]
ids = np.array(range(np.size(u)), dtype='L')
ids = np.array(range(np.size(u)), dtype='L') + 1
# Downsample
print "Downsampling..."
......
......@@ -111,6 +111,6 @@ ds[()] = h
ds = grp.create_dataset('InternalEnergy', (numPart,1), 'f')
ds[()] = u
ds = grp.create_dataset('ParticleIDs', (numPart, 1), 'L')
ds[()] = ids
ds[()] = ids + 1
file.close()
......@@ -82,7 +82,7 @@ for i in range(L):
else:
P = P + 3. + 4.*log(2.)
u[index] = P / ((gamma - 1.)*rho)
ids[index] = partId
ids[index] = partId + 1
partId = partId + 1
......
......@@ -63,13 +63,15 @@ swift_fixdt_mpi_LDADD = ../src/.libs/libswiftsim_mpi.a $(HDF5_LDFLAGS) $(HDF5_L
# Scripts to generate ICs
EXTRA_DIST = UniformBox/makeIC.py \
PerturbedBox/makeIC.py \
UniformDMBox/makeIC.py \
PerturbedBox/makeIC.py \
SedovBlast/makeIC.py SedovBlast/makeIC_fcc.py SedovBlast/solution.py \
SodShock/makeIC.py SodShock/solution.py SodShock/glass_001.hdf5 SodShock/glass_002.hdf5 SodShock/rhox.py \
CosmoVolume/getIC.sh \
BigCosmoVolume/makeIC.py \
BigPerturbedBox/makeIC_fcc.py \
GreshoVortex/makeIC.py GreshoVortex/solution.py
GreshoVortex/makeIC.py GreshoVortex/solution.py \
MultiTypes/makeIC.py
# Scripts to plot task graphs
......
###############################################################################
# This file is part of SWIFT.
# Copyright (c) 2013 Pedro Gonnet (pedro.gonnet@durham.ac.uk),
# Matthieu Schaller (matthieu.schaller@durham.ac.uk)
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import h5py
import sys
from numpy import *
# Generates a swift IC file containing a cartesian distribution of DM particles
# with a density of 1
# Parameters
periodic= 1 # 1 For periodic box
boxSize = 1.
Lgas = int(sys.argv[1]) # Number of particles along one axis
rhoGas = 2. # Density
P = 1. # Pressure
gamma = 5./3. # Gas adiabatic index
rhoDM = 1.
Ldm = int(sys.argv[2]) # Number of particles along one axis
fileName = "multiTypes.hdf5"
#---------------------------------------------------
numGas = Lgas**3
massGas = boxSize**3 * rhoGas / numGas
internalEnergy = P / ((gamma - 1.)*rhoGas)
numDM = Ldm**3
massDM = boxSize**3 * rhoDM / numDM
#--------------------------------------------------
#File
file = h5py.File(fileName, 'w')
# Header
grp = file.create_group("/Header")
grp.attrs["BoxSize"] = boxSize
grp.attrs["NumPart_Total"] = [numGas, numDM, 0, 0, 0, 0]
grp.attrs["NumPart_Total_HighWord"] = [0, 0, 0, 0, 0, 0]
grp.attrs["NumPart_ThisFile"] = [numGas, numDM, 0, 0, 0, 0]
grp.attrs["Time"] = 0.0
grp.attrs["NumFilesPerSnapshot"] = 1
grp.attrs["MassTable"] = [0.0, massDM, 0.0, 0.0, 0.0, 0.0]
grp.attrs["Flag_Entropy_ICs"] = 0
#Runtime parameters
grp = file.create_group("/RuntimePars")
grp.attrs["PeriodicBoundariesOn"] = periodic
# Gas Particle group
grp = file.create_group("/PartType0")
v = zeros((numGas, 3))
ds = grp.create_dataset('Velocities', (numGas, 3), 'f')
ds[()] = v
v = zeros(1)
m = full((numGas, 1), massGas)
ds = grp.create_dataset('Masses', (numGas,1), 'f')
ds[()] = m
m = zeros(1)
h = full((numGas, 1), 1.1255 * boxSize / Lgas)
ds = grp.create_dataset('SmoothingLength', (numGas,1), 'f')
ds[()] = h
h = zeros(1)
u = full((numGas, 1), internalEnergy)
ds = grp.create_dataset('InternalEnergy', (numGas,1), 'f')
ds[()] = u
u = zeros(1)
ids = linspace(0, numGas, numGas, endpoint=False).reshape((numGas,1))
ds = grp.create_dataset('ParticleIDs', (numGas, 1), 'L')
ds[()] = ids + 1
x = ids % Lgas;
y = ((ids - x) / Lgas) % Lgas;
z = (ids - x - Lgas * y) / Lgas**2;
coords = zeros((numGas, 3))
coords[:,0] = z[:,0] * boxSize / Lgas + boxSize / (2*Lgas)
coords[:,1] = y[:,0] * boxSize / Lgas + boxSize / (2*Lgas)
coords[:,2] = x[:,0] * boxSize / Lgas + boxSize / (2*Lgas)
ds = grp.create_dataset('Coordinates', (numGas, 3), 'd')
ds[()] = coords
# DM Particle group
grp = file.create_group("/PartType1")
v = zeros((numDM, 3))
ds = grp.create_dataset('Velocities', (numDM, 3), 'f')
ds[()] = v
v = zeros(1)
m = full((numDM, 1), massDM)
ds = grp.create_dataset('Masses', (numDM,1), 'f')
ds[()] = m
m = zeros(1)
ids = linspace(0, numDM, numDM, endpoint=False).reshape((numDM,1))
ds = grp.create_dataset('ParticleIDs', (numDM, 1), 'L')
ds[()] = ids + Lgas**3 + 1
x = ids % Ldm;
y = ((ids - x) / Ldm) % Ldm;
z = (ids - x - Ldm * y) / Ldm**2;
coords = zeros((numDM, 3))
coords[:,0] = z[:,0] * boxSize / Ldm + boxSize / (2*Ldm)
coords[:,1] = y[:,0] * boxSize / Ldm + boxSize / (2*Ldm)
coords[:,2] = x[:,0] * boxSize / Ldm + boxSize / (2*Ldm)
ds = grp.create_dataset('Coordinates', (numDM, 3), 'd')
ds[()] = coords
file.close()
......@@ -103,6 +103,6 @@ ds[()] = h
ds = grp.create_dataset('InternalEnergy', (numPart,1), 'f')
ds[()] = u
ds = grp.create_dataset('ParticleIDs', (numPart, 1), 'L')
ds[()] = ids
ds[()] = ids + 1
file.close()
......@@ -69,7 +69,7 @@ for i in range(L):
m[index] = mass
h[index] = 1.1255 * boxSize / L
u[index] = internalEnergy
ids[index] = index
ids[index] = index + 1
if sqrt((x - boxSize/2.)**2 + (y - boxSize/2.)**2 + (z - boxSize/2.)**2) < 2.01 * boxSize/L:
u[index] = u[index] + E0 / (33. * mass)
print "Particle " , index , " set to detonate."
......
......@@ -72,7 +72,7 @@ for i in range(L):
m[index] = mass
h[index] = 1.1255 * hbox
u[index] = internalEnergy
ids[index] = index
ids[index] = index + 1
if sqrt((x - boxSize/2.)**2 + (y - boxSize/2.)**2 + (z - boxSize/2.)**2) < 1.2 * hbox:
u[index] = u[index] + E0 / (28. * mass)
print "Particle " , index , " set to detonate."
......
......@@ -86,8 +86,8 @@ h = append(h1, h2,0)
u = append(u1, u2,0)
m = append(m1, m2,0)
ids = zeros(numPart, dtype='L')
for i in range(numPart):
ids[i] = i
for i in range(1, numPart+1):
ids[i-1] = i
#Final operations
h /= 2
......
......@@ -86,7 +86,7 @@ u = zeros(1)
ids = linspace(0, numPart, numPart, endpoint=False).reshape((numPart,1))
ds = grp.create_dataset('ParticleIDs', (numPart, 1), 'L')
ds[()] = ids
ds[()] = ids + 1
x = ids % L;
y = ((ids - x) / L) % L;
z = (ids - x - L * y) / L**2;
......
......@@ -98,7 +98,7 @@ for n in range(n_iterations):
u = zeros(1)
ids = linspace(offset, offset+N, N, endpoint=False).reshape((N,1))
ds_id[offset:offset+N] = ids
ds_id[offset:offset+N] = ids + 1
x = ids % L;
y = ((ids - x) / L) % L;
z = (ids - x - L * y) / L**2;
......@@ -130,10 +130,8 @@ u = full((remainder, 1), internalEnergy)
ds_u[offset:offset+remainder] = u
u = zeros(1)
print "Done", offset+remainder,"/", numPart
ids = linspace(offset, offset+remainder, remainder, endpoint=False).reshape((remainder,1))
ds_id[offset:offset+remainder] = ids
ds_id[offset:offset+remainder] = ids + 1
x = ids % L;
y = ((ids - x) / L) % L;
z = (ids - x - L * y) / L**2;
......@@ -143,6 +141,7 @@ coords[:,1] = y[:,0] * boxSize / L + boxSize / (2*L)
coords[:,2] = x[:,0] * boxSize / L + boxSize / (2*L)
ds_x[offset:offset+remainder,:] = coords
print "Done", offset+remainder,"/", numPart
......
###############################################################################
# This file is part of SWIFT.
# Copyright (c) 2013 Pedro Gonnet (pedro.gonnet@durham.ac.uk),
# Matthieu Schaller (matthieu.schaller@durham.ac.uk)
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import h5py
import sys
from numpy import *
# Generates a swift IC file containing a cartesian distribution of DM particles
# with a density of 1
# Parameters
periodic= 1 # 1 For periodic box
boxSize = 1.
rho = 1.
L = int(sys.argv[1]) # Number of particles along one axis
fileName = "uniformDMBox_%d.hdf5"%L
#---------------------------------------------------
numPart = L**3
mass = boxSize**3 * rho / numPart
#--------------------------------------------------
#File
file = h5py.File(fileName, 'w')
# Header
grp = file.create_group("/Header")
grp.attrs["BoxSize"] = boxSize
grp.attrs["NumPart_Total"] = [0, numPart, 0, 0, 0, 0]
grp.attrs["NumPart_Total_HighWord"] = [0, 0, 0, 0, 0, 0]
grp.attrs["NumPart_ThisFile"] = [0, numPart, 0, 0, 0, 0]
grp.attrs["Time"] = 0.0
grp.attrs["NumFilesPerSnapshot"] = 1
grp.attrs["MassTable"] = [0.0, mass, 0.0, 0.0, 0.0, 0.0]
grp.attrs["Flag_Entropy_ICs"] = 0
#Runtime parameters
grp = file.create_group("/RuntimePars")
grp.attrs["PeriodicBoundariesOn"] = periodic
#Particle group
grp = file.create_group("/PartType1")
v = zeros((numPart, 3))
ds = grp.create_dataset('Velocities', (numPart, 3), 'f')
ds[()] = v
v = zeros(1)
m = full((numPart, 1), mass)
ds = grp.create_dataset('Masses', (numPart,1), 'f')
ds[()] = m
m = zeros(1)
ids = linspace(0, numPart, numPart, endpoint=False).reshape((numPart,1))
ds = grp.create_dataset('ParticleIDs', (numPart, 1), 'L')
ds[()] = ids + 1
x = ids % L;
y = ((ids - x) / L) % L;
z = (ids - x - L * y) / L**2;
coords = zeros((numPart, 3))
coords[:,0] = z[:,0] * boxSize / L + boxSize / (2*L)
coords[:,1] = y[:,0] * boxSize / L + boxSize / (2*L)
coords[:,2] = x[:,0] * boxSize / L + boxSize / (2*L)
ds = grp.create_dataset('Coordinates', (numPart, 3), 'd')
ds[()] = coords
file.close()
/*******************************************************************************
* This file is part of SWIFT.
* Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk),
......@@ -58,8 +59,9 @@
int main(int argc, char *argv[]) {
int c, icount, j, k, N, periodic = 1;
long long N_total = -1;
int c, icount, periodic = 1;
size_t Ngas = 0, Ngpart = 0;
long long N_total[2] = {0, 0};
int nr_threads = 1, nr_queues = -1;
int dump_tasks = 0;
int data[2];
......@@ -67,6 +69,7 @@ int main(int argc, char *argv[]) {
double h_max = -1.0, scaling = 1.0;
double time_end = DBL_MAX;
struct part *parts = NULL;
struct gpart *gparts = NULL;
struct space s;
struct engine e;
struct UnitSystem us;
......@@ -77,7 +80,9 @@ int main(int argc, char *argv[]) {
int nr_nodes = 1, myrank = 0;
FILE *file_thread;
int with_outputs = 1;
int verbose = 0, talking;
int with_gravity = 0;
int engine_policies = 0;
int verbose = 0, talking = 0;
unsigned long long cpufreq = 0;
#ifdef WITH_MPI
......@@ -95,12 +100,15 @@ int main(int argc, char *argv[]) {
#endif
#endif
/* Choke on FP-exceptions. */
// feenableexcept( FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW );
/* Choke on FP-exceptions. */
// feenableexcept( FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW );
/* Initialize CPU frequency, this also starts time. */
clocks_set_cpufreq(cpufreq);
#ifdef WITH_MPI
/* Start by initializing MPI. */
int res, prov;
int res = 0, prov = 0;
if ((res = MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &prov)) !=
MPI_SUCCESS)
error("Call to MPI_Init failed with error %i.", res);
......@@ -126,9 +134,6 @@ int main(int argc, char *argv[]) {
&initial_partition.grid[1], &initial_partition.grid[0]);
#endif
/* Initialize CPU frequency, this also starts time. */
clocks_set_cpufreq(cpufreq);
/* Greeting message */
if (myrank == 0) greetings();
......@@ -154,7 +159,7 @@ int main(int argc, char *argv[]) {
bzero(&s, sizeof(struct space));
/* Parse the options */
while ((c = getopt(argc, argv, "a:c:d:e:f:h:m:oP:q:R:s:t:v:w:y:z:")) != -1)
while ((c = getopt(argc, argv, "a:c:d:e:f:gh:m:oP:q:R:s:t:v:w:y:z:")) != -1)
switch (c) {
case 'a':
if (sscanf(optarg, "%lf", &scaling) != 1)
......@@ -183,6 +188,9 @@ int main(int argc, char *argv[]) {
case 'f':
if (!strcpy(ICfileName, optarg)) error("Error parsing IC file name.");
break;
case 'g':
with_gravity = 1;
break;
case 'h':
if (sscanf(optarg, "%llu", &cpufreq) != 1)
error("Error parsing CPU frequency.");
......@@ -354,14 +362,14 @@ int main(int argc, char *argv[]) {
if (myrank == 0) clocks_gettime(&tic);
#if defined(WITH_MPI)
#if defined(HAVE_PARALLEL_HDF5)
read_ic_parallel(ICfileName, dim, &parts, &N, &periodic, myrank, nr_nodes,
MPI_COMM_WORLD, MPI_INFO_NULL);
read_ic_parallel(ICfileName, dim, &parts, &gparts, &Ngas, &Ngpart, &periodic,
myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL);
#else
read_ic_serial(ICfileName, dim, &parts, &N, &periodic, myrank, nr_nodes,
MPI_COMM_WORLD, MPI_INFO_NULL);
read_ic_serial(ICfileName, dim, &parts, &gparts, &Ngas, &Ngpart, &periodic,
myrank, nr_nodes, MPI_COMM_WORLD, MPI_INFO_NULL);
#endif
#else
read_ic_single(ICfileName, dim, &parts, &N, &periodic);
read_ic_single(ICfileName, dim, &parts, &gparts, &Ngas, &Ngpart, &periodic);
#endif
if (myrank == 0) {
......@@ -372,24 +380,61 @@ int main(int argc, char *argv[]) {
}
#if defined(WITH_MPI)
long long N_long = N;
MPI_Reduce(&N_long, &N_total, 1, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
long long N_long[2] = {Ngas, Ngpart};
MPI_Reduce(&N_long, &N_total, 2, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
N_total[1] -= N_total[0];
if (myrank == 0)
message("Read %lld gas particles and %lld DM particles from the ICs",
N_total[0], N_total[1]);
#else
N_total[0] = Ngas;
N_total[1] = Ngpart - Ngas;
message("Read %lld gas particles and %lld DM particles from the ICs",
N_total[0], N_total[1]);
#endif
/* MATTHIEU: Temporary fix to preserve master */
if (!with_gravity) {
free(gparts);
gparts = NULL;
for (size_t k = 0; k < Ngas; ++k) parts[k].gpart = NULL;
Ngpart = 0;
#if defined(WITH_MPI)
N_long[0] = Ngas;
N_long[1] = Ngpart;
MPI_Reduce(&N_long, &N_total, 2, MPI_LONG_LONG, MPI_SUM, 0, MPI_COMM_WORLD);
if (myrank == 0)
message(
"AFTER FIX: Read %lld gas particles and %lld DM particles from the "
"ICs",
N_total[0], N_total[1]);
#else
N_total = N;
N_total[0] = Ngas;
N_total[1] = Ngpart;
message(
"AFTER FIX: Read %lld gas particles and %lld DM particles from the ICs",
N_total[0], N_total[1]);
#endif
if (myrank == 0) message("Read %lld particles from the ICs", N_total);
}
/* MATTHIEU: End temporary fix */
/* Apply h scaling */
if (scaling != 1.0)
for (k = 0; k < N; k++) parts[k].h *= scaling;
for (size_t k = 0; k < Ngas; k++) parts[k].h *= scaling;
/* Apply shift */
if (shift[0] != 0 || shift[1] != 0 || shift[2] != 0)
for (k = 0; k < N; k++) {
if (shift[0] != 0 || shift[1] != 0 || shift[2] != 0) {
for (size_t k = 0; k < Ngas; k++) {
parts[k].x[0] += shift[0];
parts[k].x[1] += shift[1];
parts[k].x[2] += shift[2];
}
for (size_t k = 0; k < Ngpart; k++) {
gparts[k].x[0] += shift[0];
gparts[k].x[1] += shift[1];
gparts[k].x[2] += shift[2];
}
}
/* Set default number of queues. */
if (nr_queues < 0) nr_queues = nr_threads;
......@@ -399,7 +444,8 @@ int main(int argc, char *argv[]) {
/* Initialize the space with this data. */
if (myrank == 0) clocks_gettime(&tic);
space_init(&s, dim, parts, N, periodic, h_max, myrank == 0);
space_init(&s, dim, parts, gparts, Ngas, Ngpart, periodic, h_max,
myrank == 0);
if (myrank == 0 && verbose) {
clocks_gettime(&toc);
message("space_init took %.3f %s.", clocks_diff(&tic, &toc),
......@@ -415,6 +461,7 @@ int main(int argc, char *argv[]) {
message("highest-level cell dimensions are [ %i %i %i ].", s.cdim[0],
s.cdim[1], s.cdim[2]);
message("%zi parts in %i cells.", s.nr_parts, s.tot_cells);
message("%zi gparts in %i cells.", s.nr_gparts, s.tot_cells);
message("maximum depth is %d.", s.maxdepth);
// message( "cutoffs in [ %g %g ]." , s.h_min , s.h_max ); fflush(stdout);
}
......@@ -433,12 +480,15 @@ int main(int argc, char *argv[]) {
message("nr of cells at depth %i is %i.", data[0], data[1]);
}
/* Construct the engine policy */
engine_policies = ENGINE_POLICY | engine_policy_steal | engine_policy_hydro;
if (with_gravity) engine_policies |= engine_policy_external_gravity;
/* Initialize the engine with this space. */
if (myrank == 0) clocks_gettime(&tic);
if (myrank == 0) message("nr_nodes is %i.", nr_nodes);
engine_init(&e, &s, dt_max, nr_threads, nr_queues, nr_nodes, myrank,
ENGINE_POLICY | engine_policy_steal | engine_policy_hydro, 0,
time_end, dt_min, dt_max, talking);
engine_policies, 0, time_end, dt_min, dt_max, talking);
if (myrank == 0 && verbose) {
clocks_gettime(&toc);
message("engine_init took %.3f %s.", clocks_diff(&tic, &toc),
......@@ -482,9 +532,10 @@ int main(int argc, char *argv[]) {
if (myrank == 0) {
message(
"Running on %lld particles until t=%.3e with %i threads and %i "
"queues (dt_min=%.3e, dt_max=%.3e)...",
N_total, time_end, e.nr_threads, e.sched.nr_queues, e.dt_min, e.dt_max);
"Running on %lld gas particles and %lld DM particles until t=%.3e with "
"%i threads and %i queues (dt_min=%.3e, dt_max=%.3e)...",
N_total[0], N_total[1], time_end, e.nr_threads, e.sched.nr_queues,
e.dt_min, e.dt_max);
fflush(stdout);
}
......@@ -499,7 +550,7 @@ int main(int argc, char *argv[]) {
clocks_getunit());
/* Let loose a runner on the space. */
for (j = 0; !engine_is_done(&e); j++) {
for (int j = 0; !engine_is_done(&e); j++) {
/* Repartition the space amongst the nodes? */
#ifdef WITH_MPI
......
......@@ -46,9 +46,9 @@ AM_SOURCES = space.c runner.c queue.c task.c cell.c engine.c \
# Include files for distribution, not installation.
nobase_noinst_HEADERS = approx_math.h atomic.h cycle.h error.h inline.h kernel.h vector.h \
runner_doiact.h runner_doiact_grav.h units.h intrinsics.h minmax.h \