engine.h 8.88 KB
Newer Older
Pedro Gonnet's avatar
Pedro Gonnet committed
1
/*******************************************************************************
2
 * This file is part of SWIFT.
3
 * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
4
5
6
7
8
 *                    Matthieu Schaller (matthieu.schaller@durham.ac.uk)
 *               2015 Peter W. Draper (p.w.draper@durham.ac.uk)
 *                    Angus Lepper (angus.lepper@ed.ac.uk)
 *               2016 John A. Regan (john.a.regan@durham.ac.uk)
 *                    Tom Theuns (tom.theuns@durham.ac.uk)
9
 *
Pedro Gonnet's avatar
Pedro Gonnet committed
10
11
12
13
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published
 * by the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
14
 *
Pedro Gonnet's avatar
Pedro Gonnet committed
15
16
17
18
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
19
 *
Pedro Gonnet's avatar
Pedro Gonnet committed
20
21
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
22
 *
Pedro Gonnet's avatar
Pedro Gonnet committed
23
 ******************************************************************************/
24
25
#ifndef SWIFT_ENGINE_H
#define SWIFT_ENGINE_H
Pedro Gonnet's avatar
Pedro Gonnet committed
26

27
28
29
30
31
32
33
34
/* Config parameters. */
#include "../config.h"

/* MPI headers. */
#ifdef WITH_MPI
#include <mpi.h>
#endif

Pedro Gonnet's avatar
Pedro Gonnet committed
35
36
/* Some standard headers. */
#include <pthread.h>
37
#include <stdio.h>
Pedro Gonnet's avatar
Pedro Gonnet committed
38
39

/* Includes. */
40
#include "clocks.h"
41
#include "collectgroup.h"
42
#include "cooling_struct.h"
43
#include "gravity_properties.h"
Matthieu Schaller's avatar
Matthieu Schaller committed
44
#include "parser.h"
45
#include "partition.h"
46
#include "potential.h"
Matthieu Schaller's avatar
Matthieu Schaller committed
47
#include "runner.h"
48
#include "scheduler.h"
Tom Theuns's avatar
Tom Theuns committed
49
#include "sourceterms_struct.h"
Matthieu Schaller's avatar
Matthieu Schaller committed
50
51
52
#include "space.h"
#include "task.h"
#include "units.h"
Pedro Gonnet's avatar
Pedro Gonnet committed
53

Pedro Gonnet's avatar
Pedro Gonnet committed
54
/* Some constants. */
55
56
enum engine_policy {
  engine_policy_none = 0,
57
58
59
60
  engine_policy_rand = (1 << 0),
  engine_policy_steal = (1 << 1),
  engine_policy_keep = (1 << 2),
  engine_policy_block = (1 << 3),
61
62
63
64
65
66
67
68
  engine_policy_cputight = (1 << 4),
  engine_policy_mpi = (1 << 5),
  engine_policy_setaffinity = (1 << 6),
  engine_policy_hydro = (1 << 7),
  engine_policy_self_gravity = (1 << 8),
  engine_policy_external_gravity = (1 << 9),
  engine_policy_cosmology = (1 << 10),
  engine_policy_drift_all = (1 << 11),
69
70
71
72
  engine_policy_reconstruct_mpoles = (1 << 12),
  engine_policy_cooling = (1 << 13),
  engine_policy_sourceterms = (1 << 14),
  engine_policy_stars = (1 << 15)
73
};
74
#define engine_maxpolicy 15
75
extern const char *engine_policy_names[];
Pedro Gonnet's avatar
Pedro Gonnet committed
76

77
78
#define engine_queue_scale 1.2
#define engine_maxproxies 64
79
#define engine_tasksreweight 1
80
#define engine_parts_size_grow 1.05
81
#define engine_redistribute_alloc_margin 1.2
82
83
#define engine_default_energy_file_name "energy"
#define engine_default_timesteps_file_name "timesteps"
84
#define engine_max_parts_per_ghost 1000
Pedro Gonnet's avatar
Pedro Gonnet committed
85

86
87
88
/* The rank of the engine as a global variable (for messages). */
extern int engine_rank;

Pedro Gonnet's avatar
Pedro Gonnet committed
89
90
91
/* Data structure for the engine. */
struct engine {

92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
  /* Number of threads on which to run. */
  int nr_threads;

  /* The space with which the runner is associated. */
  struct space *s;

  /* The runner's threads. */
  struct runner *runners;

  /* The running policy. */
  int policy;

  /* The task scheduler. */
  struct scheduler sched;

107
108
109
  /* Common threadpool for all the engine's tasks. */
  struct threadpool threadpool;

110
  /* The minimum and maximum allowed dt */
111
  double dt_min, dt_max;
112

113
  /* Time of the simulation beginning */
114
  double timeBegin;
115
116

  /* Time of the simulation end */
117
  double timeEnd;
118
119

  /* The previous system time. */
120
  double timeOld;
121
  integertime_t ti_old;
122
123

  /* The current system time. */
124
  double time;
125
  integertime_t ti_current;
126

127
128
129
  /* The highest active bin at this time */
  timebin_t max_active_bin;

130
  /* Time step */
131
  double timeStep;
132

133
134
  /* Time base */
  double timeBase;
135
  double timeBase_inv;
136

137
  /* Minimal ti_end for the next time-step */
138
  integertime_t ti_end_min;
139

Matthieu Schaller's avatar
Matthieu Schaller committed
140
141
142
143
144
145
  /* Maximal ti_end for the next time-step */
  integertime_t ti_end_max;

  /* Maximal ti_beg for the next time-step */
  integertime_t ti_beg_max;

146
  /* Number of particles updated */
147
  size_t updates, g_updates, s_updates;
148

149
150
151
  /* Total numbers of particles in the system. */
  size_t total_nr_parts, total_nr_gparts;

152
  /* The internal system of units */
153
  const struct unit_system *internal_units;
154

155
  /* Snapshot information */
156
157
  double timeFirstSnapshot;
  double deltaTimeSnapshot;
158
  integertime_t ti_nextSnapshot;
159
  char snapshotBaseName[PARSER_MAX_LINE_SIZE];
160
  int snapshotCompression;
161
  struct unit_system *snapshotUnits;
162

163
  /* Statistics information */
164
  FILE *file_stats;
165
166
  double timeLastStatistics;
  double deltaTimeStatistics;
Pedro Gonnet's avatar
Pedro Gonnet committed
167

Matthieu Schaller's avatar
Matthieu Schaller committed
168
169
170
  /* Timesteps information */
  FILE *file_timesteps;

171
  /* The current step number. */
172
  int step;
173
174
175
176
177

  /* The number of particles updated in the previous step. */
  int count_step;

  /* Data for the threads' barrier. */
Pedro Gonnet's avatar
Pedro Gonnet committed
178
179
  pthread_barrier_t wait_barrier;
  pthread_barrier_t run_barrier;
180
181
182
183
184
185
186
187

  /* ID of the node this engine lives on. */
  int nr_nodes, nodeID;

  /* Proxies for the other nodes in this simulation. */
  struct proxy *proxies;
  int nr_proxies, *proxy_ind;

188
#ifdef SWIFT_DEBUG_TASKS
189
190
  /* Tic/toc at the start/end of a step. */
  ticks tic_step, toc_step;
191
#endif
192

193
#ifdef WITH_MPI
194
195
  /* CPU time of the last step. */
  double cputime_last_step;
196

197
198
  /* Step of last repartition. */
  int last_repartition;
199
200
#endif

201
202
  /* Wallclock time of the last time-step */
  float wallclock_time;
203
204

  /* Force the engine to rebuild? */
205
  int forcerebuild;
206
207
208

  /* Force the engine to repartition ? */
  int forcerepart;
209
  struct repartition *reparttype;
210

211
212
213
  /* Need to dump some statistics ? */
  int save_stats;

214
215
  /* Need to dump a snapshot ? */
  int dump_snapshot;
216
217
218
219
220
221

  /* How many steps have we done with the same set of tasks? */
  int tasks_age;

  /* Linked list for cell-task association. */
  struct link *links;
222
  int nr_links, size_links;
223

224
225
226
227
  /* Average number of tasks per cell. Used to estimate the sizes
   * of the various task arrays. */
  int tasks_per_cell;

228
229
  /* Are we talkative ? */
  int verbose;
230
231

  /* Physical constants definition */
Matthieu Schaller's avatar
Matthieu Schaller committed
232
  const struct phys_const *physical_constants;
233

234
235
236
  /* Properties of the hydro scheme */
  const struct hydro_props *hydro_properties;

237
238
239
  /* Properties of the self-gravity scheme */
  const struct gravity_props *gravity_properties;

240
  /* Properties of external gravitational potential */
Matthieu Schaller's avatar
Matthieu Schaller committed
241
  const struct external_potential *external_potential;
242

243
  /* Properties of the cooling scheme */
Tom Theuns's avatar
Tom Theuns committed
244
  const struct cooling_function_data *cooling_func;
245

Tom Theuns's avatar
Tom Theuns committed
246
  /* Properties of source terms */
Tom Theuns's avatar
Tom Theuns committed
247
  struct sourceterms *sourceterms;
Tom Theuns's avatar
Tom Theuns committed
248

249
250
  /* The (parsed) parameter file */
  const struct swift_params *parameter_file;
251
252
253
254

  /* Temporary struct to hold a group of deferable properties (in MPI mode
   * these are reduced together, but may not be required just yet). */
  struct collectgroup1 collect_group1;
255
};
Pedro Gonnet's avatar
Pedro Gonnet committed
256
257

/* Function prototypes. */
Pedro Gonnet's avatar
Pedro Gonnet committed
258
void engine_barrier(struct engine *e);
259
void engine_compute_next_snapshot_time(struct engine *e);
260
261
void engine_unskip(struct engine *e);
void engine_drift_all(struct engine *e);
262
void engine_drift_top_multipoles(struct engine *e);
263
void engine_reconstruct_multipoles(struct engine *e);
264
void engine_dump_snapshot(struct engine *e);
265
266
void engine_init(struct engine *e, struct space *s,
                 const struct swift_params *params, int nr_nodes, int nodeID,
267
                 int nr_threads, int Ngas, int Ndm, int with_aff, int policy,
268
                 int verbose, struct repartition *reparttype,
269
                 const struct unit_system *internal_units,
270
                 const struct phys_const *physical_constants,
271
                 const struct hydro_props *hydro,
272
                 const struct gravity_props *gravity,
Tom Theuns's avatar
Tom Theuns committed
273
                 const struct external_potential *potential,
274
                 const struct cooling_function_data *cooling_func,
Tom Theuns's avatar
Tom Theuns committed
275
                 struct sourceterms *sourceterms);
276
void engine_launch(struct engine *e);
277
void engine_prepare(struct engine *e);
278
279
void engine_init_particles(struct engine *e, int flag_entropy_ICs,
                           int clean_h_values);
280
281
void engine_step(struct engine *e);
void engine_maketasks(struct engine *e);
282
void engine_split(struct engine *e, struct partition *initial_partition);
283
284
void engine_exchange_strays(struct engine *e, size_t offset_parts,
                            int *ind_part, size_t *Npart, size_t offset_gparts,
285
286
287
                            int *ind_gpart, size_t *Ngpart,
                            size_t offset_sparts, int *ind_spart,
                            size_t *Nspart);
288
void engine_rebuild(struct engine *e, int clean_h_values);
289
void engine_repartition(struct engine *e);
290
void engine_repartition_trigger(struct engine *e);
291
292
void engine_makeproxies(struct engine *e);
void engine_redistribute(struct engine *e);
Matthieu Schaller's avatar
Matthieu Schaller committed
293
void engine_print_policy(struct engine *e);
294
int engine_is_done(struct engine *e);
Matthieu Schaller's avatar
Matthieu Schaller committed
295
void engine_pin();
296
void engine_unpin();
297
void engine_clean(struct engine *e);
298
int engine_estimate_nr_tasks(struct engine *e);
299
300

#endif /* SWIFT_ENGINE_H */