engine.h 13.3 KB
Newer Older
Pedro Gonnet's avatar
Pedro Gonnet committed
1
/*******************************************************************************
2
 * This file is part of SWIFT.
3
 * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
4
5
6
7
8
 *                    Matthieu Schaller (matthieu.schaller@durham.ac.uk)
 *               2015 Peter W. Draper (p.w.draper@durham.ac.uk)
 *                    Angus Lepper (angus.lepper@ed.ac.uk)
 *               2016 John A. Regan (john.a.regan@durham.ac.uk)
 *                    Tom Theuns (tom.theuns@durham.ac.uk)
9
 *
Pedro Gonnet's avatar
Pedro Gonnet committed
10
11
12
13
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published
 * by the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
14
 *
Pedro Gonnet's avatar
Pedro Gonnet committed
15
16
17
18
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
19
 *
Pedro Gonnet's avatar
Pedro Gonnet committed
20
21
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
22
 *
Pedro Gonnet's avatar
Pedro Gonnet committed
23
 ******************************************************************************/
24
25
#ifndef SWIFT_ENGINE_H
#define SWIFT_ENGINE_H
Pedro Gonnet's avatar
Pedro Gonnet committed
26

27
28
29
30
31
32
33
34
/* Config parameters. */
#include "../config.h"

/* MPI headers. */
#ifdef WITH_MPI
#include <mpi.h>
#endif

Pedro Gonnet's avatar
Pedro Gonnet committed
35
/* Includes. */
36
#include "barrier.h"
37
#include "chemistry_struct.h"
38
#include "clocks.h"
39
#include "collectgroup.h"
40
#include "cooling_struct.h"
41
#include "gravity_properties.h"
42
#include "mesh_gravity.h"
Matthieu Schaller's avatar
Matthieu Schaller committed
43
#include "parser.h"
44
#include "partition.h"
45
#include "potential.h"
Matthieu Schaller's avatar
Matthieu Schaller committed
46
#include "runner.h"
47
#include "scheduler.h"
Tom Theuns's avatar
Tom Theuns committed
48
#include "sourceterms_struct.h"
Matthieu Schaller's avatar
Matthieu Schaller committed
49
50
51
#include "space.h"
#include "task.h"
#include "units.h"
Pedro Gonnet's avatar
Pedro Gonnet committed
52

53
54
55
56
// cooling counters
int n_eagle_cooling_rate_calls_1;
int n_eagle_cooling_rate_calls_2;
int n_eagle_cooling_rate_calls_3;
57
int n_eagle_cooling_rate_calls_4;
58

59
60
61
/**
 * @brief The different policies the #engine can follow.
 */
62
63
enum engine_policy {
  engine_policy_none = 0,
64
65
66
67
  engine_policy_rand = (1 << 0),
  engine_policy_steal = (1 << 1),
  engine_policy_keep = (1 << 2),
  engine_policy_block = (1 << 3),
68
69
70
71
72
73
74
75
  engine_policy_cputight = (1 << 4),
  engine_policy_mpi = (1 << 5),
  engine_policy_setaffinity = (1 << 6),
  engine_policy_hydro = (1 << 7),
  engine_policy_self_gravity = (1 << 8),
  engine_policy_external_gravity = (1 << 9),
  engine_policy_cosmology = (1 << 10),
  engine_policy_drift_all = (1 << 11),
76
77
78
  engine_policy_reconstruct_mpoles = (1 << 12),
  engine_policy_cooling = (1 << 13),
  engine_policy_sourceterms = (1 << 14),
79
  engine_policy_stars = (1 << 15),
Loic Hausammann's avatar
Loic Hausammann committed
80
81
  engine_policy_structure_finding = (1 << 16),
  engine_policy_feedback = (1 << 17)
82
};
Loic Hausammann's avatar
Loic Hausammann committed
83
#define engine_maxpolicy 17
84
extern const char *engine_policy_names[];
Pedro Gonnet's avatar
Pedro Gonnet committed
85

86
87
88
89
90
91
92
93
94
/**
 * @brief The different unusual events that can take place in a time-step.
 */
enum engine_step_properties {
  engine_step_prop_none = 0,
  engine_step_prop_rebuild = (1 << 0),
  engine_step_prop_redistribute = (1 << 1),
  engine_step_prop_repartition = (1 << 2),
  engine_step_prop_statistics = (1 << 3),
95
96
  engine_step_prop_snapshot = (1 << 4),
  engine_step_prop_restarts = (1 << 5)
97
98
99
};

/* Some constants */
100
#define engine_maxproxies 64
101
#define engine_tasksreweight 1
102
#define engine_parts_size_grow 1.05
103
#define engine_redistribute_alloc_margin 1.2
104
105
#define engine_default_energy_file_name "energy"
#define engine_default_timesteps_file_name "timesteps"
106
#define engine_max_parts_per_ghost 1000
Loic Hausammann's avatar
Loic Hausammann committed
107
#define engine_max_sparts_per_ghost 1000
Pedro Gonnet's avatar
Pedro Gonnet committed
108

109
110
111
/**
 * @brief The rank of the engine as a global variable (for messages).
 */
112
113
extern int engine_rank;

Pedro Gonnet's avatar
Pedro Gonnet committed
114
115
116
/* Data structure for the engine. */
struct engine {

117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
  /* Number of threads on which to run. */
  int nr_threads;

  /* The space with which the runner is associated. */
  struct space *s;

  /* The runner's threads. */
  struct runner *runners;

  /* The running policy. */
  int policy;

  /* The task scheduler. */
  struct scheduler sched;

132
133
134
  /* Common threadpool for all the engine's tasks. */
  struct threadpool threadpool;

135
  /* The minimum and maximum allowed dt */
136
  double dt_min, dt_max;
137

138
139
140
141
142
143
  /* Maximum time-step allowed by the RMS condition in cosmology runs. */
  double dt_max_RMS_displacement;

  /* Dimensionless factor for the RMS time-step condition. */
  double max_RMS_displacement_factor;

144
  /* Time of the simulation beginning */
145
  double time_begin;
146
147

  /* Time of the simulation end */
148
  double time_end;
149
150

  /* The previous system time. */
151
  double time_old;
152
  integertime_t ti_old;
153
154

  /* The current system time. */
155
  double time;
156
  integertime_t ti_current;
157

158
159
160
  /* The highest active bin at this time */
  timebin_t max_active_bin;

161
162
163
  /* The lowest active bin at this time */
  timebin_t min_active_bin;

164
  /* Time step */
165
  double time_step;
166

167
  /* Time base */
168
169
  double time_base;
  double time_base_inv;
170

171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
  /* Minimal hydro ti_end for the next time-step */
  integertime_t ti_hydro_end_min;

  /* Maximal hydro ti_end for the next time-step */
  integertime_t ti_hydro_end_max;

  /* Maximal hydro ti_beg for the next time-step */
  integertime_t ti_hydro_beg_max;

  /* Minimal gravity ti_end for the next time-step */
  integertime_t ti_gravity_end_min;

  /* Maximal gravity ti_end for the next time-step */
  integertime_t ti_gravity_end_max;

  /* Maximal gravity ti_beg for the next time-step */
  integertime_t ti_gravity_beg_max;

  /* Minimal overall ti_end for the next time-step */
190
  integertime_t ti_end_min;
191

192
  /* Maximal overall ti_end for the next time-step */
Matthieu Schaller's avatar
Matthieu Schaller committed
193
194
  integertime_t ti_end_max;

195
  /* Maximal overall ti_beg for the next time-step */
Matthieu Schaller's avatar
Matthieu Schaller committed
196
197
  integertime_t ti_beg_max;

198
  /* Number of particles updated in the previous step */
199
  long long updates, g_updates, s_updates;
200

201
  /* Number of updates since the last rebuild */
202
203
204
  long long updates_since_rebuild;
  long long g_updates_since_rebuild;
  long long s_updates_since_rebuild;
205

206
  /* Properties of the previous step */
207
  int step_props;
208

209
  /* Total numbers of particles in the system. */
210
  long long total_nr_parts, total_nr_gparts, total_nr_sparts;
211

212
213
214
  /* Total mass in the simulation */
  double total_mass;

215
  /* The internal system of units */
216
  const struct unit_system *internal_units;
Matthieu Schaller's avatar
Matthieu Schaller committed
217

218
  /* Top-level cell locations for VELOCIraptor. */
219
  struct cell_loc *cell_loc;
220

221
  /* Snapshot information */
222
223
224
  double a_first_snapshot;
  double time_first_snapshot;
  double delta_time_snapshot;
Loic Hausammann's avatar
Loic Hausammann committed
225

226
227
  /* Output_List for the snapshots */
  struct output_list *output_list_snapshots;
228
229
230
231

  /* Integer time of the next snapshot */
  integertime_t ti_next_snapshot;

232
233
  char snapshot_base_name[PARSER_MAX_LINE_SIZE];
  int snapshot_compression;
234
  int snapshot_int_time_label_on;
235
236
  struct unit_system *snapshot_units;
  int snapshot_output_count;
237

238
  /* Structure finding information */
239
  int stf_output_freq_format;
240
241
242
243
  int delta_step_stf;
  double a_first_stf_output;
  double time_first_stf_output;
  double delta_time_stf;
Matthieu Schaller's avatar
Matthieu Schaller committed
244

245
246
  /* Output_List for the structure finding */
  struct output_list *output_list_stf;
Loic Hausammann's avatar
Loic Hausammann committed
247

248
  /* Integer time of the next stf output */
249
  integertime_t ti_next_stf;
Matthieu Schaller's avatar
Matthieu Schaller committed
250

251
  char stfBaseName[PARSER_MAX_LINE_SIZE];
252

253
  /* Statistics information */
254
255
256
257
  double a_first_statistics;
  double time_first_statistics;
  double delta_time_statistics;

258
259
  /* Output_List for the stats */
  struct output_list *output_list_stats;
Loic Hausammann's avatar
Format    
Loic Hausammann committed
260

261
262
263
264
  /* Integer time of the next statistics dump */
  integertime_t ti_next_stats;

  /* File handle for the statistics */
265
  FILE *file_stats;
Pedro Gonnet's avatar
Pedro Gonnet committed
266

267
  /* File handle for the timesteps information */
Matthieu Schaller's avatar
Matthieu Schaller committed
268
269
  FILE *file_timesteps;

270
  /* The current step number. */
271
  int step;
272
273

  /* Data for the threads' barrier. */
274
275
  swift_barrier_t wait_barrier;
  swift_barrier_t run_barrier;
276
277
278
279
280
281
282
283

  /* ID of the node this engine lives on. */
  int nr_nodes, nodeID;

  /* Proxies for the other nodes in this simulation. */
  struct proxy *proxies;
  int nr_proxies, *proxy_ind;

284
#ifdef SWIFT_DEBUG_TASKS
285
286
  /* Tic/toc at the start/end of a step. */
  ticks tic_step, toc_step;
287
#endif
288

289
#ifdef WITH_MPI
290
291
  /* CPU time of the last step. */
  double cputime_last_step;
292

293
294
  /* Step of last repartition. */
  int last_repartition;
295
296
#endif

297
298
  /* Wallclock time of the last time-step */
  float wallclock_time;
299

300
301
302
  /* Are we in the process of restaring a simulation? */
  int restarting;

303
  /* Force the engine to rebuild? */
304
  int forcerebuild;
305
306
307

  /* Force the engine to repartition ? */
  int forcerepart;
308
  struct repartition *reparttype;
309
310
311
312
313
314

  /* How many steps have we done with the same set of tasks? */
  int tasks_age;

  /* Linked list for cell-task association. */
  struct link *links;
315
  size_t nr_links, size_links;
316

317
318
  /* Average number of tasks per cell. Used to estimate the sizes
   * of the various task arrays. */
319
  size_t tasks_per_cell;
320

321
322
  /* Are we talkative ? */
  int verbose;
323
324

  /* Physical constants definition */
Matthieu Schaller's avatar
Matthieu Schaller committed
325
  const struct phys_const *physical_constants;
326

327
328
329
  /* The cosmological model */
  struct cosmology *cosmology;

330
331
332
  /* Properties of the hydro scheme */
  const struct hydro_props *hydro_properties;

Loic Hausammann's avatar
Loic Hausammann committed
333
334
335
  /* Properties of the star model */
  const struct stars_props *stars_properties;

336
  /* Properties of the self-gravity scheme */
337
  struct gravity_props *gravity_properties;
338

339
340
341
  /* The mesh used for long-range gravity forces */
  struct pm_mesh *mesh;

342
  /* Properties of external gravitational potential */
Matthieu Schaller's avatar
Matthieu Schaller committed
343
  const struct external_potential *external_potential;
344

345
  /* Properties of the cooling scheme */
Tom Theuns's avatar
Tom Theuns committed
346
  const struct cooling_function_data *cooling_func;
347

348
  /* Properties of the chemistry model */
349
  const struct chemistry_global_data *chemistry;
350

Tom Theuns's avatar
Tom Theuns committed
351
  /* Properties of source terms */
Tom Theuns's avatar
Tom Theuns committed
352
  struct sourceterms *sourceterms;
Tom Theuns's avatar
Tom Theuns committed
353

354
  /* The (parsed) parameter file */
lhausamm's avatar
lhausamm committed
355
  struct swift_params *parameter_file;
356
357
358
359

  /* Temporary struct to hold a group of deferable properties (in MPI mode
   * these are reduced together, but may not be required just yet). */
  struct collectgroup1 collect_group1;
Peter W. Draper's avatar
Peter W. Draper committed
360

361
362
363
  /* Whether to dump restart files. */
  int restart_dump;

364
365
366
  /* Whether to save previous generation of restart files. */
  int restart_save;

367
368
369
  /* Whether to dump restart files after the last step. */
  int restart_onexit;

Peter W. Draper's avatar
Peter W. Draper committed
370
  /* Name of the restart file. */
371
  const char *restart_file;
372
373
374
375
376
377

  /* Ticks between restart dumps. */
  ticks restart_dt;

  /* Time after which next dump will occur. */
  ticks restart_next;
378
379
380

  /* Maximum number of tasks needed for restarting. */
  int restart_max_tasks;
381
};
Pedro Gonnet's avatar
Pedro Gonnet committed
382
383

/* Function prototypes. */
384
void engine_addlink(struct engine *e, struct link **l, struct task *t);
Pedro Gonnet's avatar
Pedro Gonnet committed
385
void engine_barrier(struct engine *e);
386
void engine_compute_next_snapshot_time(struct engine *e);
387
void engine_compute_next_stf_time(struct engine *e);
388
void engine_compute_next_statistics_time(struct engine *e);
389
void engine_recompute_displacement_constraint(struct engine *e);
390
391
void engine_unskip(struct engine *e);
void engine_drift_all(struct engine *e);
392
void engine_drift_top_multipoles(struct engine *e);
393
void engine_reconstruct_multipoles(struct engine *e);
394
void engine_print_stats(struct engine *e);
395
void engine_check_for_dumps(struct engine *e);
396
void engine_dump_snapshot(struct engine *e);
397
void engine_init_output_lists(struct engine *e, struct swift_params *params);
Matthieu Schaller's avatar
Matthieu Schaller committed
398
399
400
void engine_init(struct engine *e, struct space *s, struct swift_params *params,
                 long long Ngas, long long Ngparts, long long Nstars,
                 int policy, int verbose, struct repartition *reparttype,
Matthieu Schaller's avatar
Matthieu Schaller committed
401
402
403
                 const struct unit_system *internal_units,
                 const struct phys_const *physical_constants,
                 struct cosmology *cosmo, const struct hydro_props *hydro,
Loic Hausammann's avatar
Loic Hausammann committed
404
                 struct gravity_props *gravity, const struct stars_props *stars,
405
                 struct pm_mesh *mesh,
Matthieu Schaller's avatar
Matthieu Schaller committed
406
407
408
409
                 const struct external_potential *potential,
                 const struct cooling_function_data *cooling_func,
                 const struct chemistry_global_data *chemistry,
                 struct sourceterms *sourceterms);
Matthieu Schaller's avatar
Matthieu Schaller committed
410
411
412
void engine_config(int restart, struct engine *e, struct swift_params *params,
                   int nr_nodes, int nodeID, int nr_threads, int with_aff,
                   int verbose, const char *restart_file);
413
void engine_launch(struct engine *e);
414
void engine_prepare(struct engine *e);
415
416
void engine_init_particles(struct engine *e, int flag_entropy_ICs,
                           int clean_h_values);
417
418
void engine_step(struct engine *e);
void engine_maketasks(struct engine *e);
419
void engine_split(struct engine *e, struct partition *initial_partition);
420
421
void engine_exchange_strays(struct engine *e, size_t offset_parts,
                            int *ind_part, size_t *Npart, size_t offset_gparts,
422
423
424
                            int *ind_gpart, size_t *Ngpart,
                            size_t offset_sparts, int *ind_spart,
                            size_t *Nspart);
425
void engine_rebuild(struct engine *e, int clean_h_values);
426
void engine_repartition(struct engine *e);
427
void engine_repartition_trigger(struct engine *e);
428
429
void engine_makeproxies(struct engine *e);
void engine_redistribute(struct engine *e);
Matthieu Schaller's avatar
Matthieu Schaller committed
430
void engine_print_policy(struct engine *e);
431
int engine_is_done(struct engine *e);
432
433
void engine_pin(void);
void engine_unpin(void);
434
void engine_clean(struct engine *e);
435
int engine_estimate_nr_tasks(struct engine *e);
436

437
#ifdef HAVE_SETAFFINITY
438
cpu_set_t *engine_entry_affinity(void);
439
440
#endif

441
442
443
/* Struct dump/restore support. */
void engine_struct_dump(struct engine *e, FILE *stream);
void engine_struct_restore(struct engine *e, FILE *stream);
444
void engine_dump_restarts(struct engine *e, int drifted_all, int final_step);
445

446
#endif /* SWIFT_ENGINE_H */