velociraptor_interface.c 15.1 KB
Newer Older
1
2
/*******************************************************************************
 * This file is part of SWIFT.
3
 * Copyright (c) 2018 James Willis (james.s.willis@durham.ac.uk)
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published
 * by the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 ******************************************************************************/

/* Config parameters. */
#include "../config.h"

23
/* Some standard headers. */
24
25
#include <unistd.h>

26
27
28
/* This object's header. */
#include "velociraptor_interface.h"

29
/* Local includes. */
30
#include "engine.h"
James Willis's avatar
James Willis committed
31
#include "hydro.h"
32
33
34
35
#include "swift_velociraptor_part.h"

#ifdef HAVE_VELOCIRAPTOR

36
37
38
39
40
41
42
43
44
45
46
47
/* Structure for passing cosmological information to VELOCIraptor. */
struct cosmoinfo {

  /*! Current expansion factor of the Universe. (cosmology.a) */
  double atime;

  /*! Reduced Hubble constant (H0 / (100km/s/Mpc) (cosmology.h) */
  double littleh;

  /*! Matter density parameter (cosmology.Omega_m) */
  double Omega_m;

48
49
50
51
52
53
54
55
56
  /*! Radiation density parameter (cosmology.Omega_r) */
  double Omega_r;

  /*! Neutrino density parameter (0 in SWIFT) */
  double Omega_nu;

  /*! Neutrino density parameter (cosmology.Omega_k) */
  double Omega_k;

57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
  /*! Baryon density parameter (cosmology.Omega_b) */
  double Omega_b;

  /*! Radiation constant density parameter (cosmology.Omega_lambda) */
  double Omega_Lambda;

  /*! Dark matter density parameter (cosmology.Omega_m - cosmology.Omega_b) */
  double Omega_cdm;

  /*! Dark-energy equation of state at the current time (cosmology.w)*/
  double w_de;
};

/* Structure for passing unit information to VELOCIraptor. */
struct unitinfo {

  /* Length conversion factor to kpc. */
  double lengthtokpc;

  /* Velocity conversion factor to km/s. */
  double velocitytokms;

  /* Mass conversion factor to solar masses. */
  double masstosolarmass;

  /* Potential conversion factor. */
  double energyperunitmass;

  /*! Newton's gravitationl constant (phys_const.const_newton_G)*/
  double gravity;

  /*! Hubble constant at the current redshift (cosmology.H) */
  double hubbleunit;
};

/* Structure to hold the location of a top-level cell. */
struct cell_loc {

  /* Coordinates x,y,z */
  double loc[3];
};

/* Structure for passing simulation information to VELOCIraptor. */
struct siminfo {
  double period, zoomhigresolutionmass, interparticlespacing, spacedimension[3];

  /* Number of top-cells. */
  int numcells;

  /*! Locations of top-level cells. */
  struct cell_loc *cell_loc;

  /*! Top-level cell width. */
  double cellwidth[3];

  /*! Inverse of the top-level cell width. */
  double icellwidth[3];

  int icosmologicalsim;
};

118
119
120
121
122
123
/* Structure for group information back to swift */
struct groupinfo {
  int index;
  long long groupid;
};

pelahi's avatar
pelahi committed
124
125
126
int InitVelociraptor(char *config_name, struct unitinfo unit_info,
                     struct siminfo sim_info, const int numthreads);

127
128
129
130
struct groupinfo *InvokeVelociraptor(
    const int snapnum, char *output_name, struct cosmoinfo cosmo_info,
    struct siminfo sim_info, const size_t num_gravity_parts,
    const size_t num_hydro_parts, struct swift_vel_part *swift_parts,
131
132
    const int *cell_node_ids, const int numthreads,
    const int return_group_flags, int *num_in_groups);
133
134

#endif /* HAVE_VELOCIRAPTOR */
135

136
/**
137
138
 * @brief Initialise VELOCIraptor with configuration, units,
 * simulation info needed to run.
139
140
141
 *
 * @param e The #engine.
 */
142
void velociraptor_init(struct engine *e) {
143

144
#ifdef HAVE_VELOCIRAPTOR
145
146
  const ticks tic = getticks();

147
  /* Internal SWIFT units */
148
149
  const struct unit_system *swift_us = e->internal_units;

150
  /* CGS units and physical constants in CGS */
151
152
153
154
155
  struct unit_system cgs_us;
  units_init_cgs(&cgs_us);
  struct phys_const cgs_pc;
  phys_const_init(&cgs_us, /*params=*/NULL, &cgs_pc);

Matthieu Schaller's avatar
Matthieu Schaller committed
156
  /* Set unit conversions. */
157
  struct unitinfo unit_info;
158
159
160
161
162
163
164
165
166
167
168
  unit_info.lengthtokpc =
      units_cgs_conversion_factor(swift_us, UNIT_CONV_LENGTH) /
      (1000. * cgs_pc.const_parsec);
  unit_info.velocitytokms =
      units_cgs_conversion_factor(swift_us, UNIT_CONV_VELOCITY) / 1.0e5;
  unit_info.masstosolarmass =
      units_cgs_conversion_factor(swift_us, UNIT_CONV_MASS) /
      cgs_pc.const_solar_mass;
  unit_info.energyperunitmass =
      units_cgs_conversion_factor(swift_us, UNIT_CONV_ENERGY_PER_UNIT_MASS) /
      (1.0e10);
Matthieu Schaller's avatar
Matthieu Schaller committed
169
  unit_info.gravity = e->physical_constants->const_newton_G;
James Willis's avatar
James Willis committed
170
  unit_info.hubbleunit = e->cosmology->H0 / e->cosmology->h;
Matthieu Schaller's avatar
Matthieu Schaller committed
171

172
173
  /* Gather some information about the simulation */
  struct siminfo sim_info;
Matthieu Schaller's avatar
Matthieu Schaller committed
174

175
176
  /* Are we running with cosmology? */
  if (e->policy & engine_policy_cosmology) {
Matthieu Schaller's avatar
Matthieu Schaller committed
177
    sim_info.icosmologicalsim = 1;
178
  } else {
Matthieu Schaller's avatar
Matthieu Schaller committed
179
    sim_info.icosmologicalsim = 0;
180
  }
181
182
183
184
185
186
187
188
189

  /* Be nice, talk! */
  if (e->verbose) {
    message("VELOCIraptor conf: Length conversion factor: %e",
            unit_info.lengthtokpc);
    message("VELOCIraptor conf: Velocity conversion factor: %e",
            unit_info.velocitytokms);
    message("VELOCIraptor conf: Mass conversion factor: %e",
            unit_info.masstosolarmass);
190
    message("VELOCIraptor conf: Internal energy conversion factor: %e",
191
192
193
194
195
196
197
            unit_info.energyperunitmass);
    message("VELOCIraptor conf: G: %e", unit_info.gravity);
    message("VELOCIraptor conf: H0/h: %e", unit_info.hubbleunit);
    message("VELOCIraptor conf: Config file name: %s", e->stf_config_file_name);
    message("VELOCIraptor conf: Cosmological Simulation: %d",
            sim_info.icosmologicalsim);
  }
Matthieu Schaller's avatar
Matthieu Schaller committed
198
199

  /* Initialise VELOCIraptor. */
200
201
  if (InitVelociraptor(e->stf_config_file_name, unit_info, sim_info,
                       e->nr_threads) != 1)
202
    error("VELOCIraptor initialisation failed.");
203
204
205
206

  if (e->verbose)
    message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
            clocks_getunit());
207
208
#else
  error("SWIFT not configure to run with VELOCIraptor.");
209
#endif /* HAVE_VELOCIRAPTOR */
210
211
212
213
214
215
}

/**
 * @brief Run VELOCIraptor with current particle data.
 *
 * @param e The #engine.
216
 * @param linked_with_snap Are we running at the same time as a snapshot dump?
217
 */
218
void velociraptor_invoke(struct engine *e, const int linked_with_snap) {
219

220
#ifdef HAVE_VELOCIRAPTOR
221

222
  struct space *s = e->s;
Matthieu Schaller's avatar
Matthieu Schaller committed
223
224
225
226
227
  struct gpart *gparts = s->gparts;
  struct part *parts = s->parts;
  const size_t nr_gparts = s->nr_gparts;
  const size_t nr_hydro_parts = s->nr_parts;
  const int nr_cells = s->nr_cells;
228
229

  const ticks tic = getticks();
230

Matthieu Schaller's avatar
Matthieu Schaller committed
231
  /* Allow thread to run on any core for the duration of the call to
232
233
   * VELOCIraptor so that  when OpenMP threads are spawned
   * they can run on any core on the processor. */
Matthieu Schaller's avatar
Matthieu Schaller committed
234
235
236
237
  const int nr_cores = sysconf(_SC_NPROCESSORS_ONLN);
  pthread_t thread = pthread_self();

  /* Set affinity mask to include all cores on the CPU for VELOCIraptor. */
238
  cpu_set_t cpuset;
Matthieu Schaller's avatar
Matthieu Schaller committed
239
240
241
242
  CPU_ZERO(&cpuset);
  for (int j = 0; j < nr_cores; j++) CPU_SET(j, &cpuset);
  pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset);

243
244
  /* Set cosmology information for this point in time */
  struct cosmoinfo cosmo_info;
245
246
247
248
  cosmo_info.atime = e->cosmology->a;
  cosmo_info.littleh = e->cosmology->h;
  cosmo_info.Omega_m = e->cosmology->Omega_m;
  cosmo_info.Omega_b = e->cosmology->Omega_b;
249
250
251
  cosmo_info.Omega_r = e->cosmology->Omega_r;
  cosmo_info.Omega_k = e->cosmology->Omega_k;
  cosmo_info.Omega_nu = 0.;
252
253
254
255
  cosmo_info.Omega_Lambda = e->cosmology->Omega_lambda;
  cosmo_info.Omega_cdm = e->cosmology->Omega_m - e->cosmology->Omega_b;
  cosmo_info.w_de = e->cosmology->w;

256
257
258
259
260
261
262
263
264
265
266
267
268
  /* Report the cosmo info we use */
  if (e->verbose) {
    message("VELOCIraptor conf: Scale factor: %e", cosmo_info.atime);
    message("VELOCIraptor conf: Little h: %e", cosmo_info.littleh);
    message("VELOCIraptor conf: Omega_m: %e", cosmo_info.Omega_m);
    message("VELOCIraptor conf: Omega_b: %e", cosmo_info.Omega_b);
    message("VELOCIraptor conf: Omega_Lambda: %e", cosmo_info.Omega_Lambda);
    message("VELOCIraptor conf: Omega_cdm: %e", cosmo_info.Omega_cdm);
    message("VELOCIraptor conf: w_de: %e", cosmo_info.w_de);
  }

  /* Update the simulation information */
  struct siminfo sim_info;
269

270
  /* Period of the box (Note we assume a cubic box!) */
271
  if (e->s->periodic) {
272
    sim_info.period = s->dim[0];
273
  } else {
274
275
    sim_info.period = 0.0;
  }
276
277

  /* Tell VELOCIraptor this is not a zoom-in simulation */
278
279
  sim_info.zoomhigresolutionmass = -1.0;

280
  /* Are we running with cosmology? */
281
282
  if (e->policy & engine_policy_cosmology) {
    sim_info.icosmologicalsim = 1;
283
    const size_t total_nr_dmparts = e->total_nr_gparts - e->total_nr_parts;
284
285
    sim_info.interparticlespacing = sim_info.period / cbrt(total_nr_dmparts);
  } else {
286
287
288
289
    sim_info.icosmologicalsim = 0;
    sim_info.interparticlespacing = -1;
  }

290
  /* Set the spatial extent of the simulation volume */
291
292
293
  sim_info.spacedimension[0] = s->dim[0];
  sim_info.spacedimension[1] = s->dim[1];
  sim_info.spacedimension[2] = s->dim[2];
294

295
  /* Store number of top-level cells */
296
  sim_info.numcells = s->nr_cells;
297
298

  /* Size and inverse size of the top-level cells in VELOCIraptor units */
299
300
301
302
303
304
  sim_info.cellwidth[0] = s->cells_top[0].width[0];
  sim_info.cellwidth[1] = s->cells_top[0].width[1];
  sim_info.cellwidth[2] = s->cells_top[0].width[2];
  sim_info.icellwidth[0] = s->iwidth[0];
  sim_info.icellwidth[1] = s->iwidth[1];
  sim_info.icellwidth[2] = s->iwidth[2];
305
306
307
308
309
310

  /* Copy the poisiton of the top-level cells */
  if (posix_memalign((void **)&sim_info.cell_loc, 32,
                     s->nr_cells * sizeof(struct cell_loc)) != 0)
    error("Failed to allocate top-level cell locations for VELOCIraptor.");
  for (int i = 0; i < s->nr_cells; i++) {
311
312
313
    sim_info.cell_loc[i].loc[0] = s->cells_top[i].loc[0];
    sim_info.cell_loc[i].loc[1] = s->cells_top[i].loc[1];
    sim_info.cell_loc[i].loc[2] = s->cells_top[i].loc[2];
314
  }
315

316
317
318
319
320
321
322
323
324
325
326
327
328
  if (e->verbose) {
    message("VELOCIraptor conf: Space dimensions: (%e,%e,%e)",
            sim_info.spacedimension[0], sim_info.spacedimension[1],
            sim_info.spacedimension[2]);
    message("VELOCIraptor conf: No. of top-level cells: %d", sim_info.numcells);
    message(
        "VELOCIraptor conf: Top-level cell locations range: (%e,%e,%e) -> "
        "(%e,%e,%e)",
        sim_info.cell_loc[0].loc[0], sim_info.cell_loc[0].loc[1],
        sim_info.cell_loc[0].loc[2],
        sim_info.cell_loc[sim_info.numcells - 1].loc[0],
        sim_info.cell_loc[sim_info.numcells - 1].loc[1],
        sim_info.cell_loc[sim_info.numcells - 1].loc[2]);
329
330
  }

331
  /* Allocate and populate array of cell node IDs. */
332
  int *cell_node_ids = NULL;
Matthieu Schaller's avatar
Matthieu Schaller committed
333
334
335
336
  if (posix_memalign((void **)&cell_node_ids, 32, nr_cells * sizeof(int)) != 0)
    error("Failed to allocate list of cells node IDs for VELOCIraptor.");
  for (int i = 0; i < nr_cells; i++) cell_node_ids[i] = s->cells_top[i].nodeID;

337
  /* Mention the number of particles being sent */
338
  if (e->verbose)
339
340
341
    message(
        "VELOCIraptor conf: MPI rank %d sending %zu gparts to VELOCIraptor.",
        engine_rank, nr_gparts);
Matthieu Schaller's avatar
Matthieu Schaller committed
342

343
  /* Append base name with the current output number */
Matthieu Schaller's avatar
Matthieu Schaller committed
344
  char outputFileName[PARSER_MAX_LINE_SIZE + 128];
345

346
347
348
349
350
351
352
353
  /* What should the filename be? */
  if (linked_with_snap) {
    snprintf(outputFileName, PARSER_MAX_LINE_SIZE + 128,
             "stf_%s_%04i.VELOCIraptor", e->snapshot_base_name,
             e->snapshot_output_count);
  } else {
    snprintf(outputFileName, PARSER_MAX_LINE_SIZE + 128, "%s_%04i.VELOCIraptor",
             e->stf_base_name, e->stf_output_count);
354
355
356
357
358
359
360
  }

  /* What is the snapshot number? */
  int snapnum;
  if (linked_with_snap) {
    snapnum = e->snapshot_output_count;
  } else {
361
    snapnum = e->stf_output_count;
362
  }
Matthieu Schaller's avatar
Matthieu Schaller committed
363
364
365
366
367
368
369
370

  /* Allocate and populate an array of swift_vel_parts to be passed to
   * VELOCIraptor. */
  struct swift_vel_part *swift_parts = NULL;
  if (posix_memalign((void **)&swift_parts, part_align,
                     nr_gparts * sizeof(struct swift_vel_part)) != 0)
    error("Failed to allocate array of particles for VELOCIraptor.");

371
  const float a_inv = e->cosmology->a_inv;
Matthieu Schaller's avatar
Matthieu Schaller committed
372
373
374

  /* Convert particle properties into VELOCIraptor units */
  for (size_t i = 0; i < nr_gparts; i++) {
375

Matthieu Schaller's avatar
Matthieu Schaller committed
376
377
378
    swift_parts[i].x[0] = gparts[i].x[0];
    swift_parts[i].x[1] = gparts[i].x[1];
    swift_parts[i].x[2] = gparts[i].x[2];
379
380
381
382
383

    swift_parts[i].v[0] = gparts[i].v_full[0] * a_inv;
    swift_parts[i].v[1] = gparts[i].v_full[1] * a_inv;
    swift_parts[i].v[2] = gparts[i].v_full[2] * a_inv;

Matthieu Schaller's avatar
Matthieu Schaller committed
384
    swift_parts[i].mass = gravity_get_mass(&gparts[i]);
James Willis's avatar
James Willis committed
385
    swift_parts[i].potential = gravity_get_comoving_potential(&gparts[i]);
386

Matthieu Schaller's avatar
Matthieu Schaller committed
387
388
    swift_parts[i].type = gparts[i].type;

389
390
    swift_parts[i].index = i;
#ifdef WITH_MPI
391
    swift_parts[i].task = e->nodeID;
392
393
394
395
#else
    swift_parts[i].task = 0;
#endif

Matthieu Schaller's avatar
Matthieu Schaller committed
396
397
    /* Set gas particle IDs from their hydro counterparts and set internal
     * energies. */
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
    switch (gparts[i].type) {

      case swift_type_gas:

        swift_parts[i].id = parts[-gparts[i].id_or_neg_offset].id;
        swift_parts[i].u = hydro_get_drifted_physical_internal_energy(
            &parts[-gparts[i].id_or_neg_offset], e->cosmology);
        break;

      case swift_type_dark_matter:

        swift_parts[i].id = gparts[i].id_or_neg_offset;
        swift_parts[i].u = 0.f;
        break;

      default:
        error("Particle type not handled by VELOCIraptor.");
415
    }
Matthieu Schaller's avatar
Matthieu Schaller committed
416
  }
417

418
419
420
421
  /* Values returned by VELOCIRaptor */
  int num_gparts_in_groups = -1;
  struct groupinfo *group_info = NULL;

Matthieu Schaller's avatar
Matthieu Schaller committed
422
  /* Call VELOCIraptor. */
423
424
  group_info = (struct groupinfo *)InvokeVelociraptor(
      snapnum, outputFileName, cosmo_info, sim_info, nr_gparts, nr_hydro_parts,
425
426
      swift_parts, cell_node_ids, e->nr_threads, linked_with_snap,
      &num_gparts_in_groups);
427
428

  /* Check that the ouput is valid */
429
  if (linked_with_snap && group_info == NULL) {
Matthieu Schaller's avatar
Matthieu Schaller committed
430
    error("Exiting. Call to VELOCIraptor failed on rank: %d.", e->nodeID);
431
  }
432
433
434
  if (!linked_with_snap && group_info != NULL) {
    error("VELOCIraptor returned an array whilst it should not have.");
  }
Matthieu Schaller's avatar
Matthieu Schaller committed
435

436
  /* Assign the group IDs back to the gparts */
437
  if (linked_with_snap) {
438

439
440
441
442
443
444
445
446
    for (int i = 0; i < num_gparts_in_groups; i++) {
      ///\todo need to update particle structure to include a group id
      // gparts[group_info[i].index].groupid=group_info[i].index;
    }

    /* Free the array returned by VELOCIraptor */
    free(group_info);
  }
Matthieu Schaller's avatar
Matthieu Schaller committed
447

448
  /* Free everything we allocated */
Matthieu Schaller's avatar
Matthieu Schaller committed
449
450
  free(cell_node_ids);
  free(swift_parts);
451
  free(sim_info.cell_loc);
Matthieu Schaller's avatar
Matthieu Schaller committed
452

453
454
455
  /* Reset the pthread affinity mask after VELOCIraptor returns. */
  pthread_setaffinity_np(thread, sizeof(cpu_set_t), engine_entry_affinity());

456
  /* Increase output counter (if not linked with snapshots) */
457
  if (!linked_with_snap) e->stf_output_count++;
458

459
460
461
  if (e->verbose)
    message("took %.3f %s.", clocks_from_ticks(getticks() - tic),
            clocks_getunit());
462
#else
Matthieu Schaller's avatar
Matthieu Schaller committed
463
  error("SWIFT not configure to run with VELOCIraptor.");
464
#endif /* HAVE_VELOCIRAPTOR */
465
}