hydro_iact.h 33 KB
Newer Older
1
2
/*******************************************************************************
 * This file is part of SWIFT.
3
 * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
4
 *                    Matthieu Schaller (matthieu.schaller@durham.ac.uk)
5
 *
6
7
8
9
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published
 * by the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
10
 *
11
12
13
14
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
15
 *
16
17
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
18
 *
19
 ******************************************************************************/
20
21
#ifndef SWIFT_GADGET2_HYDRO_IACT_H
#define SWIFT_GADGET2_HYDRO_IACT_H
22
23

/**
24
 * @file Gadget2/hydro_iact.h
25
26
 * @brief SPH interaction functions following the Gadget-2 version of SPH.
 *
27
 * The interactions computed here are the ones presented in the Gadget-2 paper
28
29
 * Springel, V., MNRAS, Volume 364, Issue 4, pp. 1105-1134.
 * We use the same numerical coefficients as the Gadget-2 code. When used with
30
31
32
 * the Spline-3 kernel, the results should be equivalent to the ones obtained
 * with Gadget-2 up to the rounding errors and interactions missed by the
 * Gadget-2 tree-code neighbours search.
33
34
 */

35
#include "cache.h"
James Willis's avatar
James Willis committed
36
#include "minmax.h"
37

38
39
40
/**
 * @brief Density loop
 */
41
42
43
__attribute__((always_inline)) INLINE static void runner_iact_density(
    float r2, float *dx, float hi, float hj, struct part *pi, struct part *pj) {

44
45
  float wi, wi_dx;
  float wj, wj_dx;
46
  float dv[3], curlvr[3];
47

48
  /* Get the masses. */
49
  const float mi = pi->mass;
50
51
52
53
54
55
56
57
58
59
60
61
62
  const float mj = pj->mass;

  /* Get r and r inverse. */
  const float r = sqrtf(r2);
  const float r_inv = 1.0f / r;

  /* Compute the kernel function for pi */
  const float hi_inv = 1.f / hi;
  const float ui = r * hi_inv;
  kernel_deval(ui, &wi, &wi_dx);

  /* Compute contribution to the density */
  pi->rho += mj * wi;
63
  pi->density.rho_dh -= mj * (hydro_dimension * wi + ui * wi_dx);
64

65
66
  /* Compute contribution to the number of neighbours */
  pi->density.wcount += wi;
67
  pi->density.wcount_dh -= (hydro_dimension * wi + ui * wi_dx);
68
69
70
71
72
73
74
75

  /* Compute the kernel function for pj */
  const float hj_inv = 1.f / hj;
  const float uj = r * hj_inv;
  kernel_deval(uj, &wj, &wj_dx);

  /* Compute contribution to the density */
  pj->rho += mi * wj;
76
  pj->density.rho_dh -= mi * (hydro_dimension * wj + uj * wj_dx);
77

78
79
  /* Compute contribution to the number of neighbours */
  pj->density.wcount += wj;
80
  pj->density.wcount_dh -= (hydro_dimension * wj + uj * wj_dx);
81

82
83
  const float faci = mj * wi_dx * r_inv;
  const float facj = mi * wj_dx * r_inv;
84

85
86
87
88
  /* Compute dv dot r */
  dv[0] = pi->v[0] - pj->v[0];
  dv[1] = pi->v[1] - pj->v[1];
  dv[2] = pi->v[2] - pj->v[2];
89
90
  const float dvdr = dv[0] * dx[0] + dv[1] * dx[1] + dv[2] * dx[2];

91
92
  pi->density.div_v -= faci * dvdr;
  pj->density.div_v -= facj * dvdr;
93
94
95
96
97
98

  /* Compute dv cross r */
  curlvr[0] = dv[1] * dx[2] - dv[2] * dx[1];
  curlvr[1] = dv[2] * dx[0] - dv[0] * dx[2];
  curlvr[2] = dv[0] * dx[1] - dv[1] * dx[0];

99
100
101
  pi->density.rot_v[0] += faci * curlvr[0];
  pi->density.rot_v[1] += faci * curlvr[1];
  pi->density.rot_v[2] += faci * curlvr[2];
102

103
104
105
  pj->density.rot_v[0] += facj * curlvr[0];
  pj->density.rot_v[1] += facj * curlvr[1];
  pj->density.rot_v[2] += facj * curlvr[2];
106
107
}

108
109
110
/**
 * @brief Density loop (non-symmetric version)
 */
111
112
113
114
115
116
117
__attribute__((always_inline)) INLINE static void runner_iact_nonsym_density(
    float r2, float *dx, float hi, float hj, struct part *pi, struct part *pj) {

  float wi, wi_dx;
  float dv[3], curlvr[3];

  /* Get the masses. */
118
  const float mj = pj->mass;
119
120

  /* Get r and r inverse. */
121
  const float r = sqrtf(r2);
122
  const float r_inv = 1.0f / r;
123

124
  /* Compute the kernel function */
125
126
127
  const float hi_inv = 1.0f / hi;
  const float ui = r * hi_inv;
  kernel_deval(ui, &wi, &wi_dx);
128
129
130

  /* Compute contribution to the density */
  pi->rho += mj * wi;
131
  pi->density.rho_dh -= mj * (hydro_dimension * wi + ui * wi_dx);
132
133
134

  /* Compute contribution to the number of neighbours */
  pi->density.wcount += wi;
135
  pi->density.wcount_dh -= (hydro_dimension * wi + ui * wi_dx);
136

137
  const float fac = mj * wi_dx * r_inv;
138

139
140
141
142
143
  /* Compute dv dot r */
  dv[0] = pi->v[0] - pj->v[0];
  dv[1] = pi->v[1] - pj->v[1];
  dv[2] = pi->v[2] - pj->v[2];
  const float dvdr = dv[0] * dx[0] + dv[1] * dx[1] + dv[2] * dx[2];
144
  pi->density.div_v -= fac * dvdr;
145

146
147
148
149
150
  /* Compute dv cross r */
  curlvr[0] = dv[1] * dx[2] - dv[2] * dx[1];
  curlvr[1] = dv[2] * dx[0] - dv[0] * dx[2];
  curlvr[2] = dv[0] * dx[1] - dv[1] * dx[0];

151
152
153
  pi->density.rot_v[0] += fac * curlvr[0];
  pi->density.rot_v[1] += fac * curlvr[1];
  pi->density.rot_v[2] += fac * curlvr[2];
154
155
}

156
#ifdef WITH_VECTORIZATION
157
158
159
160
161

/**
 * @brief Density interaction computed using 1 vector
 * (non-symmetric vectorized version).
 */
162
__attribute__((always_inline)) INLINE static void
Matthieu Schaller's avatar
Matthieu Schaller committed
163
164
165
166
167
168
169
runner_iact_nonsym_1_vec_density(vector *r2, vector *dx, vector *dy, vector *dz,
                                 vector hi_inv, vector vix, vector viy,
                                 vector viz, float *Vjx, float *Vjy, float *Vjz,
                                 float *Mj, vector *rhoSum, vector *rho_dhSum,
                                 vector *wcountSum, vector *wcount_dhSum,
                                 vector *div_vSum, vector *curlvxSum,
                                 vector *curlvySum, vector *curlvzSum,
170
                                 mask_t mask) {
171

172
  vector r, ri, ui, wi, wi_dx;
173
174
175
  vector dvx, dvy, dvz;
  vector dvdr;
  vector curlvrx, curlvry, curlvrz;
James Willis's avatar
James Willis committed
176

177
  /* Fill the vectors. */
178
179
180
181
  const vector mj = vector_load(Mj);
  const vector vjx = vector_load(Vjx);
  const vector vjy = vector_load(Vjy);
  const vector vjz = vector_load(Vjz);
182
183
184
185
186

  /* Get the radius and inverse radius. */
  ri = vec_reciprocal_sqrt(*r2);
  r.v = vec_mul(r2->v, ri.v);

187
  ui.v = vec_mul(r.v, hi_inv.v);
188
189

  /* Calculate the kernel for two particles. */
190
  kernel_deval_1_vec(&ui, &wi, &wi_dx);
191
192
193
194
195
196
197
198
199
200

  /* Compute dv. */
  dvx.v = vec_sub(vix.v, vjx.v);
  dvy.v = vec_sub(viy.v, vjy.v);
  dvz.v = vec_sub(viz.v, vjz.v);

  /* Compute dv dot r */
  dvdr.v = vec_fma(dvx.v, dx->v, vec_fma(dvy.v, dy->v, vec_mul(dvz.v, dz->v)));
  dvdr.v = vec_mul(dvdr.v, ri.v);

201
202
203
204
205
206
207
208
209
210
211
  /* Compute dv cross r */
  curlvrx.v =
      vec_fma(dvy.v, dz->v, vec_mul(vec_set1(-1.0f), vec_mul(dvz.v, dy->v)));
  curlvry.v =
      vec_fma(dvz.v, dx->v, vec_mul(vec_set1(-1.0f), vec_mul(dvx.v, dz->v)));
  curlvrz.v =
      vec_fma(dvx.v, dy->v, vec_mul(vec_set1(-1.0f), vec_mul(dvy.v, dx->v)));
  curlvrx.v = vec_mul(curlvrx.v, ri.v);
  curlvry.v = vec_mul(curlvry.v, ri.v);
  curlvrz.v = vec_mul(curlvrz.v, ri.v);

212
213
  vector wcount_dh_update;
  wcount_dh_update.v =
James Willis's avatar
James Willis committed
214
      vec_fma(vec_set1(hydro_dimension), wi.v, vec_mul(ui.v, wi_dx.v));
215

216
  /* Mask updates to intermediate vector sums for particle pi. */
217
  rhoSum->v = vec_mask_add(rhoSum->v, vec_mul(mj.v, wi.v), mask);
218
219
  rho_dhSum->v =
      vec_mask_sub(rho_dhSum->v, vec_mul(mj.v, wcount_dh_update.v), mask);
220
  wcountSum->v = vec_mask_add(wcountSum->v, wi.v, mask);
221
  wcount_dhSum->v = vec_mask_sub(wcount_dhSum->v, wcount_dh_update.v, mask);
James Willis's avatar
James Willis committed
222
223
224
225
226
227
228
229
  div_vSum->v =
      vec_mask_sub(div_vSum->v, vec_mul(mj.v, vec_mul(dvdr.v, wi_dx.v)), mask);
  curlvxSum->v = vec_mask_add(curlvxSum->v,
                              vec_mul(mj.v, vec_mul(curlvrx.v, wi_dx.v)), mask);
  curlvySum->v = vec_mask_add(curlvySum->v,
                              vec_mul(mj.v, vec_mul(curlvry.v, wi_dx.v)), mask);
  curlvzSum->v = vec_mask_add(curlvzSum->v,
                              vec_mul(mj.v, vec_mul(curlvrz.v, wi_dx.v)), mask);
230
231
}

232
/**
James Willis's avatar
James Willis committed
233
234
 * @brief Density interaction computed using 2 interleaved vectors
 * (non-symmetric vectorized version).
235
236
 */
__attribute__((always_inline)) INLINE static void
James Willis's avatar
James Willis committed
237
238
239
240
241
242
243
244
runner_iact_nonsym_2_vec_density(float *R2, float *Dx, float *Dy, float *Dz,
                                 vector hi_inv, vector vix, vector viy,
                                 vector viz, float *Vjx, float *Vjy, float *Vjz,
                                 float *Mj, vector *rhoSum, vector *rho_dhSum,
                                 vector *wcountSum, vector *wcount_dhSum,
                                 vector *div_vSum, vector *curlvxSum,
                                 vector *curlvySum, vector *curlvzSum,
                                 mask_t mask, mask_t mask2, short mask_cond) {
245

246
247
  vector r, ri, ui, wi, wi_dx;
  vector dvx, dvy, dvz;
248
249
  vector dvdr;
  vector curlvrx, curlvry, curlvrz;
250
251
  vector r_2, ri2, ui2, wi2, wi_dx2;
  vector dvx2, dvy2, dvz2;
252
253
254
  vector dvdr2;
  vector curlvrx2, curlvry2, curlvrz2;

James Willis's avatar
James Willis committed
255
  /* Fill the vectors. */
256
257
258
259
260
261
262
263
264
265
266
267
268
269
  const vector mj = vector_load(Mj);
  const vector mj2 = vector_load(&Mj[VEC_SIZE]);
  const vector vjx = vector_load(Vjx);
  const vector vjx2 = vector_load(&Vjx[VEC_SIZE]);
  const vector vjy = vector_load(Vjy);
  const vector vjy2 = vector_load(&Vjy[VEC_SIZE]);
  const vector vjz = vector_load(Vjz);
  const vector vjz2 = vector_load(&Vjz[VEC_SIZE]);
  const vector dx = vector_load(Dx);
  const vector dx2 = vector_load(&Dx[VEC_SIZE]);
  const vector dy = vector_load(Dy);
  const vector dy2 = vector_load(&Dy[VEC_SIZE]);
  const vector dz = vector_load(Dz);
  const vector dz2 = vector_load(&Dz[VEC_SIZE]);
270
271

  /* Get the radius and inverse radius. */
272
273
  const vector r2 = vector_load(R2);
  const vector r2_2 = vector_load(&R2[VEC_SIZE]);
274
275
  ri = vec_reciprocal_sqrt(r2);
  ri2 = vec_reciprocal_sqrt(r2_2);
276
277
278
  r.v = vec_mul(r2.v, ri.v);
  r_2.v = vec_mul(r2_2.v, ri2.v);

279
280
  ui.v = vec_mul(r.v, hi_inv.v);
  ui2.v = vec_mul(r_2.v, hi_inv.v);
281

James Willis's avatar
James Willis committed
282
  /* Calculate the kernel for two particles. */
283
  kernel_deval_2_vec(&ui, &wi, &wi_dx, &ui2, &wi2, &wi_dx2);
284
285
286
287
288
289
290
291
292
293
294

  /* Compute dv. */
  dvx.v = vec_sub(vix.v, vjx.v);
  dvx2.v = vec_sub(vix.v, vjx2.v);
  dvy.v = vec_sub(viy.v, vjy.v);
  dvy2.v = vec_sub(viy.v, vjy2.v);
  dvz.v = vec_sub(viz.v, vjz.v);
  dvz2.v = vec_sub(viz.v, vjz2.v);

  /* Compute dv dot r */
  dvdr.v = vec_fma(dvx.v, dx.v, vec_fma(dvy.v, dy.v, vec_mul(dvz.v, dz.v)));
James Willis's avatar
James Willis committed
295
296
  dvdr2.v =
      vec_fma(dvx2.v, dx2.v, vec_fma(dvy2.v, dy2.v, vec_mul(dvz2.v, dz2.v)));
297
298
299
300
  dvdr.v = vec_mul(dvdr.v, ri.v);
  dvdr2.v = vec_mul(dvdr2.v, ri2.v);

  /* Compute dv cross r */
James Willis's avatar
James Willis committed
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
  curlvrx.v =
      vec_fma(dvy.v, dz.v, vec_mul(vec_set1(-1.0f), vec_mul(dvz.v, dy.v)));
  curlvrx2.v =
      vec_fma(dvy2.v, dz2.v, vec_mul(vec_set1(-1.0f), vec_mul(dvz2.v, dy2.v)));
  curlvry.v =
      vec_fma(dvz.v, dx.v, vec_mul(vec_set1(-1.0f), vec_mul(dvx.v, dz.v)));
  curlvry2.v =
      vec_fma(dvz2.v, dx2.v, vec_mul(vec_set1(-1.0f), vec_mul(dvx2.v, dz2.v)));
  curlvrz.v =
      vec_fma(dvx.v, dy.v, vec_mul(vec_set1(-1.0f), vec_mul(dvy.v, dx.v)));
  curlvrz2.v =
      vec_fma(dvx2.v, dy2.v, vec_mul(vec_set1(-1.0f), vec_mul(dvy2.v, dx2.v)));
  curlvrx.v = vec_mul(curlvrx.v, ri.v);
  curlvrx2.v = vec_mul(curlvrx2.v, ri2.v);
  curlvry.v = vec_mul(curlvry.v, ri.v);
  curlvry2.v = vec_mul(curlvry2.v, ri2.v);
  curlvrz.v = vec_mul(curlvrz.v, ri.v);
  curlvrz2.v = vec_mul(curlvrz2.v, ri2.v);

320
321
  vector wcount_dh_update, wcount_dh_update2;
  wcount_dh_update.v =
James Willis's avatar
James Willis committed
322
      vec_fma(vec_set1(hydro_dimension), wi.v, vec_mul(ui.v, wi_dx.v));
323
  wcount_dh_update2.v =
James Willis's avatar
James Willis committed
324
      vec_fma(vec_set1(hydro_dimension), wi2.v, vec_mul(ui2.v, wi_dx2.v));
325

James Willis's avatar
James Willis committed
326
  /* Mask updates to intermediate vector sums for particle pi. */
327
  /* Mask only when needed. */
James Willis's avatar
James Willis committed
328
  if (mask_cond) {
329
330
    rhoSum->v = vec_mask_add(rhoSum->v, vec_mul(mj.v, wi.v), mask);
    rhoSum->v = vec_mask_add(rhoSum->v, vec_mul(mj2.v, wi2.v), mask2);
James Willis's avatar
James Willis committed
331
    rho_dhSum->v =
332
        vec_mask_sub(rho_dhSum->v, vec_mul(mj.v, wcount_dh_update.v), mask);
James Willis's avatar
James Willis committed
333
    rho_dhSum->v =
334
        vec_mask_sub(rho_dhSum->v, vec_mul(mj2.v, wcount_dh_update2.v), mask2);
335
336
    wcountSum->v = vec_mask_add(wcountSum->v, wi.v, mask);
    wcountSum->v = vec_mask_add(wcountSum->v, wi2.v, mask2);
337
338
    wcount_dhSum->v = vec_mask_sub(wcount_dhSum->v, wcount_dh_update.v, mask);
    wcount_dhSum->v = vec_mask_sub(wcount_dhSum->v, wcount_dh_update2.v, mask2);
James Willis's avatar
James Willis committed
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
    div_vSum->v = vec_mask_sub(div_vSum->v,
                               vec_mul(mj.v, vec_mul(dvdr.v, wi_dx.v)), mask);
    div_vSum->v = vec_mask_sub(
        div_vSum->v, vec_mul(mj2.v, vec_mul(dvdr2.v, wi_dx2.v)), mask2);
    curlvxSum->v = vec_mask_add(
        curlvxSum->v, vec_mul(mj.v, vec_mul(curlvrx.v, wi_dx.v)), mask);
    curlvxSum->v = vec_mask_add(
        curlvxSum->v, vec_mul(mj2.v, vec_mul(curlvrx2.v, wi_dx2.v)), mask2);
    curlvySum->v = vec_mask_add(
        curlvySum->v, vec_mul(mj.v, vec_mul(curlvry.v, wi_dx.v)), mask);
    curlvySum->v = vec_mask_add(
        curlvySum->v, vec_mul(mj2.v, vec_mul(curlvry2.v, wi_dx2.v)), mask2);
    curlvzSum->v = vec_mask_add(
        curlvzSum->v, vec_mul(mj.v, vec_mul(curlvrz.v, wi_dx.v)), mask);
    curlvzSum->v = vec_mask_add(
        curlvzSum->v, vec_mul(mj2.v, vec_mul(curlvrz2.v, wi_dx2.v)), mask2);
  } else {
356
357
    rhoSum->v = vec_add(rhoSum->v, vec_mul(mj.v, wi.v));
    rhoSum->v = vec_add(rhoSum->v, vec_mul(mj2.v, wi2.v));
358
359
    rho_dhSum->v = vec_sub(rho_dhSum->v, vec_mul(mj.v, wcount_dh_update.v));
    rho_dhSum->v = vec_sub(rho_dhSum->v, vec_mul(mj2.v, wcount_dh_update2.v));
360
361
    wcountSum->v = vec_add(wcountSum->v, wi.v);
    wcountSum->v = vec_add(wcountSum->v, wi2.v);
362
363
    wcount_dhSum->v = vec_sub(wcount_dhSum->v, wcount_dh_update.v);
    wcount_dhSum->v = vec_sub(wcount_dhSum->v, wcount_dh_update2.v);
364
    div_vSum->v = vec_sub(div_vSum->v, vec_mul(mj.v, vec_mul(dvdr.v, wi_dx.v)));
James Willis's avatar
James Willis committed
365
366
367
368
369
370
371
372
373
374
375
376
377
378
    div_vSum->v =
        vec_sub(div_vSum->v, vec_mul(mj2.v, vec_mul(dvdr2.v, wi_dx2.v)));
    curlvxSum->v =
        vec_add(curlvxSum->v, vec_mul(mj.v, vec_mul(curlvrx.v, wi_dx.v)));
    curlvxSum->v =
        vec_add(curlvxSum->v, vec_mul(mj2.v, vec_mul(curlvrx2.v, wi_dx2.v)));
    curlvySum->v =
        vec_add(curlvySum->v, vec_mul(mj.v, vec_mul(curlvry.v, wi_dx.v)));
    curlvySum->v =
        vec_add(curlvySum->v, vec_mul(mj2.v, vec_mul(curlvry2.v, wi_dx2.v)));
    curlvzSum->v =
        vec_add(curlvzSum->v, vec_mul(mj.v, vec_mul(curlvrz.v, wi_dx.v)));
    curlvzSum->v =
        vec_add(curlvzSum->v, vec_mul(mj2.v, vec_mul(curlvrz2.v, wi_dx2.v)));
379
  }
380
}
James Willis's avatar
James Willis committed
381
#endif
382

383
384
385
/**
 * @brief Force loop
 */
386
387
388
__attribute__((always_inline)) INLINE static void runner_iact_force(
    float r2, float *dx, float hi, float hj, struct part *pi, struct part *pj) {

389
390
391
  float wi, wj, wi_dx, wj_dx;

  const float fac_mu = 1.f; /* Will change with cosmological integration */
392

393
394
395
396
  const float r = sqrtf(r2);
  const float r_inv = 1.0f / r;

  /* Get some values in local variables. */
397
  const float mi = pi->mass;
398
399
400
401
402
403
  const float mj = pj->mass;
  const float rhoi = pi->rho;
  const float rhoj = pj->rho;

  /* Get the kernel for hi. */
  const float hi_inv = 1.0f / hi;
404
  const float hid_inv = pow_dimension_plus_one(hi_inv); /* 1/h^(d+1) */
405
406
  const float ui = r * hi_inv;
  kernel_deval(ui, &wi, &wi_dx);
407
  const float wi_dr = hid_inv * wi_dx;
408
409
410

  /* Get the kernel for hj. */
  const float hj_inv = 1.0f / hj;
411
  const float hjd_inv = pow_dimension_plus_one(hj_inv); /* 1/h^(d+1) */
412
413
  const float xj = r * hj_inv;
  kernel_deval(xj, &wj, &wj_dx);
414
  const float wj_dr = hjd_inv * wj_dx;
415

416
417
418
419
420
  /* Compute h-gradient terms */
  const float f_i = pi->force.f;
  const float f_j = pj->force.f;

  /* Compute pressure terms */
421
422
  const float P_over_rho2_i = pi->force.P_over_rho2;
  const float P_over_rho2_j = pj->force.P_over_rho2;
423
424

  /* Compute sound speeds */
425
426
  const float ci = pi->force.soundspeed;
  const float cj = pj->force.soundspeed;
427

428
  /* Compute dv dot r. */
429
430
431
  const float dvdr = (pi->v[0] - pj->v[0]) * dx[0] +
                     (pi->v[1] - pj->v[1]) * dx[1] +
                     (pi->v[2] - pj->v[2]) * dx[2];
432

433
  /* Balsara term */
434
435
  const float balsara_i = pi->force.balsara;
  const float balsara_j = pj->force.balsara;
Matthieu Schaller's avatar
Matthieu Schaller committed
436

437
  /* Are the particles moving towards each others ? */
438
  const float omega_ij = (dvdr < 0.f) ? dvdr : 0.f;
439
440
441
442
443
444
445
446
447
  const float mu_ij = fac_mu * r_inv * omega_ij; /* This is 0 or negative */

  /* Signal velocity */
  const float v_sig = ci + cj - 3.f * mu_ij;

  /* Now construct the full viscosity term */
  const float rho_ij = 0.5f * (rhoi + rhoj);
  const float visc = -0.25f * const_viscosity_alpha * v_sig * mu_ij *
                     (balsara_i + balsara_j) / rho_ij;
448
449

  /* Now, convolve with the kernel */
450
  const float visc_term = 0.5f * visc * (wi_dr + wj_dr) * r_inv;
451
  const float sph_term =
452
      (f_i * P_over_rho2_i * wi_dr + f_j * P_over_rho2_j * wj_dr) * r_inv;
453
454
455
456
457

  /* Eventually got the acceleration */
  const float acc = visc_term + sph_term;

  /* Use the force Luke ! */
458
459
460
  pi->a_hydro[0] -= mj * acc * dx[0];
  pi->a_hydro[1] -= mj * acc * dx[1];
  pi->a_hydro[2] -= mj * acc * dx[2];
461

462
463
464
  pj->a_hydro[0] += mi * acc * dx[0];
  pj->a_hydro[1] += mi * acc * dx[1];
  pj->a_hydro[2] += mi * acc * dx[2];
465

466
  /* Get the time derivative for h. */
467
468
  pi->force.h_dt -= mj * dvdr * r_inv / rhoj * wi_dr;
  pj->force.h_dt -= mi * dvdr * r_inv / rhoi * wj_dr;
469

470
  /* Update the signal velocity. */
471
472
  pi->force.v_sig = (pi->force.v_sig > v_sig) ? pi->force.v_sig : v_sig;
  pj->force.v_sig = (pj->force.v_sig > v_sig) ? pj->force.v_sig : v_sig;
473

474
  /* Change in entropy */
475
476
  pi->entropy_dt += mj * visc_term * dvdr;
  pj->entropy_dt += mi * visc_term * dvdr;
477
}
478
479
480
481

/**
 * @brief Force loop (non-symmetric version)
 */
482
483
484
__attribute__((always_inline)) INLINE static void runner_iact_nonsym_force(
    float r2, float *dx, float hi, float hj, struct part *pi, struct part *pj) {

485
486
487
  float wi, wj, wi_dx, wj_dx;

  const float fac_mu = 1.f; /* Will change with cosmological integration */
488

489
490
491
492
  const float r = sqrtf(r2);
  const float r_inv = 1.0f / r;

  /* Get some values in local variables. */
493
  // const float mi = pi->mass;
494
495
496
497
498
499
  const float mj = pj->mass;
  const float rhoi = pi->rho;
  const float rhoj = pj->rho;

  /* Get the kernel for hi. */
  const float hi_inv = 1.0f / hi;
500
  const float hid_inv = pow_dimension_plus_one(hi_inv); /* 1/h^(d+1) */
501
502
  const float ui = r * hi_inv;
  kernel_deval(ui, &wi, &wi_dx);
503
  const float wi_dr = hid_inv * wi_dx;
504
505
506

  /* Get the kernel for hj. */
  const float hj_inv = 1.0f / hj;
507
  const float hjd_inv = pow_dimension_plus_one(hj_inv); /* 1/h^(d+1) */
508
509
  const float xj = r * hj_inv;
  kernel_deval(xj, &wj, &wj_dx);
510
  const float wj_dr = hjd_inv * wj_dx;
511

512
513
514
515
516
  /* Compute h-gradient terms */
  const float f_i = pi->force.f;
  const float f_j = pj->force.f;

  /* Compute pressure terms */
517
518
  const float P_over_rho2_i = pi->force.P_over_rho2;
  const float P_over_rho2_j = pj->force.P_over_rho2;
519
520

  /* Compute sound speeds */
521
522
  const float ci = pi->force.soundspeed;
  const float cj = pj->force.soundspeed;
523

524
  /* Compute dv dot r. */
525
526
527
  const float dvdr = (pi->v[0] - pj->v[0]) * dx[0] +
                     (pi->v[1] - pj->v[1]) * dx[1] +
                     (pi->v[2] - pj->v[2]) * dx[2];
528

529
  /* Balsara term */
530
531
  const float balsara_i = pi->force.balsara;
  const float balsara_j = pj->force.balsara;
532
533

  /* Are the particles moving towards each others ? */
534
  const float omega_ij = (dvdr < 0.f) ? dvdr : 0.f;
535
536
537
538
539
540
541
542
543
  const float mu_ij = fac_mu * r_inv * omega_ij; /* This is 0 or negative */

  /* Signal velocity */
  const float v_sig = ci + cj - 3.f * mu_ij;

  /* Now construct the full viscosity term */
  const float rho_ij = 0.5f * (rhoi + rhoj);
  const float visc = -0.25f * const_viscosity_alpha * v_sig * mu_ij *
                     (balsara_i + balsara_j) / rho_ij;
544
545

  /* Now, convolve with the kernel */
546
  const float visc_term = 0.5f * visc * (wi_dr + wj_dr) * r_inv;
547
  const float sph_term =
548
      (f_i * P_over_rho2_i * wi_dr + f_j * P_over_rho2_j * wj_dr) * r_inv;
549
550
551

  /* Eventually got the acceleration */
  const float acc = visc_term + sph_term;
552

553
  /* Use the force Luke ! */
554
555
556
  pi->a_hydro[0] -= mj * acc * dx[0];
  pi->a_hydro[1] -= mj * acc * dx[1];
  pi->a_hydro[2] -= mj * acc * dx[2];
557

558
  /* Get the time derivative for h. */
559
  pi->force.h_dt -= mj * dvdr * r_inv / rhoj * wi_dr;
560

561
  /* Update the signal velocity. */
562
  pi->force.v_sig = (pi->force.v_sig > v_sig) ? pi->force.v_sig : v_sig;
563

564
  /* Change in entropy */
565
  pi->entropy_dt += mj * visc_term * dvdr;
566
}
567

568
#ifdef WITH_VECTORIZATION
James Willis's avatar
James Willis committed
569
570
static const vector const_viscosity_alpha_fac =
    FILL_VEC(-0.25f * const_viscosity_alpha);
571

James Willis's avatar
James Willis committed
572
573
574
575
/**
 * @brief Force interaction computed using 1 vector
 * (non-symmetric vectorized version).
 */
James Willis's avatar
James Willis committed
576
577
__attribute__((always_inline)) INLINE static void
runner_iact_nonsym_1_vec_force(
578
    vector *r2, vector *dx, vector *dy, vector *dz, vector vix, vector viy,
James Willis's avatar
James Willis committed
579
580
581
    vector viz, vector pirho, vector grad_hi, vector piPOrho2, vector balsara_i,
    vector ci, float *Vjx, float *Vjy, float *Vjz, float *Pjrho, float *Grad_hj,
    float *PjPOrho2, float *Balsara_j, float *Cj, float *Mj, vector hi_inv,
582
    vector hj_inv, vector *a_hydro_xSum, vector *a_hydro_ySum,
James Willis's avatar
James Willis committed
583
584
    vector *a_hydro_zSum, vector *h_dtSum, vector *v_sigSum,
    vector *entropy_dtSum, mask_t mask) {
585
586
587

#ifdef WITH_VECTORIZATION

588
  vector r, ri;
589
  vector dvx, dvy, dvz;
590
591
  vector xi, xj;
  vector hid_inv, hjd_inv;
592
  vector wi_dx, wj_dx, wi_dr, wj_dr, dvdr;
593
594
595
  vector piax, piay, piaz;
  vector pih_dt;
  vector v_sig;
596
  vector omega_ij, mu_ij, balsara;
597
598
599
  vector rho_ij, visc, visc_term, sph_term, acc, entropy_dt;

  /* Fill vectors. */
600
601
602
603
604
605
606
607
608
  const vector vjx = vector_load(Vjx);
  const vector vjy = vector_load(Vjy);
  const vector vjz = vector_load(Vjz);
  const vector mj = vector_load(Mj);
  const vector pjrho = vector_load(Pjrho);
  const vector grad_hj = vector_load(Grad_hj);
  const vector pjPOrho2 = vector_load(PjPOrho2);
  const vector balsara_j = vector_load(Balsara_j);
  const vector cj = vector_load(Cj);
609

610
611
  const vector fac_mu =
      vector_set1(1.f); /* Will change with cosmological integration */
612

James Willis's avatar
James Willis committed
613
  /* Load stuff. */
614
  balsara.v = vec_add(balsara_i.v, balsara_j.v);
615
616

  /* Get the radius and inverse radius. */
617
  ri = vec_reciprocal_sqrt(*r2);
618
  r.v = vec_mul(r2->v, ri.v);
619
620

  /* Get the kernel for hi. */
621
  hid_inv = pow_dimension_plus_one_vec(hi_inv);
622
  xi.v = vec_mul(r.v, hi_inv.v);
623
  kernel_eval_dWdx_force_vec(&xi, &wi_dx);
624
  wi_dr.v = vec_mul(hid_inv.v, wi_dx.v);
625
626
627

  /* Get the kernel for hj. */
  hjd_inv = pow_dimension_plus_one_vec(hj_inv);
628
  xj.v = vec_mul(r.v, hj_inv.v);
James Willis's avatar
James Willis committed
629

630
  /* Calculate the kernel. */
James Willis's avatar
James Willis committed
631
  kernel_eval_dWdx_force_vec(&xj, &wj_dx);
James Willis's avatar
James Willis committed
632

633
634
635
636
637
638
  wj_dr.v = vec_mul(hjd_inv.v, wj_dx.v);

  /* Compute dv. */
  dvx.v = vec_sub(vix.v, vjx.v);
  dvy.v = vec_sub(viy.v, vjy.v);
  dvz.v = vec_sub(viz.v, vjz.v);
639
640

  /* Compute dv dot r. */
641
  dvdr.v = vec_fma(dvx.v, dx->v, vec_fma(dvy.v, dy->v, vec_mul(dvz.v, dz->v)));
642
643
644

  /* Compute the relative velocity. (This is 0 if the particles move away from
   * each other and negative otherwise) */
645
  omega_ij.v = vec_fmin(dvdr.v, vec_setzero());
James Willis's avatar
James Willis committed
646
647
  mu_ij.v =
      vec_mul(fac_mu.v, vec_mul(ri.v, omega_ij.v)); /* This is 0 or negative */
648
649

  /* Compute signal velocity */
650
  v_sig.v = vec_fnma(vec_set1(3.f), mu_ij.v, vec_add(ci.v, cj.v));
651
652

  /* Now construct the full viscosity term */
653
  rho_ij.v = vec_mul(vec_set1(0.5f), vec_add(pirho.v, pjrho.v));
James Willis's avatar
James Willis committed
654
655
656
  visc.v = vec_div(vec_mul(const_viscosity_alpha_fac.v,
                           vec_mul(v_sig.v, vec_mul(mu_ij.v, balsara.v))),
                   rho_ij.v);
657
658

  /* Now, convolve with the kernel */
James Willis's avatar
James Willis committed
659
660
661
  visc_term.v =
      vec_mul(vec_set1(0.5f),
              vec_mul(visc.v, vec_mul(vec_add(wi_dr.v, wj_dr.v), ri.v)));
James Willis's avatar
James Willis committed
662

663
  sph_term.v =
James Willis's avatar
James Willis committed
664
665
666
      vec_mul(vec_fma(vec_mul(grad_hi.v, piPOrho2.v), wi_dr.v,
                      vec_mul(grad_hj.v, vec_mul(pjPOrho2.v, wj_dr.v))),
              ri.v);
James Willis's avatar
James Willis committed
667

668
  /* Eventually get the acceleration */
669
  acc.v = vec_add(visc_term.v, sph_term.v);
670
671

  /* Use the force, Luke! */
672
673
674
  piax.v = vec_mul(mj.v, vec_mul(dx->v, acc.v));
  piay.v = vec_mul(mj.v, vec_mul(dy->v, acc.v));
  piaz.v = vec_mul(mj.v, vec_mul(dz->v, acc.v));
675
676

  /* Get the time derivative for h. */
James Willis's avatar
James Willis committed
677
678
  pih_dt.v =
      vec_div(vec_mul(mj.v, vec_mul(dvdr.v, vec_mul(ri.v, wi_dr.v))), pjrho.v);
679
680

  /* Change in entropy */
681
  entropy_dt.v = vec_mul(mj.v, vec_mul(visc_term.v, dvdr.v));
682

683
  /* Store the forces back on the particles. */
684
685
686
687
  a_hydro_xSum->v = vec_mask_sub(a_hydro_xSum->v, piax.v, mask);
  a_hydro_ySum->v = vec_mask_sub(a_hydro_ySum->v, piay.v, mask);
  a_hydro_zSum->v = vec_mask_sub(a_hydro_zSum->v, piaz.v, mask);
  h_dtSum->v = vec_mask_sub(h_dtSum->v, pih_dt.v, mask);
688
  v_sigSum->v = vec_fmax(v_sigSum->v, vec_and_mask(v_sig.v, mask));
689
  entropy_dtSum->v = vec_mask_add(entropy_dtSum->v, entropy_dt.v, mask);
690
691
692
693
694
695
696
697
698
699

#else

  error(
      "The Gadget2 serial version of runner_iact_nonsym_force was called when "
      "the vectorised version should have been used.");

#endif
}

James Willis's avatar
James Willis committed
700
701
702
703
/**
 * @brief Force interaction computed using 2 interleaved vectors
 * (non-symmetric vectorized version).
 */
James Willis's avatar
James Willis committed
704
705
__attribute__((always_inline)) INLINE static void
runner_iact_nonsym_2_vec_force(
706
    float *R2, float *Dx, float *Dy, float *Dz, vector vix, vector viy,
James Willis's avatar
James Willis committed
707
708
709
710
711
712
    vector viz, vector pirho, vector grad_hi, vector piPOrho2, vector balsara_i,
    vector ci, float *Vjx, float *Vjy, float *Vjz, float *Pjrho, float *Grad_hj,
    float *PjPOrho2, float *Balsara_j, float *Cj, float *Mj, vector hi_inv,
    float *Hj_inv, vector *a_hydro_xSum, vector *a_hydro_ySum,
    vector *a_hydro_zSum, vector *h_dtSum, vector *v_sigSum,
    vector *entropy_dtSum, mask_t mask, mask_t mask_2, short mask_cond) {
713
714
715

#ifdef WITH_VECTORIZATION

716
717
  vector r, ri;
  vector dvx, dvy, dvz;
718
  vector ui, uj;
719
  vector hid_inv, hjd_inv;
720
  vector wi_dx, wj_dx, wi_dr, wj_dr, dvdr;
721
722
723
  vector piax, piay, piaz;
  vector pih_dt;
  vector v_sig;
724
  vector omega_ij, mu_ij, balsara;
725
726
  vector rho_ij, visc, visc_term, sph_term, acc, entropy_dt;

727
728
  vector r_2, ri_2;
  vector dvx_2, dvy_2, dvz_2;
729
  vector ui_2, uj_2;
730
  vector hjd_inv_2;
731
  vector wi_dx_2, wj_dx_2, wi_dr_2, wj_dr_2, dvdr_2;
732
733
734
735
736
737
738
  vector piax_2, piay_2, piaz_2;
  vector pih_dt_2;
  vector v_sig_2;
  vector omega_ij_2, mu_ij_2, balsara_2;
  vector rho_ij_2, visc_2, visc_term_2, sph_term_2, acc_2, entropy_dt_2;

  /* Fill vectors. */
739
740
741
742
743
744
745
746
747
748
749
750
751
752
  const vector mj = vector_load(Mj);
  const vector mj_2 = vector_load(&Mj[VEC_SIZE]);
  const vector vjx = vector_load(Vjx);
  const vector vjx_2 = vector_load(&Vjx[VEC_SIZE]);
  const vector vjy = vector_load(Vjy);
  const vector vjy_2 = vector_load(&Vjy[VEC_SIZE]);
  const vector vjz = vector_load(Vjz);
  const vector vjz_2 = vector_load(&Vjz[VEC_SIZE]);
  const vector dx = vector_load(Dx);
  const vector dx_2 = vector_load(&Dx[VEC_SIZE]);
  const vector dy = vector_load(Dy);
  const vector dy_2 = vector_load(&Dy[VEC_SIZE]);
  const vector dz = vector_load(Dz);
  const vector dz_2 = vector_load(&Dz[VEC_SIZE]);
James Willis's avatar
James Willis committed
753

754
  /* Get the radius and inverse radius. */
755
756
  const vector r2 = vector_load(R2);
  const vector r2_2 = vector_load(&R2[VEC_SIZE]);
757
758
759
760
  ri = vec_reciprocal_sqrt(r2);
  ri_2 = vec_reciprocal_sqrt(r2_2);
  r.v = vec_mul(r2.v, ri.v);
  r_2.v = vec_mul(r2_2.v, ri_2.v);
761

762
  /* Get remaining properties. */
763
764
765
766
767
768
769
770
771
772
773
774
  const vector pjrho = vector_load(Pjrho);
  const vector pjrho_2 = vector_load(&Pjrho[VEC_SIZE]);
  const vector grad_hj = vector_load(Grad_hj);
  const vector grad_hj_2 = vector_load(&Grad_hj[VEC_SIZE]);
  const vector pjPOrho2 = vector_load(PjPOrho2);
  const vector pjPOrho2_2 = vector_load(&PjPOrho2[VEC_SIZE]);
  const vector balsara_j = vector_load(Balsara_j);
  const vector balsara_j_2 = vector_load(&Balsara_j[VEC_SIZE]);
  const vector cj = vector_load(Cj);
  const vector cj_2 = vector_load(&Cj[VEC_SIZE]);
  const vector hj_inv = vector_load(Hj_inv);
  const vector hj_inv_2 = vector_load(&Hj_inv[VEC_SIZE]);
775

776
777
  const vector fac_mu =
      vector_set1(1.f); /* Will change with cosmological integration */
778

779
780
781
  /* Find the balsara switch. */
  balsara.v = vec_add(balsara_i.v, balsara_j.v);
  balsara_2.v = vec_add(balsara_i.v, balsara_j_2.v);
782
783

  /* Get the kernel for hi. */
784
  hid_inv = pow_dimension_plus_one_vec(hi_inv);
785
786
787
788
789
790
  ui.v = vec_mul(r.v, hi_inv.v);
  ui_2.v = vec_mul(r_2.v, hi_inv.v);
  kernel_eval_dWdx_force_vec(&ui, &wi_dx);
  kernel_eval_dWdx_force_vec(&ui_2, &wi_dx_2);
  wi_dr.v = vec_mul(hid_inv.v, wi_dx.v);
  wi_dr_2.v = vec_mul(hid_inv.v, wi_dx_2.v);
791
792
793
794

  /* Get the kernel for hj. */
  hjd_inv = pow_dimension_plus_one_vec(hj_inv);
  hjd_inv_2 = pow_dimension_plus_one_vec(hj_inv_2);
795
796
  uj.v = vec_mul(r.v, hj_inv.v);
  uj_2.v = vec_mul(r_2.v, hj_inv_2.v);
James Willis's avatar
James Willis committed
797

798
  /* Calculate the kernel for two particles. */
799
800
  kernel_eval_dWdx_force_vec(&uj, &wj_dx);
  kernel_eval_dWdx_force_vec(&uj_2, &wj_dx_2);
James Willis's avatar
James Willis committed
801

802
803
  wj_dr.v = vec_mul(hjd_inv.v, wj_dx.v);
  wj_dr_2.v = vec_mul(hjd_inv_2.v, wj_dx_2.v);
804

805
806
807
808
809
810
811
  /* Compute dv. */
  dvx.v = vec_sub(vix.v, vjx.v);
  dvx_2.v = vec_sub(vix.v, vjx_2.v);
  dvy.v = vec_sub(viy.v, vjy.v);
  dvy_2.v = vec_sub(viy.v, vjy_2.v);
  dvz.v = vec_sub(viz.v, vjz.v);
  dvz_2.v = vec_sub(viz.v, vjz_2.v);
812
813

  /* Compute dv dot r. */
814
  dvdr.v = vec_fma(dvx.v, dx.v, vec_fma(dvy.v, dy.v, vec_mul(dvz.v, dz.v)));
James Willis's avatar
James Willis committed
815
816
  dvdr_2.v = vec_fma(dvx_2.v, dx_2.v,
                     vec_fma(dvy_2.v, dy_2.v, vec_mul(dvz_2.v, dz_2.v)));
817
818
819
820
821

  /* Compute the relative velocity. (This is 0 if the particles move away from
   * each other and negative otherwise) */
  omega_ij.v = vec_fmin(dvdr.v, vec_setzero());
  omega_ij_2.v = vec_fmin(dvdr_2.v, vec_setzero());
James Willis's avatar
James Willis committed
822
823
824
825
  mu_ij.v =
      vec_mul(fac_mu.v, vec_mul(ri.v, omega_ij.v)); /* This is 0 or negative */
  mu_ij_2.v = vec_mul(
      fac_mu.v, vec_mul(ri_2.v, omega_ij_2.v)); /* This is 0 or negative */
826
827

  /* Compute signal velocity */
828
829
  v_sig.v = vec_fnma(vec_set1(3.f), mu_ij.v, vec_add(ci.v, cj.v));
  v_sig_2.v = vec_fnma(vec_set1(3.f), mu_ij_2.v, vec_add(ci.v, cj_2.v));
830
831

  /* Now construct the full viscosity term */
832
833
834
  rho_ij.v = vec_mul(vec_set1(0.5f), vec_add(pirho.v, pjrho.v));
  rho_ij_2.v = vec_mul(vec_set1(0.5f), vec_add(pirho.v, pjrho_2.v));

James Willis's avatar
James Willis committed
835
836
837
838
839
840
841
  visc.v = vec_div(vec_mul(const_viscosity_alpha_fac.v,
                           vec_mul(v_sig.v, vec_mul(mu_ij.v, balsara.v))),
                   rho_ij.v);
  visc_2.v =
      vec_div(vec_mul(const_viscosity_alpha_fac.v,
                      vec_mul(v_sig_2.v, vec_mul(mu_ij_2.v, balsara_2.v))),
              rho_ij_2.v);
842
843

  /* Now, convolve with the kernel */
James Willis's avatar
James Willis committed
844
845
846
847
848
849
850
  visc_term.v =
      vec_mul(vec_set1(0.5f),
              vec_mul(visc.v, vec_mul(vec_add(wi_dr.v, wj_dr.v), ri.v)));
  visc_term_2.v = vec_mul(
      vec_set1(0.5f),
      vec_mul(visc_2.v, vec_mul(vec_add(wi_dr_2.v, wj_dr_2.v), ri_2.v)));

851
852
853
  vector grad_hi_mul_piPOrho2;
  grad_hi_mul_piPOrho2.v = vec_mul(grad_hi.v, piPOrho2.v);

854
  sph_term.v =
James Willis's avatar
James Willis committed
855
856
857
858
859
860
861
      vec_mul(vec_fma(grad_hi_mul_piPOrho2.v, wi_dr.v,
                      vec_mul(grad_hj.v, vec_mul(pjPOrho2.v, wj_dr.v))),
              ri.v);
  sph_term_2.v =
      vec_mul(vec_fma(grad_hi_mul_piPOrho2.v, wi_dr_2.v,
                      vec_mul(grad_hj_2.v, vec_mul(pjPOrho2_2.v, wj_dr_2.v))),
              ri_2.v);
862
863

  /* Eventually get the acceleration */
864
865
  acc.v = vec_add(visc_term.v, sph_term.v);
  acc_2.v = vec_add(visc_term_2.v, sph_term_2.v);
866
867

  /* Use the force, Luke! */
868
869
870
871
872
873
  piax.v = vec_mul(mj.v, vec_mul(dx.v, acc.v));
  piax_2.v = vec_mul(mj_2.v, vec_mul(dx_2.v, acc_2.v));
  piay.v = vec_mul(mj.v, vec_mul(dy.v, acc.v));
  piay_2.v = vec_mul(mj_2.v, vec_mul(dy_2.v, acc_2.v));
  piaz.v = vec_mul(mj.v, vec_mul(dz.v, acc.v));
  piaz_2.v = vec_mul(mj_2.v, vec_mul(dz_2.v, acc_2.v));
874
875

  /* Get the time derivative for h. */
James Willis's avatar
James Willis committed
876
877
878
879
880
  pih_dt.v =
      vec_div(vec_mul(mj.v, vec_mul(dvdr.v, vec_mul(ri.v, wi_dr.v))), pjrho.v);
  pih_dt_2.v =
      vec_div(vec_mul(mj_2.v, vec_mul(dvdr_2.v, vec_mul(ri_2.v, wi_dr_2.v))),
              pjrho_2.v);
881
882

  /* Change in entropy */
883
884
  entropy_dt.v = vec_mul(mj.v, vec_mul(visc_term.v, dvdr.v));
  entropy_dt_2.v = vec_mul(mj_2.v, vec_mul(visc_term_2.v, dvdr_2.v));
885
886

  /* Store the forces back on the particles. */
James Willis's avatar
James Willis committed
887
  if (mask_cond) {
888
889
890
891
892
893
894
895
    a_hydro_xSum->v = vec_mask_sub(a_hydro_xSum->v, piax.v, mask);
    a_hydro_xSum->v = vec_mask_sub(a_hydro_xSum->v, piax_2.v, mask_2);
    a_hydro_ySum->v = vec_mask_sub(a_hydro_ySum->v, piay.v, mask);
    a_hydro_ySum->v = vec_mask_sub(a_hydro_ySum->v, piay_2.v, mask_2);
    a_hydro_zSum->v = vec_mask_sub(a_hydro_zSum->v, piaz.v, mask);
    a_hydro_zSum->v = vec_mask_sub(a_hydro_zSum->v, piaz_2.v, mask_2);
    h_dtSum->v = vec_mask_sub(h_dtSum->v, pih_dt.v, mask);
    h_dtSum->v = vec_mask_sub(h_dtSum->v, pih_dt_2.v, mask_2);
896
897
    v_sigSum->v = vec_fmax(v_sigSum->v, vec_and_mask(v_sig.v, mask));
    v_sigSum->v = vec_fmax(v_sigSum->v, vec_and_mask(v_sig_2.v, mask_2));
898
899
    entropy_dtSum->v = vec_mask_add(entropy_dtSum->v, entropy_dt.v, mask);
    entropy_dtSum->v = vec_mask_add(entropy_dtSum->v, entropy_dt_2.v, mask_2);
James Willis's avatar
James Willis committed
900
  } else {
901
902
903
904
905
906
907
908
909
910
911
912
913
    a_hydro_xSum->v = vec_sub(a_hydro_xSum->v, piax.v);
    a_hydro_xSum->v = vec_sub(a_hydro_xSum->v, piax_2.v);
    a_hydro_ySum->v = vec_sub(a_hydro_ySum->v, piay.v);
    a_hydro_ySum->v = vec_sub(a_hydro_ySum->v, piay_2.v);
    a_hydro_zSum->v = vec_sub(a_hydro_zSum->v, piaz.v);
    a_hydro_zSum->v = vec_sub(a_hydro_zSum->v, piaz_2.v);
    h_dtSum->v = vec_sub(h_dtSum->v, pih_dt.v);
    h_dtSum->v = vec_sub(h_dtSum->v, pih_dt_2.v);
    v_sigSum->v = vec_fmax(v_sigSum->v, v_sig.v);
    v_sigSum->v = vec_fmax(v_sigSum->v, v_sig_2.v);
    entropy_dtSum->v = vec_add(entropy_dtSum->v, entropy_dt.v);
    entropy_dtSum->v = vec_add(entropy_dtSum->v, entropy_dt_2.v);
  }
914
915
916
917
918
919
920
921
#else

  error(
      "The Gadget2 serial version of runner_iact_nonsym_force was called when "
      "the vectorised version should have been used.");

#endif
}
922

923
924
#endif

925
#endif /* SWIFT_GADGET2_HYDRO_IACT_H */