runner_doiact_vec.c 60.9 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
/*******************************************************************************
 * This file is part of SWIFT.
 * Copyright (c) 2016 James Willis (james.s.willis@durham.ac.uk)
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published
 * by the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 ******************************************************************************/

/* Config parameters. */
#include "../config.h"

/* This object's header. */
#include "runner_doiact_vec.h"

26
27
28
/* Local headers. */
#include "active.h"

29
30
31
#ifdef WITH_VECTORIZATION
static const vector kernel_gamma2_vec = FILL_VEC(kernel_gamma2);

James Willis's avatar
James Willis committed
32
33
34
/**
 * @brief Compute the vector remainder interactions from the secondary cache.
 *
Matthieu Schaller's avatar
Matthieu Schaller committed
35
 * @param int_cache (return) secondary #cache of interactions between two
James Willis's avatar
James Willis committed
36
 * particles.
James Willis's avatar
James Willis committed
37
 * @param icount Interaction count.
Matthieu Schaller's avatar
Matthieu Schaller committed
38
 * @param rhoSum (return) #vector holding the cumulative sum of the density
James Willis's avatar
James Willis committed
39
 * update on pi.
Matthieu Schaller's avatar
Matthieu Schaller committed
40
 * @param rho_dhSum (return) #vector holding the cumulative sum of the density
James Willis's avatar
James Willis committed
41
 * gradient update on pi.
Matthieu Schaller's avatar
Matthieu Schaller committed
42
 * @param wcountSum (return) #vector holding the cumulative sum of the wcount
James Willis's avatar
James Willis committed
43
 * update on pi.
Matthieu Schaller's avatar
Matthieu Schaller committed
44
 * @param wcount_dhSum (return) #vector holding the cumulative sum of the wcount
James Willis's avatar
James Willis committed
45
 * gradient update on pi.
Matthieu Schaller's avatar
Matthieu Schaller committed
46
 * @param div_vSum (return) #vector holding the cumulative sum of the divergence
James Willis's avatar
James Willis committed
47
 * update on pi.
Matthieu Schaller's avatar
Matthieu Schaller committed
48
 * @param curlvxSum (return) #vector holding the cumulative sum of the curl of
James Willis's avatar
James Willis committed
49
 * vx update on pi.
Matthieu Schaller's avatar
Matthieu Schaller committed
50
 * @param curlvySum (return) #vector holding the cumulative sum of the curl of
James Willis's avatar
James Willis committed
51
 * vy update on pi.
Matthieu Schaller's avatar
Matthieu Schaller committed
52
 * @param curlvzSum (return) #vector holding the cumulative sum of the curl of
James Willis's avatar
James Willis committed
53
 * vz update on pi.
James Willis's avatar
James Willis committed
54
55
56
57
 * @param v_hi_inv #vector of 1/h for pi.
 * @param v_vix #vector of x velocity of pi.
 * @param v_viy #vector of y velocity of pi.
 * @param v_viz #vector of z velocity of pi.
Matthieu Schaller's avatar
Matthieu Schaller committed
58
 * @param icount_align (return) Interaction count after the remainder
James Willis's avatar
James Willis committed
59
 * interactions have been performed, should be a multiple of the vector length.
James Willis's avatar
James Willis committed
60
 */
James Willis's avatar
James Willis committed
61
__attribute__((always_inline)) INLINE static void calcRemInteractions(
Matthieu Schaller's avatar
Matthieu Schaller committed
62
63
64
65
66
    struct c2_cache *const int_cache, const int icount, vector *rhoSum,
    vector *rho_dhSum, vector *wcountSum, vector *wcount_dhSum,
    vector *div_vSum, vector *curlvxSum, vector *curlvySum, vector *curlvzSum,
    vector v_hi_inv, vector v_vix, vector v_viy, vector v_viz,
    int *icount_align) {
67

68
  mask_t int_mask, int_mask2;
James Willis's avatar
James Willis committed
69
70

  /* Work out the number of remainder interactions and pad secondary cache. */
71
72
73
74
75
76
  *icount_align = icount;
  int rem = icount % (NUM_VEC_PROC * VEC_SIZE);
  if (rem != 0) {
    int pad = (NUM_VEC_PROC * VEC_SIZE) - rem;
    *icount_align += pad;

77
    /* Initialise masks to true. */
78
79
    vec_init_mask_true(int_mask);
    vec_init_mask_true(int_mask2);
80

James Willis's avatar
James Willis committed
81
82
83
    /* Pad secondary cache so that there are no contributions in the interaction
     * function. */
    for (int i = icount; i < *icount_align; i++) {
84
85
86
87
88
89
90
91
      int_cache->mq[i] = 0.f;
      int_cache->r2q[i] = 1.f;
      int_cache->dxq[i] = 0.f;
      int_cache->dyq[i] = 0.f;
      int_cache->dzq[i] = 0.f;
      int_cache->vxq[i] = 0.f;
      int_cache->vyq[i] = 0.f;
      int_cache->vzq[i] = 0.f;
92
93
94
95
    }

    /* Zero parts of mask that represent the padded values.*/
    if (pad < VEC_SIZE) {
James Willis's avatar
James Willis committed
96
      vec_pad_mask(int_mask2, pad);
James Willis's avatar
James Willis committed
97
    } else {
James Willis's avatar
James Willis committed
98
      vec_pad_mask(int_mask, VEC_SIZE - rem);
99
      vec_zero_mask(int_mask2);
100
101
    }

James Willis's avatar
James Willis committed
102
103
    /* Perform remainder interaction and remove remainder from aligned
     * interaction count. */
104
    *icount_align = icount - rem;
James Willis's avatar
James Willis committed
105
106
107
108
109
110
    runner_iact_nonsym_2_vec_density(
        &int_cache->r2q[*icount_align], &int_cache->dxq[*icount_align],
        &int_cache->dyq[*icount_align], &int_cache->dzq[*icount_align],
        v_hi_inv, v_vix, v_viy, v_viz, &int_cache->vxq[*icount_align],
        &int_cache->vyq[*icount_align], &int_cache->vzq[*icount_align],
        &int_cache->mq[*icount_align], rhoSum, rho_dhSum, wcountSum,
James Willis's avatar
James Willis committed
111
112
        wcount_dhSum, div_vSum, curlvxSum, curlvySum, curlvzSum, int_mask,
        int_mask2, 1);
113
114
115
  }
}

James Willis's avatar
James Willis committed
116
/**
James Willis's avatar
James Willis committed
117
118
 * @brief Left-packs the values needed by an interaction into the secondary
 * cache (Supports AVX, AVX2 and AVX512 instruction sets).
James Willis's avatar
James Willis committed
119
120
 *
 * @param mask Contains which particles need to interact.
Matthieu Schaller's avatar
Matthieu Schaller committed
121
 * @param pjd Index of the particle to store into.
James Willis's avatar
James Willis committed
122
123
124
125
126
 * @param v_r2 #vector of the separation between two particles squared.
 * @param v_dx #vector of the x separation between two particles.
 * @param v_dy #vector of the y separation between two particles.
 * @param v_dz #vector of the z separation between two particles.
 * @param cell_cache #cache of all particles in the cell.
Matthieu Schaller's avatar
Matthieu Schaller committed
127
 * @param int_cache (return) secondary #cache of interactions between two
James Willis's avatar
James Willis committed
128
 * particles.
James Willis's avatar
James Willis committed
129
130
 * @param icount Interaction count.
 * @param rhoSum #vector holding the cumulative sum of the density update on pi.
James Willis's avatar
James Willis committed
131
132
133
134
135
136
137
138
139
140
141
142
143
144
 * @param rho_dhSum #vector holding the cumulative sum of the density gradient
 * update on pi.
 * @param wcountSum #vector holding the cumulative sum of the wcount update on
 * pi.
 * @param wcount_dhSum #vector holding the cumulative sum of the wcount gradient
 * update on pi.
 * @param div_vSum #vector holding the cumulative sum of the divergence update
 * on pi.
 * @param curlvxSum #vector holding the cumulative sum of the curl of vx update
 * on pi.
 * @param curlvySum #vector holding the cumulative sum of the curl of vy update
 * on pi.
 * @param curlvzSum #vector holding the cumulative sum of the curl of vz update
 * on pi.
James Willis's avatar
James Willis committed
145
146
147
148
149
 * @param v_hi_inv #vector of 1/h for pi.
 * @param v_vix #vector of x velocity of pi.
 * @param v_viy #vector of y velocity of pi.
 * @param v_viz #vector of z velocity of pi.
 */
James Willis's avatar
James Willis committed
150
__attribute__((always_inline)) INLINE static void storeInteractions(
151
    const int mask, const int pjd, vector *v_r2, vector *v_dx, vector *v_dy,
James Willis's avatar
James Willis committed
152
153
154
155
156
    vector *v_dz, const struct cache *const cell_cache,
    struct c2_cache *const int_cache, int *icount, vector *rhoSum,
    vector *rho_dhSum, vector *wcountSum, vector *wcount_dhSum,
    vector *div_vSum, vector *curlvxSum, vector *curlvySum, vector *curlvzSum,
    vector v_hi_inv, vector v_vix, vector v_viy, vector v_viz) {
James Willis's avatar
James Willis committed
157
158
159

/* Left-pack values needed into the secondary cache using the interaction mask.
 */
160
#if defined(HAVE_AVX2) || defined(HAVE_AVX512_F)
161
162
163
164
165
166
167
  mask_t packed_mask;
  VEC_FORM_PACKED_MASK(mask, packed_mask);

  VEC_LEFT_PACK(v_r2->v, packed_mask, &int_cache->r2q[*icount]);
  VEC_LEFT_PACK(v_dx->v, packed_mask, &int_cache->dxq[*icount]);
  VEC_LEFT_PACK(v_dy->v, packed_mask, &int_cache->dyq[*icount]);
  VEC_LEFT_PACK(v_dz->v, packed_mask, &int_cache->dzq[*icount]);
James Willis's avatar
James Willis committed
168
169
170
171
172
173
174
175
  VEC_LEFT_PACK(vec_load(&cell_cache->m[pjd]), packed_mask,
                &int_cache->mq[*icount]);
  VEC_LEFT_PACK(vec_load(&cell_cache->vx[pjd]), packed_mask,
                &int_cache->vxq[*icount]);
  VEC_LEFT_PACK(vec_load(&cell_cache->vy[pjd]), packed_mask,
                &int_cache->vyq[*icount]);
  VEC_LEFT_PACK(vec_load(&cell_cache->vz[pjd]), packed_mask,
                &int_cache->vzq[*icount]);
176
177
178

  /* Increment interaction count by number of bits set in mask. */
  (*icount) += __builtin_popcount(mask);
179
#else
James Willis's avatar
James Willis committed
180
  /* Quicker to do it serially in AVX rather than use intrinsics. */
James Willis's avatar
James Willis committed
181
  for (int bit_index = 0; bit_index < VEC_SIZE; bit_index++) {
182
183
    if (mask & (1 << bit_index)) {
      /* Add this interaction to the queue. */
184
185
186
187
188
189
190
191
      int_cache->r2q[*icount] = v_r2->f[bit_index];
      int_cache->dxq[*icount] = v_dx->f[bit_index];
      int_cache->dyq[*icount] = v_dy->f[bit_index];
      int_cache->dzq[*icount] = v_dz->f[bit_index];
      int_cache->mq[*icount] = cell_cache->m[pjd + bit_index];
      int_cache->vxq[*icount] = cell_cache->vx[pjd + bit_index];
      int_cache->vyq[*icount] = cell_cache->vy[pjd + bit_index];
      int_cache->vzq[*icount] = cell_cache->vz[pjd + bit_index];
192
193
194
195

      (*icount)++;
    }
  }
196

James Willis's avatar
James Willis committed
197
198
#endif /* defined(HAVE_AVX2) || defined(HAVE_AVX512_F) */

James Willis's avatar
James Willis committed
199
  /* Flush the c2 cache if it has reached capacity. */
James Willis's avatar
James Willis committed
200
  if (*icount >= (C2_CACHE_SIZE - (NUM_VEC_PROC * VEC_SIZE))) {
201
202

    int icount_align = *icount;
James Willis's avatar
James Willis committed
203

James Willis's avatar
James Willis committed
204
    /* Peform remainder interactions. */
Matthieu Schaller's avatar
Matthieu Schaller committed
205
206
207
    calcRemInteractions(int_cache, *icount, rhoSum, rho_dhSum, wcountSum,
                        wcount_dhSum, div_vSum, curlvxSum, curlvySum, curlvzSum,
                        v_hi_inv, v_vix, v_viy, v_viz, &icount_align);
208

209
    mask_t int_mask, int_mask2;
210
211
    vec_init_mask_true(int_mask);
    vec_init_mask_true(int_mask2);
James Willis's avatar
James Willis committed
212
213

    /* Perform interactions. */
James Willis's avatar
James Willis committed
214
215
216
217
218
219
    for (int pjd = 0; pjd < icount_align; pjd += (NUM_VEC_PROC * VEC_SIZE)) {
      runner_iact_nonsym_2_vec_density(
          &int_cache->r2q[pjd], &int_cache->dxq[pjd], &int_cache->dyq[pjd],
          &int_cache->dzq[pjd], v_hi_inv, v_vix, v_viy, v_viz,
          &int_cache->vxq[pjd], &int_cache->vyq[pjd], &int_cache->vzq[pjd],
          &int_cache->mq[pjd], rhoSum, rho_dhSum, wcountSum, wcount_dhSum,
220
          div_vSum, curlvxSum, curlvySum, curlvzSum, int_mask, int_mask2, 0);
221
    }
James Willis's avatar
James Willis committed
222
223

    /* Reset interaction count. */
224
225
226
    *icount = 0;
  }
}
227

228
/**
James Willis's avatar
James Willis committed
229
230
 * @brief Compute the vector remainder force interactions from the secondary
 * cache.
231
232
233
234
 *
 * @param int_cache (return) secondary #cache of interactions between two
 * particles.
 * @param icount Interaction count.
James Willis's avatar
James Willis committed
235
236
 * @param a_hydro_xSum (return) #vector holding the cumulative sum of the x
 * acceleration
237
 * update on pi.
James Willis's avatar
James Willis committed
238
239
 * @param a_hydro_ySum (return) #vector holding the cumulative sum of the y
 * acceleration
240
 * update on pi.
James Willis's avatar
James Willis committed
241
242
 * @param a_hydro_zSum (return) #vector holding the cumulative sum of the z
 * acceleration
James Willis's avatar
James Willis committed
243
 * update on pi.
James Willis's avatar
James Willis committed
244
245
246
247
248
249
 * @param h_dtSum (return) #vector holding the cumulative sum of the time
 * derivative of the smoothing length update on pi.
 * @param v_sigSum (return) #vector holding the maximum of the signal velocity
 * update on pi.
 * @param entropy_dtSum (return) #vector holding the cumulative sum of the time
 * derivative of the entropy
250
251
252
253
254
 * update on pi.
 * @param v_hi_inv #vector of 1/h for pi.
 * @param v_vix #vector of x velocity of pi.
 * @param v_viy #vector of y velocity of pi.
 * @param v_viz #vector of z velocity of pi.
James Willis's avatar
James Willis committed
255
256
257
258
259
 * @param v_rhoi #vector of density of pi.
 * @param v_grad_hi #vector of smoothing length gradient of pi.
 * @param v_pOrhoi2 #vector of pressure over density squared of pi.
 * @param v_balsara_i #vector of balsara switch of pi.
 * @param v_ci #vector of sound speed of pi.
260
 * @param icount_align (return) Interaction count after the remainder
James Willis's avatar
James Willis committed
261
262
 * @param num_vec_proc #int of the number of vectors to use to perform
 * interaction.
263
264
265
266
267
 * interactions have been performed, should be a multiple of the vector length.
 */
__attribute__((always_inline)) INLINE static void calcRemForceInteractions(
    struct c2_cache *const int_cache, const int icount, vector *a_hydro_xSum,
    vector *a_hydro_ySum, vector *a_hydro_zSum, vector *h_dtSum,
268
269
270
    vector *v_sigSum, vector *entropy_dtSum, vector v_hi_inv, vector v_vix,
    vector v_viy, vector v_viz, vector v_rhoi, vector v_grad_hi,
    vector v_pOrhoi2, vector v_balsara_i, vector v_ci, int *icount_align,
James Willis's avatar
James Willis committed
271
    int num_vec_proc) {
272

273
  mask_t int_mask, int_mask2;
274
275
276

  /* Work out the number of remainder interactions and pad secondary cache. */
  *icount_align = icount;
277
  int rem = icount % (num_vec_proc * VEC_SIZE);
278
  if (rem != 0) {
279
    int pad = (num_vec_proc * VEC_SIZE) - rem;
280
281
    *icount_align += pad;

282
    /* Initialise masks to true. */
283
284
    vec_init_mask_true(int_mask);
    vec_init_mask_true(int_mask2);
285

286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
    /* Pad secondary cache so that there are no contributions in the interaction
     * function. */
    for (int i = icount; i < *icount_align; i++) {
      int_cache->mq[i] = 0.f;
      int_cache->r2q[i] = 1.f;
      int_cache->dxq[i] = 0.f;
      int_cache->dyq[i] = 0.f;
      int_cache->dzq[i] = 0.f;
      int_cache->vxq[i] = 0.f;
      int_cache->vyq[i] = 0.f;
      int_cache->vzq[i] = 0.f;
      int_cache->rhoq[i] = 1.f;
      int_cache->grad_hq[i] = 1.f;
      int_cache->pOrho2q[i] = 1.f;
      int_cache->balsaraq[i] = 1.f;
      int_cache->soundspeedq[i] = 1.f;
      int_cache->h_invq[i] = 1.f;
    }

    /* Zero parts of mask that represent the padded values.*/
    if (pad < VEC_SIZE) {
James Willis's avatar
James Willis committed
307
      vec_pad_mask(int_mask2, pad);
308
    } else {
James Willis's avatar
James Willis committed
309
      vec_pad_mask(int_mask, VEC_SIZE - rem);
310
      vec_zero_mask(int_mask2);
311
312
313
314
315
    }

    /* Perform remainder interaction and remove remainder from aligned
     * interaction count. */
    *icount_align = icount - rem;
316
    runner_iact_nonsym_2_vec_force(
James Willis's avatar
James Willis committed
317
318
319
320
321
322
323
324
325
326
        &int_cache->r2q[*icount_align], &int_cache->dxq[*icount_align],
        &int_cache->dyq[*icount_align], &int_cache->dzq[*icount_align], v_vix,
        v_viy, v_viz, v_rhoi, v_grad_hi, v_pOrhoi2, v_balsara_i, v_ci,
        &int_cache->vxq[*icount_align], &int_cache->vyq[*icount_align],
        &int_cache->vzq[*icount_align], &int_cache->rhoq[*icount_align],
        &int_cache->grad_hq[*icount_align], &int_cache->pOrho2q[*icount_align],
        &int_cache->balsaraq[*icount_align],
        &int_cache->soundspeedq[*icount_align], &int_cache->mq[*icount_align],
        v_hi_inv, &int_cache->h_invq[*icount_align], a_hydro_xSum, a_hydro_ySum,
        a_hydro_zSum, h_dtSum, v_sigSum, entropy_dtSum, int_mask, int_mask2, 1);
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
  }
}

/**
 * @brief Left-packs the values needed by an interaction into the secondary
 * cache (Supports AVX, AVX2 and AVX512 instruction sets).
 *
 * @param mask Contains which particles need to interact.
 * @param pjd Index of the particle to store into.
 * @param v_r2 #vector of the separation between two particles squared.
 * @param v_dx #vector of the x separation between two particles.
 * @param v_dy #vector of the y separation between two particles.
 * @param v_dz #vector of the z separation between two particles.
 * @param cell_cache #cache of all particles in the cell.
 * @param int_cache (return) secondary #cache of interactions between two
 * particles.
 * @param icount Interaction count.
James Willis's avatar
James Willis committed
344
345
346
347
348
 * @param a_hydro_xSum (return) #vector holding the cumulative sum of the x
 * acceleration
 * update on pi.
 * @param a_hydro_ySum (return) #vector holding the cumulative sum of the y
 * acceleration
349
 * update on pi.
James Willis's avatar
James Willis committed
350
351
 * @param a_hydro_zSum (return) #vector holding the cumulative sum of the z
 * acceleration
James Willis's avatar
James Willis committed
352
 * update on pi.
James Willis's avatar
James Willis committed
353
354
355
 * @param h_dtSum (return) #vector holding the cumulative sum of the time
 * derivative of the smoothing length update on pi.
 * @param v_sigSum (return) #vector holding the maximum of the signal velocity
James Willis's avatar
James Willis committed
356
 * update on pi.
James Willis's avatar
James Willis committed
357
358
 * @param entropy_dtSum (return) #vector holding the cumulative sum of the time
 * derivative of the entropy
359
360
361
362
363
 * update on pi.
 * @param v_hi_inv #vector of 1/h for pi.
 * @param v_vix #vector of x velocity of pi.
 * @param v_viy #vector of y velocity of pi.
 * @param v_viz #vector of z velocity of pi.
James Willis's avatar
James Willis committed
364
365
366
367
368
 * @param v_rhoi #vector of density of pi.
 * @param v_grad_hi #vector of smoothing length gradient of pi.
 * @param v_pOrhoi2 #vector of pressure over density squared of pi.
 * @param v_balsara_i #vector of balsara switch of pi.
 * @param v_ci #vector of sound speed of pi.
James Willis's avatar
James Willis committed
369
370
 * @param num_vec_proc #int of the number of vectors to use to perform
 * interaction.
James Willis's avatar
James Willis committed
371
 * interactions have been performed, should be a multiple of the vector length.
372
373
374
 */
__attribute__((always_inline)) INLINE static void storeForceInteractions(
    const int mask, const int pjd, vector *v_r2, vector *v_dx, vector *v_dy,
James Willis's avatar
James Willis committed
375
376
377
    vector *v_dz, const struct cache *const cell_cache,
    struct c2_cache *const int_cache, int *icount, vector *a_hydro_xSum,
    vector *a_hydro_ySum, vector *a_hydro_zSum, vector *h_dtSum,
378
379
    vector *v_sigSum, vector *entropy_dtSum, vector v_hi_inv, vector v_vix,
    vector v_viy, vector v_viz, vector v_rhoi, vector v_grad_hi,
380
    vector v_pOrhoi2, vector v_balsara_i, vector v_ci, int num_vec_proc) {
381
382
383
384

/* Left-pack values needed into the secondary cache using the interaction mask.
 */
#if defined(HAVE_AVX2) || defined(HAVE_AVX512_F)
385
386
  /* Invert hj. */
  vector v_hj, v_hj_inv;
387
  v_hj.v = vec_load(&cell_cache->h[pjd]);
388
389
  v_hj_inv = vec_reciprocal(v_hj);

390
391
  mask_t packed_mask;
  VEC_FORM_PACKED_MASK(mask, packed_mask);
James Willis's avatar
James Willis committed
392

393
394
395
396
  VEC_LEFT_PACK(v_r2->v, packed_mask, &int_cache->r2q[*icount]);
  VEC_LEFT_PACK(v_dx->v, packed_mask, &int_cache->dxq[*icount]);
  VEC_LEFT_PACK(v_dy->v, packed_mask, &int_cache->dyq[*icount]);
  VEC_LEFT_PACK(v_dz->v, packed_mask, &int_cache->dzq[*icount]);
James Willis's avatar
James Willis committed
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
  VEC_LEFT_PACK(vec_load(&cell_cache->m[pjd]), packed_mask,
                &int_cache->mq[*icount]);
  VEC_LEFT_PACK(vec_load(&cell_cache->vx[pjd]), packed_mask,
                &int_cache->vxq[*icount]);
  VEC_LEFT_PACK(vec_load(&cell_cache->vy[pjd]), packed_mask,
                &int_cache->vyq[*icount]);
  VEC_LEFT_PACK(vec_load(&cell_cache->vz[pjd]), packed_mask,
                &int_cache->vzq[*icount]);
  VEC_LEFT_PACK(vec_load(&cell_cache->rho[pjd]), packed_mask,
                &int_cache->rhoq[*icount]);
  VEC_LEFT_PACK(vec_load(&cell_cache->grad_h[pjd]), packed_mask,
                &int_cache->grad_hq[*icount]);
  VEC_LEFT_PACK(vec_load(&cell_cache->pOrho2[pjd]), packed_mask,
                &int_cache->pOrho2q[*icount]);
  VEC_LEFT_PACK(vec_load(&cell_cache->balsara[pjd]), packed_mask,
                &int_cache->balsaraq[*icount]);
  VEC_LEFT_PACK(vec_load(&cell_cache->soundspeed[pjd]), packed_mask,
                &int_cache->soundspeedq[*icount]);
415
  VEC_LEFT_PACK(v_hj_inv.v, packed_mask, &int_cache->h_invq[*icount]);
James Willis's avatar
James Willis committed
416

417
418
  /* Increment interaction count by number of bits set in mask. */
  (*icount) += __builtin_popcount(mask);
419
420
421
422
423
424
425
426
427
428
429
430
431
#else
  /* Quicker to do it serially in AVX rather than use intrinsics. */
  for (int bit_index = 0; bit_index < VEC_SIZE; bit_index++) {
    if (mask & (1 << bit_index)) {
      /* Add this interaction to the queue. */
      int_cache->r2q[*icount] = v_r2->f[bit_index];
      int_cache->dxq[*icount] = v_dx->f[bit_index];
      int_cache->dyq[*icount] = v_dy->f[bit_index];
      int_cache->dzq[*icount] = v_dz->f[bit_index];
      int_cache->mq[*icount] = cell_cache->m[pjd + bit_index];
      int_cache->vxq[*icount] = cell_cache->vx[pjd + bit_index];
      int_cache->vyq[*icount] = cell_cache->vy[pjd + bit_index];
      int_cache->vzq[*icount] = cell_cache->vz[pjd + bit_index];
James Willis's avatar
James Willis committed
432

433
434
435
436
437
      int_cache->rhoq[*icount] = cell_cache->rho[pjd + bit_index];
      int_cache->grad_hq[*icount] = cell_cache->grad_h[pjd + bit_index];
      int_cache->pOrho2q[*icount] = cell_cache->pOrho2[pjd + bit_index];
      int_cache->balsaraq[*icount] = cell_cache->balsara[pjd + bit_index];
      int_cache->soundspeedq[*icount] = cell_cache->soundspeed[pjd + bit_index];
438
      int_cache->h_invq[*icount] = 1.f / cell_cache->h[pjd + bit_index];
439
440
441
442
443
444
445
446

      (*icount)++;
    }
  }

#endif /* defined(HAVE_AVX2) || defined(HAVE_AVX512_F) */

  /* Flush the c2 cache if it has reached capacity. */
447
  if (*icount >= (C2_CACHE_SIZE - (num_vec_proc * VEC_SIZE))) {
448
449
450
451

    int icount_align = *icount;

    /* Peform remainder interactions. */
James Willis's avatar
James Willis committed
452
453
454
455
    calcRemForceInteractions(int_cache, *icount, a_hydro_xSum, a_hydro_ySum,
                             a_hydro_zSum, h_dtSum, v_sigSum, entropy_dtSum,
                             v_hi_inv, v_vix, v_viy, v_viz, v_rhoi, v_grad_hi,
                             v_pOrhoi2, v_balsara_i, v_ci, &icount_align, 2);
456

457
458
459
    /* Initialise masks to true in case remainder interactions have been
     * performed. */
    mask_t int_mask, int_mask2;
460
461
    vec_init_mask_true(int_mask);
    vec_init_mask_true(int_mask2);
462

463
    /* Perform interactions. */
464
    for (int pjd = 0; pjd < icount_align; pjd += (num_vec_proc * VEC_SIZE)) {
465

466
      runner_iact_nonsym_2_vec_force(
James Willis's avatar
James Willis committed
467
468
469
470
471
472
473
474
475
          &int_cache->r2q[pjd], &int_cache->dxq[pjd], &int_cache->dyq[pjd],
          &int_cache->dzq[pjd], v_vix, v_viy, v_viz, v_rhoi, v_grad_hi,
          v_pOrhoi2, v_balsara_i, v_ci, &int_cache->vxq[pjd],
          &int_cache->vyq[pjd], &int_cache->vzq[pjd], &int_cache->rhoq[pjd],
          &int_cache->grad_hq[pjd], &int_cache->pOrho2q[pjd],
          &int_cache->balsaraq[pjd], &int_cache->soundspeedq[pjd],
          &int_cache->mq[pjd], v_hi_inv, &int_cache->h_invq[pjd], a_hydro_xSum,
          a_hydro_ySum, a_hydro_zSum, h_dtSum, v_sigSum, entropy_dtSum,
          int_mask, int_mask2, 0);
476
477
478
479
480
481
482
    }

    /* Reset interaction count. */
    *icount = 0;
  }
}

483
/**
484
485
 * @brief Populates the arrays max_index_i and max_index_j with the maximum
 * indices of
James Willis's avatar
James Willis committed
486
487
488
 * particles into their neighbouring cells. Also finds the first pi that
 * interacts with any particle in cj and the last pj that interacts with any
 * particle in ci.
489
 *
James Willis's avatar
James Willis committed
490
491
492
493
494
495
 * @param ci #cell pointer to ci
 * @param cj #cell pointer to cj
 * @param sort_i #entry array for particle distance in ci
 * @param sort_j #entry array for particle distance in cj
 * @param dx_max maximum particle movement allowed in cell
 * @param rshift cutoff shift
496
497
498
499
 * @param hi_max Maximal smoothing length in cell ci
 * @param hj_max Maximal smoothing length in cell cj
 * @param di_max Maximal position on the axis that can interact in cell ci
 * @param dj_min Minimal position on the axis that can interact in cell ci
500
501
 * @param max_index_i array to hold the maximum distances of pi particles into
 * cell
James Willis's avatar
James Willis committed
502
 * cj
503
504
 * @param max_index_j array to hold the maximum distances of pj particles into
 * cell
James Willis's avatar
James Willis committed
505
 * cj
James Willis's avatar
James Willis committed
506
507
 * @param init_pi first pi to interact with a pj particle
 * @param init_pj last pj to interact with a pi particle
508
 * @param e The #engine.
James Willis's avatar
James Willis committed
509
 */
510
__attribute__((always_inline)) INLINE static void populate_max_index_no_cache(
James Willis's avatar
James Willis committed
511
512
    const struct cell *ci, const struct cell *cj,
    const struct entry *restrict sort_i, const struct entry *restrict sort_j,
513
514
    const float dx_max, const float rshift, const double hi_max,
    const double hj_max, const double di_max, const double dj_min,
515
    int *max_index_i, int *max_index_j, int *init_pi, int *init_pj,
516
    const struct engine *e) {
517

518
519
  const struct part *restrict parts_i = ci->parts;
  const struct part *restrict parts_j = cj->parts;
520
521

  int first_pi = 0, last_pj = cj->count - 1;
522
  int temp;
523

James Willis's avatar
James Willis committed
524
525
  /* Find the leftmost active particle in cell i that interacts with any
   * particle in cell j. */
526
  first_pi = ci->count;
527
  int active_id = first_pi - 1;
528
  while (first_pi > 0 && sort_i[first_pi - 1].d + dx_max + hi_max > dj_min) {
529
    first_pi--;
530
531
532
    /* Store the index of the particle if it is active. */
    if (part_is_active(&parts_i[sort_i[first_pi].i], e)) active_id = first_pi;
  }
James Willis's avatar
James Willis committed
533

534
535
  /* Set the first active pi in range of any particle in cell j. */
  first_pi = active_id;
536

537
  /* Find the maximum index into cell j for each particle in range in cell i. */
James Willis's avatar
James Willis committed
538
  if (first_pi < ci->count) {
539

540
541
    /* Start from the first particle in cell j. */
    temp = 0;
542

543
    const struct part *pi = &parts_i[sort_i[first_pi].i];
544

545
    /* Loop through particles in cell j until they are not in range of pi. */
James Willis's avatar
James Willis committed
546
547
548
    while (temp <= cj->count &&
           (sort_i[first_pi].d + (pi->h * kernel_gamma + dx_max - rshift) >
            sort_j[temp].d))
549
      temp++;
550

551
    max_index_i[first_pi] = temp;
552

553
    /* Populate max_index_i for remaining particles that are within range. */
James Willis's avatar
James Willis committed
554
    for (int i = first_pi + 1; i < ci->count; i++) {
555
      temp = max_index_i[i - 1];
556
      pi = &parts_i[sort_i[i].i];
557

James Willis's avatar
James Willis committed
558
559
560
      while (temp <= cj->count &&
             (sort_i[i].d + (pi->h * kernel_gamma + dx_max - rshift) >
              sort_j[temp].d))
561
        temp++;
562

563
      max_index_i[i] = temp;
564
    }
James Willis's avatar
James Willis committed
565
  } else {
566
567
    /* Make sure that max index is set to first particle in cj.*/
    max_index_i[ci->count - 1] = 0;
568
569
  }

James Willis's avatar
James Willis committed
570
571
  /* Find the rightmost active particle in cell j that interacts with any
   * particle in cell i. */
572
  last_pj = -1;
573
  active_id = last_pj;
James Willis's avatar
James Willis committed
574
575
  while (last_pj < cj->count &&
         sort_j[last_pj + 1].d - hj_max - dx_max < di_max) {
576
    last_pj++;
577
    /* Store the index of the particle if it is active. */
578
    if (part_is_active(&parts_j[sort_j[last_pj].i], e)) active_id = last_pj;
579
  }
James Willis's avatar
James Willis committed
580

581
  /* Set the last active pj in range of any particle in cell i. */
582
  last_pj = active_id;
Matthieu Schaller's avatar
Matthieu Schaller committed
583

584
  /* Find the maximum index into cell i for each particle in range in cell j. */
585
  if (last_pj > 0) {
Matthieu Schaller's avatar
Matthieu Schaller committed
586

587
588
    /* Start from the last particle in cell i. */
    temp = ci->count - 1;
589

590
    const struct part *pj = &parts_j[sort_j[last_pj].i];
591

592
    /* Loop through particles in cell i until they are not in range of pj. */
James Willis's avatar
James Willis committed
593
594
595
    while (temp > 0 &&
           sort_j[last_pj].d - dx_max - (pj->h * kernel_gamma) <
               sort_i[temp].d - rshift)
596
      temp--;
597

598
    max_index_j[last_pj] = temp;
599

600
    /* Populate max_index_j for remaining particles that are within range. */
James Willis's avatar
James Willis committed
601
    for (int i = last_pj - 1; i >= 0; i--) {
602
      temp = max_index_j[i + 1];
603
      pj = &parts_j[sort_j[i].i];
604

James Willis's avatar
James Willis committed
605
606
607
      while (temp > 0 &&
             sort_j[i].d - dx_max - (pj->h * kernel_gamma) <
                 sort_i[temp].d - rshift)
608
        temp--;
609

610
611
      max_index_j[i] = temp;
    }
James Willis's avatar
James Willis committed
612
  } else {
613
    /* Make sure that max index is set to last particle in ci.*/
James Willis's avatar
James Willis committed
614
    max_index_j[0] = ci->count - 1;
615
616
  }

James Willis's avatar
James Willis committed
617
618
  *init_pi = first_pi;
  *init_pj = last_pj;
619
}
James Willis's avatar
James Willis committed
620
#endif /* WITH_VECTORIZATION */
621
622

/**
James Willis's avatar
James Willis committed
623
624
 * @brief Compute the cell self-interaction (non-symmetric) using vector
 * intrinsics with one particle pi at a time.
625
626
627
628
 *
 * @param r The #runner.
 * @param c The #cell.
 */
James Willis's avatar
James Willis committed
629
630
__attribute__((always_inline)) INLINE void runner_doself1_density_vec(
    struct runner *r, struct cell *restrict c) {
631
632

#ifdef WITH_VECTORIZATION
633
  const struct engine *e = r->e;
634
635
636
637
638
639
  struct part *restrict pi;
  int count_align;
  int num_vec_proc = NUM_VEC_PROC;

  struct part *restrict parts = c->parts;
  const int count = c->count;
James Willis's avatar
James Willis committed
640

641
642
  vector v_hi, v_vix, v_viy, v_viz, v_hig2, v_r2;

James Willis's avatar
James Willis committed
643
  TIMER_TIC
644

645
646
  if (!cell_is_active(c, e)) return;

647
  if (!cell_are_part_drifted(c, e)) error("Interacting undrifted cell.");
648

James Willis's avatar
James Willis committed
649
  /* Get the particle cache from the runner and re-allocate
650
   * the cache if it is not big enough for the cell. */
651
  struct cache *restrict cell_cache = &r->ci_cache;
James Willis's avatar
James Willis committed
652
653
654

  if (cell_cache->count < count) {
    cache_init(cell_cache, count);
655
656
  }

James Willis's avatar
James Willis committed
657
  /* Read the particles from the cell and store them locally in the cache. */
James Willis's avatar
James Willis committed
658
  cache_read_particles(c, cell_cache);
659
660
661
662

  /* Create secondary cache to store particle interactions. */
  struct c2_cache int_cache;
  int icount = 0, icount_align = 0;
663
664
665
666
667
668
669
670

  /* Loop over the particles in the cell. */
  for (int pid = 0; pid < count; pid++) {

    /* Get a pointer to the ith particle. */
    pi = &parts[pid];

    /* Is the ith particle active? */
671
    if (!part_is_active(pi, e)) continue;
672
673
674
675
676

    vector pix, piy, piz;

    const float hi = cell_cache->h[pid];

James Willis's avatar
James Willis committed
677
    /* Fill particle pi vectors. */
678
679
680
681
682
683
684
685
686
687
688
    pix.v = vec_set1(cell_cache->x[pid]);
    piy.v = vec_set1(cell_cache->y[pid]);
    piz.v = vec_set1(cell_cache->z[pid]);
    v_hi.v = vec_set1(hi);
    v_vix.v = vec_set1(cell_cache->vx[pid]);
    v_viy.v = vec_set1(cell_cache->vy[pid]);
    v_viz.v = vec_set1(cell_cache->vz[pid]);

    const float hig2 = hi * hi * kernel_gamma2;
    v_hig2.v = vec_set1(hig2);

James Willis's avatar
James Willis committed
689
    /* Reset cumulative sums of update vectors. */
James Willis's avatar
James Willis committed
690
691
692
    vector rhoSum, rho_dhSum, wcountSum, wcount_dhSum, div_vSum, curlvxSum,
        curlvySum, curlvzSum;

James Willis's avatar
James Willis committed
693
    /* Get the inverse of hi. */
694
    vector v_hi_inv;
James Willis's avatar
James Willis committed
695

696
    v_hi_inv = vec_reciprocal(v_hi);
James Willis's avatar
James Willis committed
697

698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
    rhoSum.v = vec_setzero();
    rho_dhSum.v = vec_setzero();
    wcountSum.v = vec_setzero();
    wcount_dhSum.v = vec_setzero();
    div_vSum.v = vec_setzero();
    curlvxSum.v = vec_setzero();
    curlvySum.v = vec_setzero();
    curlvzSum.v = vec_setzero();

    /* Pad cache if there is a serial remainder. */
    count_align = count;
    int rem = count % (num_vec_proc * VEC_SIZE);
    if (rem != 0) {
      int pad = (num_vec_proc * VEC_SIZE) - rem;

      count_align += pad;
714
715
716
717
718
719
720
721

      /* Set positions to the same as particle pi so when the r2 > 0 mask is
       * applied these extra contributions are masked out.*/
      for (int i = count; i < count_align; i++) {
        cell_cache->x[i] = pix.f[0];
        cell_cache->y[i] = piy.f[0];
        cell_cache->z[i] = piz.f[0];
      }
722
723
724
725
726
    }

    vector pjx, pjy, pjz;
    vector pjx2, pjy2, pjz2;

James Willis's avatar
James Willis committed
727
728
    /* Find all of particle pi's interacions and store needed values in the
     * secondary cache.*/
729
730
731
732
733
734
    for (int pjd = 0; pjd < count_align; pjd += (num_vec_proc * VEC_SIZE)) {

      /* Load 2 sets of vectors from the particle cache. */
      pjx.v = vec_load(&cell_cache->x[pjd]);
      pjy.v = vec_load(&cell_cache->y[pjd]);
      pjz.v = vec_load(&cell_cache->z[pjd]);
735

736
737
738
      pjx2.v = vec_load(&cell_cache->x[pjd + VEC_SIZE]);
      pjy2.v = vec_load(&cell_cache->y[pjd + VEC_SIZE]);
      pjz2.v = vec_load(&cell_cache->z[pjd + VEC_SIZE]);
James Willis's avatar
James Willis committed
739

740
741
742
743
      /* Compute the pairwise distance. */
      vector v_dx_tmp, v_dy_tmp, v_dz_tmp;
      vector v_dx_tmp2, v_dy_tmp2, v_dz_tmp2, v_r2_2;

James Willis's avatar
James Willis committed
744
745
      v_dx_tmp.v = vec_sub(pix.v, pjx.v);
      v_dx_tmp2.v = vec_sub(pix.v, pjx2.v);
746
      v_dy_tmp.v = vec_sub(piy.v, pjy.v);
James Willis's avatar
James Willis committed
747
      v_dy_tmp2.v = vec_sub(piy.v, pjy2.v);
748
      v_dz_tmp.v = vec_sub(piz.v, pjz.v);
James Willis's avatar
James Willis committed
749
750
751
752
      v_dz_tmp2.v = vec_sub(piz.v, pjz2.v);

      v_r2.v = vec_mul(v_dx_tmp.v, v_dx_tmp.v);
      v_r2_2.v = vec_mul(v_dx_tmp2.v, v_dx_tmp2.v);
753
      v_r2.v = vec_fma(v_dy_tmp.v, v_dy_tmp.v, v_r2.v);
James Willis's avatar
James Willis committed
754
      v_r2_2.v = vec_fma(v_dy_tmp2.v, v_dy_tmp2.v, v_r2_2.v);
755
      v_r2.v = vec_fma(v_dz_tmp.v, v_dz_tmp.v, v_r2.v);
James Willis's avatar
James Willis committed
756
757
      v_r2_2.v = vec_fma(v_dz_tmp2.v, v_dz_tmp2.v, v_r2_2.v);

758
      /* Form a mask from r2 < hig2 and r2 > 0.*/
James Willis's avatar
James Willis committed
759
760
      mask_t v_doi_mask, v_doi_mask_self_check, v_doi_mask2,
          v_doi_mask2_self_check;
761
      int doi_mask, doi_mask_self_check, doi_mask2, doi_mask2_self_check;
762

James Willis's avatar
James Willis committed
763
      /* Form r2 > 0 mask and r2 < hig2 mask. */
764
      vec_create_mask(v_doi_mask_self_check, vec_cmp_gt(v_r2.v, vec_setzero()));
765
      vec_create_mask(v_doi_mask, vec_cmp_lt(v_r2.v, v_hig2.v));
766

James Willis's avatar
James Willis committed
767
      /* Form r2 > 0 mask and r2 < hig2 mask. */
James Willis's avatar
James Willis committed
768
769
      vec_create_mask(v_doi_mask2_self_check,
                      vec_cmp_gt(v_r2_2.v, vec_setzero()));
770
      vec_create_mask(v_doi_mask2, vec_cmp_lt(v_r2_2.v, v_hig2.v));
771

772
773
774
775
776
777
      /* Form integer masks. */
      doi_mask_self_check = vec_form_int_mask(v_doi_mask_self_check);
      doi_mask = vec_form_int_mask(v_doi_mask);

      doi_mask2_self_check = vec_form_int_mask(v_doi_mask2_self_check);
      doi_mask2 = vec_form_int_mask(v_doi_mask2);
James Willis's avatar
James Willis committed
778

779
780
781
      /* Combine the two masks. */
      doi_mask = doi_mask & doi_mask_self_check;
      doi_mask2 = doi_mask2 & doi_mask2_self_check;
782

James Willis's avatar
James Willis committed
783
784
      /* If there are any interactions left pack interaction values into c2
       * cache. */
785
      if (doi_mask) {
James Willis's avatar
James Willis committed
786
        storeInteractions(doi_mask, pjd, &v_r2, &v_dx_tmp, &v_dy_tmp, &v_dz_tmp,
James Willis's avatar
James Willis committed
787
788
789
790
791
792
793
794
                          cell_cache, &int_cache, &icount, &rhoSum, &rho_dhSum,
                          &wcountSum, &wcount_dhSum, &div_vSum, &curlvxSum,
                          &curlvySum, &curlvzSum, v_hi_inv, v_vix, v_viy,
                          v_viz);
      }
      if (doi_mask2) {
        storeInteractions(doi_mask2, pjd + VEC_SIZE, &v_r2_2, &v_dx_tmp2,
                          &v_dy_tmp2, &v_dz_tmp2, cell_cache, &int_cache,
James Willis's avatar
James Willis committed
795
796
797
                          &icount, &rhoSum, &rho_dhSum, &wcountSum,
                          &wcount_dhSum, &div_vSum, &curlvxSum, &curlvySum,
                          &curlvzSum, v_hi_inv, v_vix, v_viy, v_viz);
798
799
800
      }
    }

James Willis's avatar
James Willis committed
801
    /* Perform padded vector remainder interactions if any are present. */
Matthieu Schaller's avatar
Matthieu Schaller committed
802
803
804
    calcRemInteractions(&int_cache, icount, &rhoSum, &rho_dhSum, &wcountSum,
                        &wcount_dhSum, &div_vSum, &curlvxSum, &curlvySum,
                        &curlvzSum, v_hi_inv, v_vix, v_viy, v_viz,
James Willis's avatar
James Willis committed
805
806
807
808
                        &icount_align);

    /* Initialise masks to true in case remainder interactions have been
     * performed. */
809
    mask_t int_mask, int_mask2;
810
811
    vec_init_mask_true(int_mask);
    vec_init_mask_true(int_mask2);
812
813

    /* Perform interaction with 2 vectors. */
James Willis's avatar
James Willis committed
814
815
816
817
818
819
    for (int pjd = 0; pjd < icount_align; pjd += (num_vec_proc * VEC_SIZE)) {
      runner_iact_nonsym_2_vec_density(
          &int_cache.r2q[pjd], &int_cache.dxq[pjd], &int_cache.dyq[pjd],
          &int_cache.dzq[pjd], v_hi_inv, v_vix, v_viy, v_viz,
          &int_cache.vxq[pjd], &int_cache.vyq[pjd], &int_cache.vzq[pjd],
          &int_cache.mq[pjd], &rhoSum, &rho_dhSum, &wcountSum, &wcount_dhSum,
James Willis's avatar
James Willis committed
820
821
          &div_vSum, &curlvxSum, &curlvySum, &curlvzSum, int_mask, int_mask2,
          0);
822
823
    }

James Willis's avatar
James Willis committed
824
825
826
827
828
829
830
831
832
833
    /* Perform horizontal adds on vector sums and store result in particle pi.
     */
    VEC_HADD(rhoSum, pi->rho);
    VEC_HADD(rho_dhSum, pi->density.rho_dh);
    VEC_HADD(wcountSum, pi->density.wcount);
    VEC_HADD(wcount_dhSum, pi->density.wcount_dh);
    VEC_HADD(div_vSum, pi->density.div_v);
    VEC_HADD(curlvxSum, pi->density.rot_v[0]);
    VEC_HADD(curlvySum, pi->density.rot_v[1]);
    VEC_HADD(curlvzSum, pi->density.rot_v[2]);
834
835
836
837
838

    /* Reset interaction count. */
    icount = 0;
  } /* loop over all particles. */

James Willis's avatar
James Willis committed
839
  TIMER_TOC(timer_doself_density);
840
#endif /* WITH_VECTORIZATION */
841
842
}

843
/**
James Willis's avatar
James Willis committed
844
 * @brief Compute the force cell self-interaction (non-symmetric) using vector
845
846
847
848
849
850
851
852
853
854
855
856
 * intrinsics with one particle pi at a time.
 *
 * @param r The #runner.
 * @param c The #cell.
 */
__attribute__((always_inline)) INLINE void runner_doself2_force_vec(
    struct runner *r, struct cell *restrict c) {

#ifdef WITH_VECTORIZATION
  const struct engine *e = r->e;
  struct part *restrict pi;
  int count_align;
James Willis's avatar
James Willis committed
857
  const int num_vec_proc = 1;
858
859
860
861
862
863
864

  struct part *restrict parts = c->parts;
  const int count = c->count;

  vector v_hi, v_vix, v_viy, v_viz, v_hig2, v_r2;
  vector v_rhoi, v_grad_hi, v_pOrhoi2, v_balsara_i, v_ci;

865
  TIMER_TIC
866
867
868

  if (!cell_is_active(c, e)) return;

869
  if (!cell_are_part_drifted(c, e)) error("Interacting undrifted cell.");
870
871
872
873
874
875
876
877
878
879

  /* Get the particle cache from the runner and re-allocate
   * the cache if it is not big enough for the cell. */
  struct cache *restrict cell_cache = &r->ci_cache;

  if (cell_cache->count < count) {
    cache_init(cell_cache, count);
  }

  /* Read the particles from the cell and store them locally in the cache. */
880
  cache_read_force_particles(c, cell_cache);
881

James Willis's avatar
James Willis committed
882
#ifdef SWIFT_DEBUG_CHECKS
James Willis's avatar
James Willis committed
883
  for (int i = 0; i < count; i++) {
James Willis's avatar
James Willis committed
884
885
886
887
888
    pi = &c->parts[i];
    /* Check that particles have been drifted to the current time */
    if (pi->ti_drift != e->ti_current)
      error("Particle pi not drifted to current time");
  }
James Willis's avatar
James Willis committed
889
}
James Willis's avatar
James Willis committed
890
891
#endif

892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
/* Loop over the particles in the cell. */
for (int pid = 0; pid < count; pid++) {

  /* Get a pointer to the ith particle. */
  pi = &parts[pid];

  /* Is the ith particle active? */
  if (!part_is_active(pi, e)) continue;

  vector pix, piy, piz;

  const float hi = cell_cache->h[pid];

  /* Fill particle pi vectors. */
  pix.v = vec_set1(cell_cache->x[pid]);
  piy.v = vec_set1(cell_cache->y[pid]);
  piz.v = vec_set1(cell_cache->z[pid]);
  v_hi.v = vec_set1(hi);
  v_vix.v = vec_set1(cell_cache->vx[pid]);
  v_viy.v = vec_set1(cell_cache->vy[pid]);
  v_viz.v = vec_set1(cell_cache->vz[pid]);

  v_rhoi.v = vec_set1(cell_cache->rho[pid]);
  v_grad_hi.v = vec_set1(cell_cache->grad_h[pid]);
  v_pOrhoi2.v = vec_set1(cell_cache->pOrho2[pid]);
  v_balsara_i.v = vec_set1(cell_cache->balsara[pid]);
  v_ci.v = vec_set1(cell_cache->soundspeed[pid]);

  const float hig2 = hi * hi * kernel_gamma2;
  v_hig2.v = vec_set1(hig2);

  /* Reset cumulative sums of update vectors. */
  vector a_hydro_xSum, a_hydro_ySum, a_hydro_zSum, h_dtSum, v_sigSum,
      entropy_dtSum;

  /* Get the inverse of hi. */
  vector v_hi_inv;

  v_hi_inv = vec_reciprocal(v_hi);

  a_hydro_xSum.v = vec_setzero();
  a_hydro_ySum.v = vec_setzero();
  a_hydro_zSum.v = vec_setzero();
  h_dtSum.v = vec_setzero();
  v_sigSum.v = vec_set1(pi->force.v_sig);
  entropy_dtSum.v = vec_setzero();

  /* Pad cache if there is a serial remainder. */
  count_align = count;
  int rem = count % (num_vec_proc * VEC_SIZE);
  if (rem != 0) {
    int pad = (num_vec_proc * VEC_SIZE) - rem;

    count_align += pad;

    /* Set positions to the same as particle pi so when the r2 > 0 mask is
     * applied these extra contributions are masked out.*/
    for (int i = count; i < count_align; i++) {
      cell_cache->x[i] = pix.f[0];
      cell_cache->y[i] = piy.f[0];
      cell_cache->z[i] = piz.f[0];
      cell_cache->h[i] = 1.f;
    }
  }

  vector pjx, pjy, pjz, hj, hjg2;

  /* Find all of particle pi's interacions and store needed values in the
   * secondary cache.*/
  for (int pjd = 0; pjd < count_align; pjd += (num_vec_proc * VEC_SIZE)) {

    int cj_cache_idx = pjd;
    /* Load 2 sets of vectors from the particle cache. */
    pjx.v = vec_load(&cell_cache->x[pjd]);
    pjy.v = vec_load(&cell_cache->y[pjd]);
    pjz.v = vec_load(&cell_cache->z[pjd]);
    hj.v = vec_load(&cell_cache->h[pjd]);
    hjg2.v = vec_mul(vec_mul(hj.v, hj.v), kernel_gamma2_vec.v);

    /* Compute the pairwise distance. */
    vector v_dx_tmp, v_dy_tmp, v_dz_tmp;

    v_dx_tmp.v = vec_sub(pix.v, pjx.v);
    v_dy_tmp.v = vec_sub(piy.v, pjy.v);
    v_dz_tmp.v = vec_sub(piz.v, pjz.v);

    v_r2.v = vec_mul(v_dx_tmp.v, v_dx_tmp.v);
    v_r2.v = vec_fma(v_dy_tmp.v, v_dy_tmp.v, v_r2.v);
    v_r2.v = vec_fma(v_dz_tmp.v, v_dz_tmp.v, v_r2.v);

    /* Form r2 > 0 mask, r2 < hig2 mask and r2 < hjg2 mask. */
    mask_t v_doi_mask, v_doi_mask_self_check;
    int doi_mask;

    /* Form r2 > 0 mask.*/
    vec_create_mask(v_doi_mask_self_check, vec_cmp_gt(v_r2.v, vec_setzero()));

    /* Form a mask from r2 < hig2 mask and r2 < hjg2 mask. */
    vector v_h2;
    v_h2.v = vec_fmax(v_hig2.v, hjg2.v);
    vec_create_mask(v_doi_mask, vec_cmp_lt(v_r2.v, v_h2.v));

    /* Combine all 3 masks and form integer mask. */
    v_doi_mask.v = vec_and(v_doi_mask.v, v_doi_mask_self_check.v);
    doi_mask = vec_form_int_mask(v_doi_mask);

    /* If there are any interactions left pack interaction values into c2
     * cache. */
    if (doi_mask) {
      vector v_hj, v_hj_inv;
      v_hj.v = vec_load(&cell_cache->h[cj_cache_idx]);
      v_hj_inv = vec_reciprocal(v_hj);

      runner_iact_nonsym_1_vec_force(
              &v_r2, &v_dx_tmp, &v_dy_tmp, &v_dz_tmp, v_vix, v_viy, v_viz, 
              v_rhoi, v_grad_hi, v_pOrhoi2, v_balsara_i, v_ci,
              &cell_cache->vx[cj_cache_idx], &cell_cache->vy[cj_cache_idx],
              &cell_cache->vz[cj_cache_idx], &cell_cache->rho[cj_cache_idx], &cell_cache->grad_h[cj_cache_idx],
              &cell_cache->pOrho2[cj_cache_idx], &cell_cache->balsara[cj_cache_idx], &cell_cache->soundspeed[cj_cache_idx], &cell_cache->m[cj_cache_idx], v_hi_inv, v_hj_inv, &a_hydro_xSum, &a_hydro_ySum, &a_hydro_zSum,
          &h_dtSum, &v_sigSum, &entropy_dtSum, v_doi_mask);

    }

  } /* Loop over all other particles. */

  VEC_HADD(a_hydro_xSum, pi->a_hydro[0]);
  VEC_HADD(a_hydro_ySum, pi->a_hydro[1]);
  VEC_HADD(a_hydro_zSum, pi->a_hydro[2]);
  VEC_HADD(h_dtSum, pi->force.h_dt);
  VEC_HMAX(v_sigSum, pi->force.v_sig);
  VEC_HADD(entropy_dtSum, pi->entropy_dt);

} /* loop over all particles. */

TIMER_TOC(timer_doself_force);
#endif /* WITH_VECTORIZATION */
}
__attribute__((always_inline)) INLINE void runner_doself2_force_vec_2(
    struct runner *r, struct cell *restrict c) {
#ifdef WITH_VECTORIZATION
  const struct engine *e = r->e;
  struct part *restrict pi;
  int count_align;
  const int num_vec_proc = 1;//2;

  struct part *restrict parts = c->parts;
  const int count = c->count;

  vector v_hi, v_vix, v_viy, v_viz, v_hig2, v_r2;
  vector v_rhoi, v_grad_hi, v_pOrhoi2, v_balsara_i, v_ci;

  TIMER_TIC

  if (!cell_is_active(c, e)) return;

  if (!cell_are_part_drifted(c, e)) error("Interacting undrifted cell.");

  /* Get the particle cache from the runner and re-allocate
   * the cache if it is not big enough for the cell. */
  struct cache *restrict cell_cache = &r->ci_cache;

  if (cell_cache->count < count) {
    cache_init(cell_cache, count);
  }

  /* Read the particles from the cell and store them locally in the cache. */
  cache_read_force_particles(c, cell_cache);

#ifdef SWIFT_DEBUG_CHECKS
  for (int i = 0; i < count; i++) {
    pi = &c->parts[i];
    /* Check that particles have been drifted to the current time */
    if (pi->ti_drift != e->ti_current)
      error("Particle pi not drifted to current time");
  }
}
#endif

James Willis's avatar
James Willis committed
1070
1071
1072
/* Create secondary cache to store particle interactions. */
struct c2_cache int_cache;
int icount = 0, icount_align = 0;
1073

James Willis's avatar
James Willis committed
1074
1075
/* Loop over the particles in the cell. */
for (int pid = 0; pid < count; pid++) {
1076

James Willis's avatar
James Willis committed
1077
1078
  /* Get a pointer to the ith particle. */
  pi = &parts[pid];
1079

James Willis's avatar
James Willis committed
1080
1081
  /* Is the ith particle active? */
  if (!part_is_active(pi, e)) continue;
1082

James Willis's avatar
James Willis committed
1083
  vector pix, piy, piz;
1084

James Willis's avatar
James Willis committed
1085
  const float hi = cell_cache->h[pid];
1086

James Willis's avatar
James Willis committed
1087
1088
1089
1090
1091
1092
1093
1094
  /* Fill particle pi vectors. */
  pix.v = vec_set1(cell_cache->x[pid]);
  piy.v = vec_set1(cell_cache->y[pid]);
  piz.v = vec_set1(cell_cache->z[pid]);
  v_hi.v = vec_set1(hi);
  v_vix.v = vec_set1(cell_cache->vx[pid]);
  v_viy.v = vec_set1(cell_cache->vy[pid]);
  v_viz.v = vec_set1(cell_cache->vz[pid]);
1095

James Willis's avatar
James Willis committed
1096
1097
1098
1099
1100
  v_rhoi.v = vec_set1(cell_cache->rho[pid]);
  v_grad_hi.v = vec_set1(cell_cache->grad_h[pid]);
  v_pOrhoi2.v = vec_set1(cell_cache->pOrho2[pid]);
  v_balsara_i.v = vec_set1(cell_cache->balsara[pid]);
  v_ci.v = vec_set1(cell_cache->soundspeed[pid]);
1101

James Willis's avatar
James Willis committed
1102
1103
  const float hig2 = hi * hi * kernel_gamma2;
  v_hig2.v = vec_set1(hig2);
1104

James Willis's avatar
James Willis committed
1105
1106
1107
  /* Reset cumulative sums of update vectors. */
  vector a_hydro_xSum, a_hydro_ySum, a_hydro_zSum, h_dtSum, v_sigSum,
      entropy_dtSum;
1108

James Willis's avatar
James Willis committed
1109
1110
  /* Get the inverse of hi. */
  vector v_hi_inv;
1111