scheduler.c 55.3 KB
Newer Older
1
2
/*******************************************************************************
 * This file is part of SWIFT.
3
 * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
4
 *                    Matthieu Schaller (matthieu.schaller@durham.ac.uk)
5
 *               2016 Peter W. Draper (p.w.draper@durham.ac.uk)
6
 *
7
8
9
10
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published
 * by the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
11
 *
12
13
14
15
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
16
 *
17
18
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
19
 *
20
21
22
23
24
25
 ******************************************************************************/

/* Config parameters. */
#include "../config.h"

/* Some standard headers. */
26
27
28
#include <limits.h>
#include <math.h>
#include <pthread.h>
29
30
31
32
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

33
34
/* MPI headers. */
#ifdef WITH_MPI
35
#include <mpi.h>
36
37
#endif

38
39
40
/* This object's header. */
#include "scheduler.h"

41
42
/* Local headers. */
#include "atomic.h"
43
#include "cycle.h"
44
#include "engine.h"
45
#include "error.h"
46
#include "intrinsics.h"
47
#include "kernel_hydro.h"
48
#include "queue.h"
49
#include "sort_part.h"
50
51
#include "space.h"
#include "task.h"
52
#include "timers.h"
53

54
55
56
57
58
/**
 * @brief Re-set the list of active tasks.
 */
void scheduler_clear_active(struct scheduler *s) { s->active_count = 0; }

59
60
61
62
63
64
65
/**
 * @brief Add an unlock_task to the given task.
 *
 * @param s The #scheduler.
 * @param ta The unlocking #task.
 * @param tb The #task that will be unlocked.
 */
66
67
void scheduler_addunlock(struct scheduler *s, struct task *ta,
                         struct task *tb) {
68
69
70
71
72
#ifdef SWIFT_DEBUG_CHECKS
  if (ta == NULL) error("Unlocking task is NULL.");
  if (tb == NULL) error("Unlocked task is NULL.");
#endif

73
74
75
76
77
  /* Get an index at which to store this unlock. */
  const int ind = atomic_inc(&s->nr_unlocks);

  /* Does the buffer need to be grown? */
  if (ind == s->size_unlocks) {
78
    /* Allocate the new buffer. */
79
80
81
    struct task **unlocks_new;
    int *unlock_ind_new;
    const int size_unlocks_new = s->size_unlocks * 2;
82
83
    if ((unlocks_new = (struct task **)malloc(sizeof(struct task *) *
                                              size_unlocks_new)) == NULL ||
84
85
        (unlock_ind_new = (int *)malloc(sizeof(int) * size_unlocks_new)) ==
            NULL)
86
      error("Failed to re-allocate unlocks.");
87

88
    /* Wait for all writes to the old buffer to complete. */
89
90
91
    while (s->completed_unlock_writes < ind)
      ;

92
    /* Copy the buffers. */
93
94
95
96
97
98
    memcpy(unlocks_new, s->unlocks, sizeof(struct task *) * ind);
    memcpy(unlock_ind_new, s->unlock_ind, sizeof(int) * ind);
    free(s->unlocks);
    free(s->unlock_ind);
    s->unlocks = unlocks_new;
    s->unlock_ind = unlock_ind_new;
99

100
    /* Publish the new buffer size. */
101
102
    s->size_unlocks = size_unlocks_new;
  }
103

104
  /* Wait for there to actually be space at my index. */
105
106
  while (ind > s->size_unlocks)
    ;
107
108
109
110

  /* Write the unlock to the scheduler. */
  s->unlocks[ind] = tb;
  s->unlock_ind[ind] = ta - s->tasks;
111
  atomic_inc(&s->completed_unlock_writes);
lhausamm's avatar
lhausamm committed
112
113
114
115
116

  scheduler_write_dependency(ta,tb);
}

void scheduler_write_dependency(struct task *ta, struct task *tb) {
117
118
119
120
121
122
123
124
125
126
  int test = ta->ci->ti_end_min == 0 && tb->ci->ti_end_min == 0;
  test = test && ta->ci->parent == NULL && tb->ci->parent == NULL;
  if (test)
    {
      char tmp[200];
      char *line = NULL;
      size_t len = 0;
      ssize_t read;
      FILE *f;
      
lhausamm's avatar
lhausamm committed
127
128
129
130
131
132
      sprintf(tmp, "\t%s_%s->%s_%s;\n",
	      taskID_names[ta->type],
	      subtaskID_names[ta->subtype],
	      taskID_names[tb->type],
	      subtaskID_names[tb->subtype]
	      );
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149

      f = fopen("test_graph.viz", "r");

      if (f != NULL)
	{
	  test = 1;
	  while (test && (read = getline(&line, &len, f)) != -1)
	    {
	      if (strcmp(tmp, line) == 0)
		{
		  test = 0;
		}
	      
	    }
      
	  fclose(f);
	}
lhausamm's avatar
lhausamm committed
150
151
152
153
154
155
      else
	{
	  f = fopen("test_graph.viz", "w");
	  fprintf(f,"digraph task_dep {\n\tcompound=true;\n\tratio=0.66;\n\tnode[nodesep=0.15];\n");
	  fclose(f);
	}
156
157
158
159
160
161
162
163
164

      if (test)
	{
	  f = fopen("test_graph.viz", "a");
	  fprintf(f, tmp);
	  fclose(f);
	}

    }
lhausamm's avatar
lhausamm committed
165

166
167
}

lhausamm's avatar
lhausamm committed
168

169
/**
170
 * @brief Split a hydrodynamic task if too large.
171
 *
172
173
 * @param t The #task
 * @param s The #scheduler we are working in.
174
 */
175
static void scheduler_splittask_hydro(struct task *t, struct scheduler *s) {
176

177
178
179
  /* Iterate on this task until we're done with it. */
  int redo = 1;
  while (redo) {
180

181
182
    /* Reset the redo flag. */
    redo = 0;
183

184
    /* Non-splittable task? */
185
186
    if ((t->ci == NULL) || (t->type == task_type_pair && t->cj == NULL) ||
        t->ci->count == 0 || (t->cj != NULL && t->cj->count == 0)) {
187
      t->type = task_type_none;
188
189
      t->subtype = task_subtype_none;
      t->cj = NULL;
190
191
192
      t->skip = 1;
      break;
    }
193

194
195
    /* Self-interaction? */
    if (t->type == task_type_self) {
196

197
198
199
200
201
      /* Get a handle on the cell involved. */
      struct cell *ci = t->ci;

      /* Foreign task? */
      if (ci->nodeID != s->nodeID) {
202
        t->skip = 1;
203
        break;
204
205
      }

206
      /* Is this cell even split and the task does not violate h ? */
207
      if (cell_can_split_self_task(ci)) {
208

209
        /* Make a sub? */
210
        if (scheduler_dosub && ci->count < space_subsize_self) {
211

212
213
214
215
216
          /* convert to a self-subtask. */
          t->type = task_type_sub_self;

          /* Otherwise, make tasks explicitly. */
        } else {
217

218
219
220
221
222
223
224
225
          /* Take a step back (we're going to recycle the current task)... */
          redo = 1;

          /* Add the self tasks. */
          int first_child = 0;
          while (ci->progeny[first_child] == NULL) first_child++;
          t->ci = ci->progeny[first_child];
          for (int k = first_child + 1; k < 8; k++)
226
            if (ci->progeny[k] != NULL && ci->progeny[k]->count)
227
              scheduler_splittask_hydro(
228
                  scheduler_addtask(s, task_type_self, t->subtype, 0, 0,
229
                                    ci->progeny[k], NULL),
230
231
                  s);

232
233
          /* Make a task for each pair of progeny */
          for (int j = 0; j < 8; j++)
234
            if (ci->progeny[j] != NULL && ci->progeny[j]->count)
235
              for (int k = j + 1; k < 8; k++)
236
                if (ci->progeny[k] != NULL && ci->progeny[k]->count)
237
238
                  scheduler_splittask_hydro(
                      scheduler_addtask(s, task_type_pair, t->subtype,
239
                                        sub_sid_flag[j][k], 0, ci->progeny[j],
240
241
                                        ci->progeny[k]),
                      s);
242
        }
243
      } /* Cell is split */
244

245
    } /* Self interaction */
246

247
248
    /* Pair interaction? */
    else if (t->type == task_type_pair) {
249

250
251
252
      /* Get a handle on the cells involved. */
      struct cell *ci = t->ci;
      struct cell *cj = t->cj;
253

254
255
256
257
258
      /* Foreign task? */
      if (ci->nodeID != s->nodeID && cj->nodeID != s->nodeID) {
        t->skip = 1;
        break;
      }
259

260
261
262
      /* Get the sort ID, use space_getsid and not t->flags
         to make sure we get ci and cj swapped if needed. */
      double shift[3];
Matthieu Schaller's avatar
Matthieu Schaller committed
263
      const int sid = space_getsid(s->space, &ci, &cj, shift);
264

265
      /* Should this task be split-up? */
266
      if (cell_can_split_pair_task(ci) && cell_can_split_pair_task(cj)) {
267
268

        /* Replace by a single sub-task? */
269
270
        if (scheduler_dosub && /* Use division to avoid integer overflow. */
            ci->count * sid_scale[sid] < space_subsize_pair / cj->count &&
271
            !sort_is_corner(sid)) {
272
273
274
275
276

          /* Make this task a sub task. */
          t->type = task_type_sub_pair;

          /* Otherwise, split it. */
277
278
        } else {

279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
          /* Take a step back (we're going to recycle the current task)... */
          redo = 1;

          /* For each different sorting type... */
          switch (sid) {

            case 0: /* (  1 ,  1 ,  1 ) */
              t->ci = ci->progeny[7];
              t->cj = cj->progeny[0];
              t->flags = 0;
              break;

            case 1: /* (  1 ,  1 ,  0 ) */
              t->ci = ci->progeny[6];
              t->cj = cj->progeny[0];
              t->flags = 1;
295
              scheduler_splittask_hydro(
296
                  scheduler_addtask(s, task_type_pair, t->subtype, 1, 0,
297
                                    ci->progeny[7], cj->progeny[1]),
298
                  s);
299
              scheduler_splittask_hydro(
300
                  scheduler_addtask(s, task_type_pair, t->subtype, 0, 0,
301
                                    ci->progeny[6], cj->progeny[1]),
302
                  s);
303
              scheduler_splittask_hydro(
304
                  scheduler_addtask(s, task_type_pair, t->subtype, 2, 0,
305
                                    ci->progeny[7], cj->progeny[0]),
306
307
308
309
310
311
312
313
314
315
316
317
318
                  s);
              break;

            case 2: /* (  1 ,  1 , -1 ) */
              t->ci = ci->progeny[6];
              t->cj = cj->progeny[1];
              t->flags = 2;
              break;

            case 3: /* (  1 ,  0 ,  1 ) */
              t->ci = ci->progeny[5];
              t->cj = cj->progeny[0];
              t->flags = 3;
319
              scheduler_splittask_hydro(
320
                  scheduler_addtask(s, task_type_pair, t->subtype, 3, 0,
321
                                    ci->progeny[7], cj->progeny[2]),
322
                  s);
323
              scheduler_splittask_hydro(
324
                  scheduler_addtask(s, task_type_pair, t->subtype, 0, 0,
325
                                    ci->progeny[5], cj->progeny[2]),
326
                  s);
327
              scheduler_splittask_hydro(
328
                  scheduler_addtask(s, task_type_pair, t->subtype, 6, 0,
329
                                    ci->progeny[7], cj->progeny[0]),
330
331
332
333
334
335
336
                  s);
              break;

            case 4: /* (  1 ,  0 ,  0 ) */
              t->ci = ci->progeny[4];
              t->cj = cj->progeny[0];
              t->flags = 4;
337
              scheduler_splittask_hydro(
338
                  scheduler_addtask(s, task_type_pair, t->subtype, 5, 0,
339
                                    ci->progeny[5], cj->progeny[0]),
340
                  s);
341
              scheduler_splittask_hydro(
342
                  scheduler_addtask(s, task_type_pair, t->subtype, 7, 0,
343
                                    ci->progeny[6], cj->progeny[0]),
344
                  s);
345
              scheduler_splittask_hydro(
346
                  scheduler_addtask(s, task_type_pair, t->subtype, 8, 0,
347
                                    ci->progeny[7], cj->progeny[0]),
348
                  s);
349
              scheduler_splittask_hydro(
350
                  scheduler_addtask(s, task_type_pair, t->subtype, 3, 0,
351
                                    ci->progeny[4], cj->progeny[1]),
352
                  s);
353
              scheduler_splittask_hydro(
354
                  scheduler_addtask(s, task_type_pair, t->subtype, 4, 0,
355
                                    ci->progeny[5], cj->progeny[1]),
356
                  s);
357
              scheduler_splittask_hydro(
358
                  scheduler_addtask(s, task_type_pair, t->subtype, 6, 0,
359
                                    ci->progeny[6], cj->progeny[1]),
360
                  s);
361
              scheduler_splittask_hydro(
362
                  scheduler_addtask(s, task_type_pair, t->subtype, 7, 0,
363
                                    ci->progeny[7], cj->progeny[1]),
364
                  s);
365
              scheduler_splittask_hydro(
366
                  scheduler_addtask(s, task_type_pair, t->subtype, 1, 0,
367
                                    ci->progeny[4], cj->progeny[2]),
368
                  s);
369
              scheduler_splittask_hydro(
370
                  scheduler_addtask(s, task_type_pair, t->subtype, 2, 0,
371
                                    ci->progeny[5], cj->progeny[2]),
372
                  s);
373
              scheduler_splittask_hydro(
374
                  scheduler_addtask(s, task_type_pair, t->subtype, 4, 0,
375
                                    ci->progeny[6], cj->progeny[2]),
376
                  s);
377
              scheduler_splittask_hydro(
378
                  scheduler_addtask(s, task_type_pair, t->subtype, 5, 0,
379
                                    ci->progeny[7], cj->progeny[2]),
380
                  s);
381
              scheduler_splittask_hydro(
382
                  scheduler_addtask(s, task_type_pair, t->subtype, 0, 0,
383
                                    ci->progeny[4], cj->progeny[3]),
384
                  s);
385
              scheduler_splittask_hydro(
386
                  scheduler_addtask(s, task_type_pair, t->subtype, 1, 0,
387
                                    ci->progeny[5], cj->progeny[3]),
388
                  s);
389
              scheduler_splittask_hydro(
390
                  scheduler_addtask(s, task_type_pair, t->subtype, 3, 0,
391
                                    ci->progeny[6], cj->progeny[3]),
392
                  s);
393
              scheduler_splittask_hydro(
394
                  scheduler_addtask(s, task_type_pair, t->subtype, 4, 0,
395
                                    ci->progeny[7], cj->progeny[3]),
396
397
398
399
400
401
402
                  s);
              break;

            case 5: /* (  1 ,  0 , -1 ) */
              t->ci = ci->progeny[4];
              t->cj = cj->progeny[1];
              t->flags = 5;
403
              scheduler_splittask_hydro(
404
                  scheduler_addtask(s, task_type_pair, t->subtype, 5, 0,
405
                                    ci->progeny[6], cj->progeny[3]),
406
                  s);
407
              scheduler_splittask_hydro(
408
                  scheduler_addtask(s, task_type_pair, t->subtype, 2, 0,
409
                                    ci->progeny[4], cj->progeny[3]),
410
                  s);
411
              scheduler_splittask_hydro(
412
                  scheduler_addtask(s, task_type_pair, t->subtype, 8, 0,
413
                                    ci->progeny[6], cj->progeny[1]),
414
415
416
417
418
419
420
421
422
423
424
425
426
                  s);
              break;

            case 6: /* (  1 , -1 ,  1 ) */
              t->ci = ci->progeny[5];
              t->cj = cj->progeny[2];
              t->flags = 6;
              break;

            case 7: /* (  1 , -1 ,  0 ) */
              t->ci = ci->progeny[4];
              t->cj = cj->progeny[3];
              t->flags = 6;
427
              scheduler_splittask_hydro(
428
                  scheduler_addtask(s, task_type_pair, t->subtype, 8, 0,
429
                                    ci->progeny[5], cj->progeny[2]),
430
                  s);
431
              scheduler_splittask_hydro(
432
                  scheduler_addtask(s, task_type_pair, t->subtype, 7, 0,
433
                                    ci->progeny[4], cj->progeny[2]),
434
                  s);
435
              scheduler_splittask_hydro(
436
                  scheduler_addtask(s, task_type_pair, t->subtype, 7, 0,
437
                                    ci->progeny[5], cj->progeny[3]),
438
439
440
441
442
443
444
445
446
447
448
449
450
                  s);
              break;

            case 8: /* (  1 , -1 , -1 ) */
              t->ci = ci->progeny[4];
              t->cj = cj->progeny[3];
              t->flags = 8;
              break;

            case 9: /* (  0 ,  1 ,  1 ) */
              t->ci = ci->progeny[3];
              t->cj = cj->progeny[0];
              t->flags = 9;
451
              scheduler_splittask_hydro(
452
                  scheduler_addtask(s, task_type_pair, t->subtype, 9, 0,
453
                                    ci->progeny[7], cj->progeny[4]),
454
                  s);
455
              scheduler_splittask_hydro(
456
                  scheduler_addtask(s, task_type_pair, t->subtype, 0, 0,
457
                                    ci->progeny[3], cj->progeny[4]),
458
                  s);
459
              scheduler_splittask_hydro(
460
                  scheduler_addtask(s, task_type_pair, t->subtype, 8, 0,
461
                                    ci->progeny[7], cj->progeny[0]),
462
463
464
465
466
467
468
                  s);
              break;

            case 10: /* (  0 ,  1 ,  0 ) */
              t->ci = ci->progeny[2];
              t->cj = cj->progeny[0];
              t->flags = 10;
469
              scheduler_splittask_hydro(
470
                  scheduler_addtask(s, task_type_pair, t->subtype, 11, 0,
471
                                    ci->progeny[3], cj->progeny[0]),
472
                  s);
473
              scheduler_splittask_hydro(
474
                  scheduler_addtask(s, task_type_pair, t->subtype, 7, 0,
475
                                    ci->progeny[6], cj->progeny[0]),
476
                  s);
477
              scheduler_splittask_hydro(
478
                  scheduler_addtask(s, task_type_pair, t->subtype, 6, 0,
479
                                    ci->progeny[7], cj->progeny[0]),
480
                  s);
481
              scheduler_splittask_hydro(
482
                  scheduler_addtask(s, task_type_pair, t->subtype, 9, 0,
483
                                    ci->progeny[2], cj->progeny[1]),
484
                  s);
485
              scheduler_splittask_hydro(
486
                  scheduler_addtask(s, task_type_pair, t->subtype, 10, 0,
487
                                    ci->progeny[3], cj->progeny[1]),
488
                  s);
489
              scheduler_splittask_hydro(
490
                  scheduler_addtask(s, task_type_pair, t->subtype, 8, 0,
491
                                    ci->progeny[6], cj->progeny[1]),
492
                  s);
493
              scheduler_splittask_hydro(
494
                  scheduler_addtask(s, task_type_pair, t->subtype, 7, 0,
495
                                    ci->progeny[7], cj->progeny[1]),
496
                  s);
497
              scheduler_splittask_hydro(
498
                  scheduler_addtask(s, task_type_pair, t->subtype, 1, 0,
499
                                    ci->progeny[2], cj->progeny[4]),
500
                  s);
501
              scheduler_splittask_hydro(
502
                  scheduler_addtask(s, task_type_pair, t->subtype, 2, 0,
503
                                    ci->progeny[3], cj->progeny[4]),
504
                  s);
505
              scheduler_splittask_hydro(
506
                  scheduler_addtask(s, task_type_pair, t->subtype, 10, 0,
507
                                    ci->progeny[6], cj->progeny[4]),
508
                  s);
509
              scheduler_splittask_hydro(
510
                  scheduler_addtask(s, task_type_pair, t->subtype, 11, 0,
511
                                    ci->progeny[7], cj->progeny[4]),
512
                  s);
513
              scheduler_splittask_hydro(
514
                  scheduler_addtask(s, task_type_pair, t->subtype, 0, 0,
515
                                    ci->progeny[2], cj->progeny[5]),
516
                  s);
517
              scheduler_splittask_hydro(
518
                  scheduler_addtask(s, task_type_pair, t->subtype, 1, 0,
519
                                    ci->progeny[3], cj->progeny[5]),
520
                  s);
521
              scheduler_splittask_hydro(
522
                  scheduler_addtask(s, task_type_pair, t->subtype, 9, 0,
523
                                    ci->progeny[6], cj->progeny[5]),
524
                  s);
525
              scheduler_splittask_hydro(
526
                  scheduler_addtask(s, task_type_pair, t->subtype, 10, 0,
527
                                    ci->progeny[7], cj->progeny[5]),
528
529
530
531
532
533
534
                  s);
              break;

            case 11: /* (  0 ,  1 , -1 ) */
              t->ci = ci->progeny[2];
              t->cj = cj->progeny[1];
              t->flags = 11;
535
              scheduler_splittask_hydro(
536
                  scheduler_addtask(s, task_type_pair, t->subtype, 11, 0,
537
                                    ci->progeny[6], cj->progeny[5]),
538
                  s);
539
              scheduler_splittask_hydro(
540
                  scheduler_addtask(s, task_type_pair, t->subtype, 2, 0,
541
                                    ci->progeny[2], cj->progeny[5]),
542
                  s);
543
              scheduler_splittask_hydro(
544
                  scheduler_addtask(s, task_type_pair, t->subtype, 6, 0,
545
                                    ci->progeny[6], cj->progeny[1]),
546
547
548
549
550
551
552
                  s);
              break;

            case 12: /* (  0 ,  0 ,  1 ) */
              t->ci = ci->progeny[1];
              t->cj = cj->progeny[0];
              t->flags = 12;
553
              scheduler_splittask_hydro(
554
                  scheduler_addtask(s, task_type_pair, t->subtype, 11, 0,
555
                                    ci->progeny[3], cj->progeny[0]),
556
                  s);
557
              scheduler_splittask_hydro(
558
                  scheduler_addtask(s, task_type_pair, t->subtype, 5, 0,
559
                                    ci->progeny[5], cj->progeny[0]),
560
                  s);
561
              scheduler_splittask_hydro(
562
                  scheduler_addtask(s, task_type_pair, t->subtype, 2, 0,
563
                                    ci->progeny[7], cj->progeny[0]),
564
                  s);
565
              scheduler_splittask_hydro(
566
                  scheduler_addtask(s, task_type_pair, t->subtype, 9, 0,
567
                                    ci->progeny[1], cj->progeny[2]),
568
                  s);
569
              scheduler_splittask_hydro(
570
                  scheduler_addtask(s, task_type_pair, t->subtype, 12, 0,
571
                                    ci->progeny[3], cj->progeny[2]),
572
                  s);
573
              scheduler_splittask_hydro(
574
                  scheduler_addtask(s, task_type_pair, t->subtype, 8, 0,
575
                                    ci->progeny[5], cj->progeny[2]),
576
                  s);
577
              scheduler_splittask_hydro(
578
                  scheduler_addtask(s, task_type_pair, t->subtype, 5, 0,
579
                                    ci->progeny[7], cj->progeny[2]),
580
                  s);
581
              scheduler_splittask_hydro(
582
                  scheduler_addtask(s, task_type_pair, t->subtype, 3, 0,
583
                                    ci->progeny[1], cj->progeny[4]),
584
                  s);
585
              scheduler_splittask_hydro(
586
                  scheduler_addtask(s, task_type_pair, t->subtype, 6, 0,
587
                                    ci->progeny[3], cj->progeny[4]),
588
                  s);
589
              scheduler_splittask_hydro(
590
                  scheduler_addtask(s, task_type_pair, t->subtype, 12, 0,
591
                                    ci->progeny[5], cj->progeny[4]),
592
                  s);
593
              scheduler_splittask_hydro(
594
                  scheduler_addtask(s, task_type_pair, t->subtype, 11, 0,
595
                                    ci->progeny[7], cj->progeny[4]),
596
                  s);
597
              scheduler_splittask_hydro(
598
                  scheduler_addtask(s, task_type_pair, t->subtype, 0, 0,
599
                                    ci->progeny[1], cj->progeny[6]),
600
                  s);
601
              scheduler_splittask_hydro(
602
                  scheduler_addtask(s, task_type_pair, t->subtype, 3, 0,
603
                                    ci->progeny[3], cj->progeny[6]),
604
                  s);
605
              scheduler_splittask_hydro(
606
                  scheduler_addtask(s, task_type_pair, t->subtype, 9, 0,
607
                                    ci->progeny[5], cj->progeny[6]),
608
                  s);
609
              scheduler_splittask_hydro(
610
                  scheduler_addtask(s, task_type_pair, t->subtype, 12, 0,
611
                                    ci->progeny[7], cj->progeny[6]),
612
613
614
                  s);
              break;
          } /* switch(sid) */
615
616
        }

617
618
619
620
621
622
623
624
625
626
627
        /* Otherwise, break it up if it is too large? */
      } else if (scheduler_doforcesplit && ci->split && cj->split &&
                 (ci->count > space_maxsize / cj->count)) {

        // message( "force splitting pair with %i and %i parts." , ci->count ,
        // cj->count );

        /* Replace the current task. */
        t->type = task_type_none;

        for (int j = 0; j < 8; j++)
628
          if (ci->progeny[j] != NULL && ci->progeny[j]->count)
629
            for (int k = 0; k < 8; k++)
630
              if (cj->progeny[k] != NULL && cj->progeny[k]->count) {
631
632
                struct task *tl =
                    scheduler_addtask(s, task_type_pair, t->subtype, 0, 0,
633
                                      ci->progeny[j], cj->progeny[k]);
634
                scheduler_splittask_hydro(tl, s);
635
636
637
638
639
640
641
                tl->flags = space_getsid(s->space, &t->ci, &t->cj, shift);
              }
      }
    } /* pair interaction? */
  }   /* iterate over the current task. */
}

642
643
644
645
646
647
/**
 * @brief Split a gravity task if too large.
 *
 * @param t The #task
 * @param s The #scheduler we are working in.
 */
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
static void scheduler_splittask_gravity(struct task *t, struct scheduler *s) {

  /* Iterate on this task until we're done with it. */
  int redo = 1;
  while (redo) {

    /* Reset the redo flag. */
    redo = 0;

    /* Non-splittable task? */
    if ((t->ci == NULL) || (t->type == task_type_pair && t->cj == NULL)) {
      t->type = task_type_none;
      t->subtype = task_subtype_none;
      t->cj = NULL;
      t->skip = 1;
      break;
    }

    /* Self-interaction? */
    if (t->type == task_type_self) {

      /* Get a handle on the cell involved. */
      struct cell *ci = t->ci;

      /* Foreign task? */
      if (ci->nodeID != s->nodeID) {
        t->skip = 1;
        break;
      }

678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
      /* Should we split this task? */
      if (ci->split && ci->gcount > space_subsize_self_grav) {

        /* Take a step back (we're going to recycle the current task)... */
        redo = 1;

        /* Add the self tasks. */
        int first_child = 0;
        while (ci->progeny[first_child] == NULL) first_child++;
        t->ci = ci->progeny[first_child];
        for (int k = first_child + 1; k < 8; k++)
          if (ci->progeny[k] != NULL)
            scheduler_splittask_gravity(
                scheduler_addtask(s, task_type_self, t->subtype, 0, 0,
                                  ci->progeny[k], NULL),
                s);

        /* Make a task for each pair of progeny */
        if (t->subtype != task_subtype_external_grav) {
          for (int j = 0; j < 8; j++)
            if (ci->progeny[j] != NULL)
              for (int k = j + 1; k < 8; k++)
                if (ci->progeny[k] != NULL)
                  scheduler_splittask_gravity(
                      scheduler_addtask(s, task_type_pair, t->subtype,
                                        sub_sid_flag[j][k], 0, ci->progeny[j],
                                        ci->progeny[k]),
                      s);
706
        }
707
      } /* Cell is split */
708
709
710
711
712

    } /* Self interaction */

    /* Pair interaction? */
    else if (t->type == task_type_pair) {
713
714
715

      /* Get a handle on the cells involved. */
      struct cell *ci = t->ci;
716
717
718
719
720
721
722
723
      struct cell *cj = t->cj;

      /* Foreign task? */
      if (ci->nodeID != s->nodeID && cj->nodeID != s->nodeID) {
        t->skip = 1;
        break;
      }
    } /* pair interaction? */
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
  }   /* iterate over the current task. */
}

/**
 * @brief Mapper function to split tasks that may be too large.
 *
 * @param map_data the tasks to process
 * @param num_elements the number of tasks.
 * @param extra_data The #scheduler we are working in.
 */
void scheduler_splittasks_mapper(void *map_data, int num_elements,
                                 void *extra_data) {

  /* Extract the parameters. */
  struct scheduler *s = (struct scheduler *)extra_data;
  struct task *tasks = (struct task *)map_data;
740

741
742
  for (int ind = 0; ind < num_elements; ind++) {
    struct task *t = &tasks[ind];
743
744
745
746
747
748
749
750
751
752

    /* Invoke the correct splitting strategy */
    if (t->subtype == task_subtype_density) {
      scheduler_splittask_hydro(t, s);
    } else if (t->subtype == task_subtype_external_grav) {
      scheduler_splittask_gravity(t, s);
    } else if (t->subtype == task_subtype_grav) {
      scheduler_splittask_gravity(t, s);
    } else if (t->type == task_type_grav_top_level ||
               t->type == task_type_grav_ghost) {
753
      /* For future use */
754
755
756
    } else {
      error("Unexpected task sub-type");
    }
757
  }
758
}
759

Matthieu Schaller's avatar
Matthieu Schaller committed
760
761
762
763
764
/**
 * @brief Splits all the tasks in the scheduler that are too large.
 *
 * @param s The #scheduler.
 */
765
void scheduler_splittasks(struct scheduler *s) {
766

767
768
  /* Call the mapper on each current task. */
  threadpool_map(s->threadpool, scheduler_splittasks_mapper, s->tasks,
769
                 s->nr_tasks, sizeof(struct task), 0, s);
770
771
}

772
773
774
775
776
777
778
/**
 * @brief Add a #task to the #scheduler.
 *
 * @param s The #scheduler we are working in.
 * @param type The type of the task.
 * @param subtype The sub-type of the task.
 * @param flags The flags of the task.
779
780
 * @param implicit If true, only use this task to unlock dependencies, i.e.
 *        this task is never enqueued.
781
782
783
 * @param ci The first cell to interact.
 * @param cj The second cell to interact.
 */
784
struct task *scheduler_addtask(struct scheduler *s, enum task_types type,
785
786
                               enum task_subtypes subtype, int flags,
                               int implicit, struct cell *ci, struct cell *cj) {
787

788
789
790
791
792
793
#ifdef SWIFT_DEBUG_CHECKS
  if (ci == NULL && cj != NULL)
    error("Added a task with ci==NULL and cj!=NULL type=%s/%s",
          taskID_names[type], subtaskID_names[subtype]);
#endif

794
  /* Get the next free task. */
Pedro Gonnet's avatar
Pedro Gonnet committed
795
  const int ind = atomic_inc(&s->tasks_next);
Matthieu Schaller's avatar
Matthieu Schaller committed
796

797
  /* Overflow? */
798
799
800
801
802
  if (ind >= s->size)
    error(
        "Task list overflow (%d). Need to increase "
        "Scheduler:tasks_per_cell.",
        ind);
803
804

  /* Get a pointer to the new task. */
Pedro Gonnet's avatar
Pedro Gonnet committed
805
  struct task *t = &s->tasks[ind];
806
807
808
809
810

  /* Copy the data. */
  t->type = type;
  t->subtype = subtype;
  t->flags = flags;
811
  t->wait = 0;
812
813
  t->ci = ci;
  t->cj = cj;
814
  t->skip = 1; /* Mark tasks as skip by default. */
815
  t->implicit = implicit;
816
817
818
  t->weight = 0;
  t->rank = 0;
  t->nr_unlock_tasks = 0;
819
#ifdef SWIFT_DEBUG_TASKS
820
  t->rid = -1;
821
822
  t->tic = 0;
  t->toc = 0;
823
#endif
824
825
826
827
828
829
830
831
832

  /* Add an index for it. */
  // lock_lock( &s->lock );
  s->tasks_ind[atomic_inc(&s->nr_tasks)] = ind;
  // lock_unlock_blind( &s->lock );

  /* Return a pointer to the new task. */
  return t;
}
833

834
835
836
837
838
839
840
841
/**
 * @brief Set the unlock pointers in each task.
 *
 * @param s The #scheduler.
 */
void scheduler_set_unlocks(struct scheduler *s) {

  /* Store the counts for each task. */
842
843
  short int *counts;
  if ((counts = (short int *)malloc(sizeof(short int) * s->nr_tasks)) == NULL)