collectgroup.c 7.15 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
/*******************************************************************************
 * This file is part of SWIFT.
 * Copyright (c) 2017 Peter W. Draper (p.w.draper@durham.ac.uk)
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published
 * by the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 ******************************************************************************/

/* Config parameters. */
#include "../config.h"

/* MPI headers. */
#ifdef WITH_MPI
#include <mpi.h>
#endif

/* This object's header. */
#include "collectgroup.h"

/* Local headers. */
#include "engine.h"
#include "error.h"

#ifdef WITH_MPI

/* Local collections for MPI reduces. */
struct mpicollectgroup1 {
  size_t updates, g_updates, s_updates;
40
41
  integertime_t ti_hydro_end_min;
  integertime_t ti_gravity_end_min;
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
  int forcerebuild;
};

/* Forward declarations. */
static void mpicollect_create_MPI_type();

/**
 * @brief MPI datatype for the #mpicollectgroup1 structure.
 */
static MPI_Datatype mpicollectgroup1_type;

/**
 * @brief MPI operator to reduce #mpicollectgroup1 structures.
 */
static MPI_Op mpicollectgroup1_reduce_op;

#endif

/**
 * @brief Perform any once only initialisations. Must be called once.
 */
void collectgroup_init() {

#ifdef WITH_MPI
  /* Initialise the MPI types. */
  mpicollect_create_MPI_type();
#endif
}

/**
 * @brief Apply the collectgroup1 values to the engine by copying all the
 * values to the engine fields.
 *
 * @param grp1 The #collectgroup1
 * @param e The #engine
 */
void collectgroup1_apply(struct collectgroup1 *grp1, struct engine *e) {
79
80
81
82
83
84
85
86
87
  e->ti_hydro_end_min = grp1->ti_hydro_end_min;
  e->ti_hydro_end_max = grp1->ti_hydro_end_max;
  e->ti_hydro_beg_max = grp1->ti_hydro_beg_max;
  e->ti_gravity_end_min = grp1->ti_gravity_end_min;
  e->ti_gravity_end_max = grp1->ti_gravity_end_max;
  e->ti_gravity_beg_max = grp1->ti_gravity_beg_max;
  e->ti_end_min = min(e->ti_hydro_end_min, e->ti_gravity_end_min);
  e->ti_end_max = max(e->ti_hydro_end_max, e->ti_gravity_end_max);
  e->ti_beg_max = max(e->ti_hydro_beg_max, e->ti_gravity_beg_max);
88
89
90
91
92
93
94
95
96
97
98
  e->updates = grp1->updates;
  e->g_updates = grp1->g_updates;
  e->s_updates = grp1->s_updates;
  e->forcerebuild = grp1->forcerebuild;
}

/**
 * @brief Initialises a collectgroup1 struct ready for processing.
 *
 * @param grp1 The #collectgroup1 to initialise
 * @param updates the number of updated hydro particles on this node this step.
Matthieu Schaller's avatar
Matthieu Schaller committed
99
100
 * @param g_updates the number of updated gravity particles on this node this
 * step.
101
102
103
104
105
106
107
108
 * @param s_updates the number of updated star particles on this node this step.
 * @param ti_end_min the minimum end time for next time step after this step.
 * @param ti_end_max the maximum end time for next time step after this step.
 * @param ti_beg_max the maximum begin time for next time step after this step.
 * @param forcerebuild whether a rebuild is required after this step.
 */
void collectgroup1_init(struct collectgroup1 *grp1, size_t updates,
                        size_t g_updates, size_t s_updates,
109
110
111
112
113
114
                        integertime_t ti_hydro_end_min,
                        integertime_t ti_hydro_end_max,
                        integertime_t ti_hydro_beg_max,
                        integertime_t ti_gravity_end_min,
                        integertime_t ti_gravity_end_max,
                        integertime_t ti_gravity_beg_max, int forcerebuild) {
115
116
117
  grp1->updates = updates;
  grp1->g_updates = g_updates;
  grp1->s_updates = s_updates;
118
119
120
121
122
123
  grp1->ti_hydro_end_min = ti_hydro_end_min;
  grp1->ti_hydro_end_max = ti_hydro_end_max;
  grp1->ti_hydro_beg_max = ti_hydro_beg_max;
  grp1->ti_gravity_end_min = ti_gravity_end_min;
  grp1->ti_gravity_end_max = ti_gravity_end_max;
  grp1->ti_gravity_beg_max = ti_gravity_beg_max;
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
  grp1->forcerebuild = forcerebuild;
}

/**
 * @brief Do any processing necessary to the group before it can be used.
 *
 * This may involve an MPI reduction across all nodes.
 *
 * @param grp1 the #collectgroup1 struct already initialised by a call
 *             to collectgroup1_init.
 */
void collectgroup1_reduce(struct collectgroup1 *grp1) {

#ifdef WITH_MPI

  /* Populate an MPI group struct and reduce this across all nodes. */
  struct mpicollectgroup1 mpigrp11;
  mpigrp11.updates = grp1->updates;
  mpigrp11.g_updates = grp1->g_updates;
  mpigrp11.s_updates = grp1->s_updates;
144
145
  mpigrp11.ti_hydro_end_min = grp1->ti_hydro_end_min;
  mpigrp11.ti_gravity_end_min = grp1->ti_gravity_end_min;
146
147
148
149
  mpigrp11.forcerebuild = grp1->forcerebuild;

  struct mpicollectgroup1 mpigrp12;
  if (MPI_Allreduce(&mpigrp11, &mpigrp12, 1, mpicollectgroup1_type,
Matthieu Schaller's avatar
Matthieu Schaller committed
150
                    mpicollectgroup1_reduce_op, MPI_COMM_WORLD) != MPI_SUCCESS)
151
152
153
154
155
156
    error("Failed to reduce mpicollection1.");

  /* And update. */
  grp1->updates = mpigrp12.updates;
  grp1->g_updates = mpigrp12.g_updates;
  grp1->s_updates = mpigrp12.s_updates;
157
158
  grp1->ti_hydro_end_min = mpigrp12.ti_hydro_end_min;
  grp1->ti_gravity_end_min = mpigrp12.ti_gravity_end_min;
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
  grp1->forcerebuild = mpigrp12.forcerebuild;

#endif
}

#ifdef WITH_MPI
/**
 * @brief Do the reduction of two structs.
 *
 * @param mpigrp11 the first struct, this is updated on exit.
 * @param mpigrp12 the second struct
 */
static void doreduce1(struct mpicollectgroup1 *mpigrp11,
                      const struct mpicollectgroup1 *mpigrp12) {

  /* Do what is needed for each part of the collection. */
  /* Sum of updates. */
  mpigrp11->updates += mpigrp12->updates;
  mpigrp11->g_updates += mpigrp12->g_updates;
  mpigrp11->s_updates += mpigrp12->s_updates;

  /* Minimum end time. */
181
182
183
184
  mpigrp11->ti_hydro_end_min =
      min(mpigrp11->ti_hydro_end_min, mpigrp12->ti_hydro_end_min);
  mpigrp11->ti_gravity_end_min =
      min(mpigrp11->ti_gravity_end_min, mpigrp12->ti_gravity_end_min);
185
186
187
188
189
190
191

  /* Everyone must agree to not rebuild. */
  if (mpigrp11->forcerebuild || mpigrp12->forcerebuild)
    mpigrp11->forcerebuild = 1;
}

/**
192
 * @brief MPI reduce operator for #mpicollectgroup1 structures.
193
194
 */
static void mpicollectgroup1_reduce(void *in, void *inout, int *len,
Matthieu Schaller's avatar
Matthieu Schaller committed
195
                                    MPI_Datatype *datatype) {
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216

  for (int i = 0; i < *len; ++i)
    doreduce1(&((struct mpicollectgroup1 *)inout)[0],
              &((const struct mpicollectgroup1 *)in)[i]);
}

/**
 * @brief Registers any MPI collection types and reduction functions.
 */
static void mpicollect_create_MPI_type() {

  if (MPI_Type_contiguous(sizeof(struct mpicollectgroup1), MPI_BYTE,
                          &mpicollectgroup1_type) != MPI_SUCCESS ||
      MPI_Type_commit(&mpicollectgroup1_type) != MPI_SUCCESS) {
    error("Failed to create MPI type for mpicollection1.");
  }

  /* Create the reduction operation */
  MPI_Op_create(mpicollectgroup1_reduce, 1, &mpicollectgroup1_reduce_op);
}
#endif