proxy.c 15 KB
Newer Older
1
2
/*******************************************************************************
 * This file is part of SWIFT.
3
 * Copyright (c) 2013 Pedro Gonnet (pedro.gonnet@durham.ac.uk)
4
 *
5
6
7
8
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published
 * by the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
9
 *
10
11
12
13
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
14
 *
15
16
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
17
 *
18
19
20
21
22
23
24
25
26
 ******************************************************************************/

/* Config parameters. */
#include "../config.h"

/* Some standard headers. */
#include <float.h>
#include <limits.h>
#include <sched.h>
27
28
29
30
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
31
32
33

/* MPI headers. */
#ifdef WITH_MPI
34
#include <mpi.h>
35
36
#endif

37
/* This object's header. */
38
#include "proxy.h"
39
40

/* Local headers. */
41
42
43
44
45
#include "error.h"

/**
 * @brief Exchange cells with a remote node.
 *
46
 * @param p The #proxy.
47
 */
48
49

void proxy_cells_exch1(struct proxy *p) {
50
51
52

#ifdef WITH_MPI

53
54
  /* Get the number of pcells we will need to send. */
  p->size_pcells_out = 0;
55
  for (int k = 0; k < p->nr_cells_out; k++)
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
    p->size_pcells_out += p->cells_out[k]->pcell_size;

  /* Send the number of pcells. */
  if (MPI_Isend(&p->size_pcells_out, 1, MPI_INT, p->nodeID,
                p->mynodeID * proxy_tag_shift + proxy_tag_count, MPI_COMM_WORLD,
                &p->req_cells_count_out) != MPI_SUCCESS)
    error("Failed to isend nr of pcells.");
  // message( "isent pcell count (%i) from node %i to node %i." ,
  // p->size_pcells_out , p->mynodeID , p->nodeID ); fflush(stdout);

  /* Allocate and fill the pcell buffer. */
  if (p->pcells_out != NULL) free(p->pcells_out);
  if ((p->pcells_out = malloc(sizeof(struct pcell) * p->size_pcells_out)) ==
      NULL)
    error("Failed to allocate pcell_out buffer.");
71
  for (int ind = 0, k = 0; k < p->nr_cells_out; k++) {
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
    memcpy(&p->pcells_out[ind], p->cells_out[k]->pcell,
           sizeof(struct pcell) * p->cells_out[k]->pcell_size);
    ind += p->cells_out[k]->pcell_size;
  }

  /* Send the pcell buffer. */
  if (MPI_Isend(p->pcells_out, sizeof(struct pcell) * p->size_pcells_out,
                MPI_BYTE, p->nodeID,
                p->mynodeID * proxy_tag_shift + proxy_tag_cells, MPI_COMM_WORLD,
                &p->req_cells_out) != MPI_SUCCESS)
    error("Failed to pcell_out buffer.");
  // message( "isent pcells (%i) from node %i to node %i." , p->size_pcells_out
  // , p->mynodeID , p->nodeID ); fflush(stdout);

  /* Receive the number of pcells. */
  if (MPI_Irecv(&p->size_pcells_in, 1, MPI_INT, p->nodeID,
                p->nodeID * proxy_tag_shift + proxy_tag_count, MPI_COMM_WORLD,
                &p->req_cells_count_in) != MPI_SUCCESS)
    error("Failed to irecv nr of pcells.");
// message( "irecv pcells count on node %i from node %i." , p->mynodeID ,
// p->nodeID ); fflush(stdout);

94
#else
95
  error("SWIFT was not compiled with MPI support.");
96
#endif
97
}
98

99
void proxy_cells_exch2(struct proxy *p) {
100
101
102

#ifdef WITH_MPI

103
104
105
106
107
108
109
110
111
112
113
114
115
116
  /* Re-allocate the pcell_in buffer. */
  if (p->pcells_in != NULL) free(p->pcells_in);
  if ((p->pcells_in = (struct pcell *)malloc(sizeof(struct pcell) *
                                             p->size_pcells_in)) == NULL)
    error("Failed to allocate pcell_in buffer.");

  /* Receive the particle buffers. */
  if (MPI_Irecv(p->pcells_in, sizeof(struct pcell) * p->size_pcells_in,
                MPI_BYTE, p->nodeID,
                p->nodeID * proxy_tag_shift + proxy_tag_cells, MPI_COMM_WORLD,
                &p->req_cells_in) != MPI_SUCCESS)
    error("Failed to irecv part data.");
// message( "irecv pcells (%i) on node %i from node %i." , p->size_pcells_in ,
// p->mynodeID , p->nodeID ); fflush(stdout);
117
118

#else
119
  error("SWIFT was not compiled with MPI support.");
120
#endif
121
}
122
123
124
125
126
127
128
129

/**
 * @brief Add a cell to the given proxy's input list.
 *
 * @param p The #proxy.
 * @param c The #cell.
 */

130
131
132
void proxy_addcell_in(struct proxy *p, struct cell *c) {

  /* Check if the cell is already registered with the proxy. */
133
  for (int k = 0; k < p->nr_cells_in; k++)
134
135
136
137
138
    if (p->cells_in[k] == c) return;

  /* Do we need to grow the number of in cells? */
  if (p->nr_cells_in == p->size_cells_in) {
    p->size_cells_in *= proxy_buffgrow;
139
    struct cell **temp;
140
    if ((temp = malloc(sizeof(struct cell *) * p->size_cells_in)) == NULL)
141
      error("Failed to allocate incoming cell list.");
142
143
144
145
146
147
148
149
150
    memcpy(temp, p->cells_in, sizeof(struct cell *) * p->nr_cells_in);
    free(p->cells_in);
    p->cells_in = temp;
  }

  /* Add the cell. */
  p->cells_in[p->nr_cells_in] = c;
  p->nr_cells_in += 1;
}
151
152
153
154
155
156
157
158

/**
 * @brief Add a cell to the given proxy's output list.
 *
 * @param p The #proxy.
 * @param c The #cell.
 */

159
160
161
void proxy_addcell_out(struct proxy *p, struct cell *c) {

  /* Check if the cell is already registered with the proxy. */
162
  for (int k = 0; k < p->nr_cells_out; k++)
163
164
165
166
167
    if (p->cells_out[k] == c) return;

  /* Do we need to grow the number of out cells? */
  if (p->nr_cells_out == p->size_cells_out) {
    p->size_cells_out *= proxy_buffgrow;
168
    struct cell **temp;
169
170
171
172
173
174
175
176
177
178
179
    if ((temp = malloc(sizeof(struct cell *) * p->size_cells_out)) == NULL)
      error("Failed to allocate outgoing cell list.");
    memcpy(temp, p->cells_out, sizeof(struct cell *) * p->nr_cells_out);
    free(p->cells_out);
    p->cells_out = temp;
  }

  /* Add the cell. */
  p->cells_out[p->nr_cells_out] = c;
  p->nr_cells_out += 1;
}
180
181
182
183

/**
 * @brief Exchange particles with a remote node.
 *
184
 * @param p The #proxy.
185
 */
186
187

void proxy_parts_exch1(struct proxy *p) {
188
189
190

#ifdef WITH_MPI

191
  /* Send the number of particles. */
192
193
194
  p->buff_out[0] = p->nr_parts_out;
  p->buff_out[1] = p->nr_gparts_out;
  if (MPI_Isend(p->buff_out, 2, MPI_INT, p->nodeID,
195
196
197
                p->mynodeID * proxy_tag_shift + proxy_tag_count, MPI_COMM_WORLD,
                &p->req_parts_count_out) != MPI_SUCCESS)
    error("Failed to isend nr of parts.");
198
199
  /* message( "isent particle counts [%i, %i] from node %i to node %i." ,
  p->buff_out[0], p->buff_out[1], p->mynodeID , p->nodeID ); fflush(stdout); */
200
201
202

  /* Send the particle buffers. */
  if (p->nr_parts_out > 0) {
203
204
    if (MPI_Isend(p->parts_out, p->nr_parts_out, part_mpi_type, p->nodeID,
                  p->mynodeID * proxy_tag_shift + proxy_tag_parts,
205
                  MPI_COMM_WORLD, &p->req_parts_out) != MPI_SUCCESS ||
206
        MPI_Isend(p->xparts_out, p->nr_parts_out, xpart_mpi_type, p->nodeID,
207
                  p->mynodeID * proxy_tag_shift + proxy_tag_xparts,
208
                  MPI_COMM_WORLD, &p->req_xparts_out) != MPI_SUCCESS)
209
210
211
      error("Failed to isend part data.");
    // message( "isent particle data (%i) to node %i." , p->nr_parts_out ,
    // p->nodeID ); fflush(stdout);
212
    /*for (int k = 0; k < p->nr_parts_out; k++)
213
214
      message("sending particle %lli, x=[%.3e %.3e %.3e], h=%.3e, to node %i.",
              p->parts_out[k].id, p->parts_out[k].x[0], p->parts_out[k].x[1],
215
              p->parts_out[k].x[2], p->parts_out[k].h, p->nodeID);*/
216
  }
217
  if (p->nr_gparts_out > 0) {
218
    if (MPI_Isend(p->gparts_out, p->nr_gparts_out, gpart_mpi_type, p->nodeID,
219
220
221
222
223
224
                  p->mynodeID * proxy_tag_shift + proxy_tag_gparts,
                  MPI_COMM_WORLD, &p->req_gparts_out) != MPI_SUCCESS)
      error("Failed to isend part data.");
    // message( "isent gpart data (%i) to node %i." , p->nr_parts_out ,
    // p->nodeID ); fflush(stdout);
  }
225
226

  /* Receive the number of particles. */
227
  if (MPI_Irecv(p->buff_in, 2, MPI_INT, p->nodeID,
228
229
230
231
                p->nodeID * proxy_tag_shift + proxy_tag_count, MPI_COMM_WORLD,
                &p->req_parts_count_in) != MPI_SUCCESS)
    error("Failed to irecv nr of parts.");

232
#else
233
  error("SWIFT was not compiled with MPI support.");
234
#endif
235
}
236

237
void proxy_parts_exch2(struct proxy *p) {
238
239
240

#ifdef WITH_MPI

241
242
243
  /* Unpack the incomming parts counts. */
  p->nr_parts_in = p->buff_in[0];
  p->nr_gparts_in = p->buff_in[1];
Pedro Gonnet's avatar
Pedro Gonnet committed
244

245
246
247
248
249
250
251
  /* Is there enough space in the buffer? */
  if (p->nr_parts_in > p->size_parts_in) {
    do {
      p->size_parts_in *= proxy_buffgrow;
    } while (p->nr_parts_in > p->size_parts_in);
    free(p->parts_in);
    free(p->xparts_in);
252
253
    if ((p->parts_in = (struct part *)malloc(sizeof(struct part) *
                                             p->size_parts_in)) == NULL ||
254
255
256
257
        (p->xparts_in = (struct xpart *)malloc(sizeof(struct xpart) *
                                               p->size_parts_in)) == NULL)
      error("Failed to re-allocate parts_in buffers.");
  }
258
259
260
261
262
  if (p->nr_gparts_in > p->size_gparts_in) {
    do {
      p->size_gparts_in *= proxy_buffgrow;
    } while (p->nr_gparts_in > p->size_gparts_in);
    free(p->gparts_in);
Pedro Gonnet's avatar
Pedro Gonnet committed
263
264
    if ((p->gparts_in = (struct gpart *)malloc(sizeof(struct gpart) *
                                               p->size_gparts_in)) == NULL)
265
266
      error("Failed to re-allocate gparts_in buffers.");
  }
267
268
269

  /* Receive the particle buffers. */
  if (p->nr_parts_in > 0) {
270
271
272
273
274
    if (MPI_Irecv(p->parts_in, p->nr_parts_in, part_mpi_type, p->nodeID,
                  p->nodeID * proxy_tag_shift + proxy_tag_parts, MPI_COMM_WORLD,
                  &p->req_parts_in) != MPI_SUCCESS ||
        MPI_Irecv(p->xparts_in, p->nr_parts_in, xpart_mpi_type, p->nodeID,
                  p->nodeID * proxy_tag_shift + proxy_tag_xparts,
275
                  MPI_COMM_WORLD, &p->req_xparts_in) != MPI_SUCCESS)
276
277
278
279
      error("Failed to irecv part data.");
    // message( "irecv particle data (%i) from node %i." , p->nr_parts_in ,
    // p->nodeID ); fflush(stdout);
  }
280
  if (p->nr_gparts_in > 0) {
281
    if (MPI_Irecv(p->gparts_in, p->nr_gparts_in, gpart_mpi_type, p->nodeID,
Pedro Gonnet's avatar
Pedro Gonnet committed
282
                  p->nodeID * proxy_tag_shift + proxy_tag_gparts,
283
284
285
286
287
                  MPI_COMM_WORLD, &p->req_gparts_in) != MPI_SUCCESS)
      error("Failed to irecv gpart data.");
    // message( "irecv gpart data (%i) from node %i." , p->nr_gparts_in ,
    // p->nodeID ); fflush(stdout);
  }
288
289

#else
290
  error("SWIFT was not compiled with MPI support.");
291
#endif
292
}
293
294
295
296
297
298
299
300
301
302

/**
 * @brief Load parts onto a proxy for exchange.
 *
 * @param p The #proxy.
 * @param parts Pointer to an array of #part to send.
 * @param xparts Pointer to an array of #xpart to send.
 * @param N The number of parts.
 */

Pedro Gonnet's avatar
const.    
Pedro Gonnet committed
303
304
void proxy_parts_load(struct proxy *p, const struct part *parts,
                      const struct xpart *xparts, int N) {
305
306
307
308
309
310
311
312

  /* Is there enough space in the buffer? */
  if (p->nr_parts_out + N > p->size_parts_out) {
    do {
      p->size_parts_out *= proxy_buffgrow;
    } while (p->nr_parts_out + N > p->size_parts_out);
    struct part *tp;
    struct xpart *txp;
313
    if ((tp = (struct part *)malloc(sizeof(struct part) * p->size_parts_out)) ==
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
            NULL ||
        (txp = (struct xpart *)malloc(sizeof(struct xpart) *
                                      p->size_parts_out)) == NULL)
      error("Failed to re-allocate parts_out buffers.");
    memcpy(tp, p->parts_out, sizeof(struct part) * p->nr_parts_out);
    memcpy(txp, p->xparts_out, sizeof(struct xpart) * p->nr_parts_out);
    free(p->parts_out);
    free(p->xparts_out);
    p->parts_out = tp;
    p->xparts_out = txp;
  }

  /* Copy the parts and xparts data to the buffer. */
  memcpy(&p->parts_out[p->nr_parts_out], parts, sizeof(struct part) * N);
  memcpy(&p->xparts_out[p->nr_parts_out], xparts, sizeof(struct xpart) * N);

  /* Increase the counters. */
  p->nr_parts_out += N;
}
333

334
335
336
337
338
339
340
341
/**
 * @brief Load parts onto a proxy for exchange.
 *
 * @param p The #proxy.
 * @param gparts Pointer to an array of #gpart to send.
 * @param N The number of parts.
 */

Pedro Gonnet's avatar
const.    
Pedro Gonnet committed
342
void proxy_gparts_load(struct proxy *p, const struct gpart *gparts, int N) {
343
344
345
346
347
348
349

  /* Is there enough space in the buffer? */
  if (p->nr_gparts_out + N > p->size_gparts_out) {
    do {
      p->size_gparts_out *= proxy_buffgrow;
    } while (p->nr_gparts_out + N > p->size_gparts_out);
    struct gpart *tp;
Pedro Gonnet's avatar
Pedro Gonnet committed
350
351
    if ((tp = (struct gpart *)malloc(sizeof(struct gpart) *
                                     p->size_gparts_out)) == NULL)
352
353
354
355
356
357
358
      error("Failed to re-allocate gparts_out buffers.");
    memcpy(tp, p->gparts_out, sizeof(struct gpart) * p->nr_gparts_out);
    free(p->gparts_out);
    p->gparts_out = tp;
  }

  /* Copy the parts and xparts data to the buffer. */
Pedro Gonnet's avatar
Pedro Gonnet committed
359
  memcpy(&p->gparts_out[p->nr_gparts_out], gparts, sizeof(struct gpart) * N);
360
361
362
363
364

  /* Increase the counters. */
  p->nr_gparts_out += N;
}

365
366
367
368
/**
 * @brief Initialize the given proxy.
 *
 * @param p The #proxy.
369
 * @param mynodeID The node this proxy is running on.
370
371
 * @param nodeID The node with which this proxy will communicate.
 */
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397

void proxy_init(struct proxy *p, int mynodeID, int nodeID) {

  /* Set the nodeID. */
  p->mynodeID = mynodeID;
  p->nodeID = nodeID;

  /* Allocate the cell send and receive buffers, if needed. */
  if (p->cells_in == NULL) {
    p->size_cells_in = proxy_buffinit;
    if ((p->cells_in =
             (struct cell **)malloc(sizeof(void *) * p->size_cells_in)) == NULL)
      error("Failed to allocate cells_in buffer.");
  }
  p->nr_cells_in = 0;
  if (p->cells_out == NULL) {
    p->size_cells_out = proxy_buffinit;
    if ((p->cells_out = (struct cell **)malloc(sizeof(void *) *
                                               p->size_cells_out)) == NULL)
      error("Failed to allocate cells_out buffer.");
  }
  p->nr_cells_out = 0;

  /* Allocate the part send and receive buffers, if needed. */
  if (p->parts_in == NULL) {
    p->size_parts_in = proxy_buffinit;
398
399
    if ((p->parts_in = (struct part *)malloc(sizeof(struct part) *
                                             p->size_parts_in)) == NULL ||
400
401
402
403
404
405
406
        (p->xparts_in = (struct xpart *)malloc(sizeof(struct xpart) *
                                               p->size_parts_in)) == NULL)
      error("Failed to allocate parts_in buffers.");
  }
  p->nr_parts_in = 0;
  if (p->parts_out == NULL) {
    p->size_parts_out = proxy_buffinit;
407
408
    if ((p->parts_out = (struct part *)malloc(sizeof(struct part) *
                                              p->size_parts_out)) == NULL ||
409
410
411
412
413
        (p->xparts_out = (struct xpart *)malloc(sizeof(struct xpart) *
                                                p->size_parts_out)) == NULL)
      error("Failed to allocate parts_out buffers.");
  }
  p->nr_parts_out = 0;
414
415
416
417

  /* Allocate the gpart send and receive buffers, if needed. */
  if (p->gparts_in == NULL) {
    p->size_gparts_in = proxy_buffinit;
Pedro Gonnet's avatar
Pedro Gonnet committed
418
419
    if ((p->gparts_in = (struct gpart *)malloc(sizeof(struct gpart) *
                                               p->size_gparts_in)) == NULL)
420
421
422
423
424
      error("Failed to allocate gparts_in buffers.");
  }
  p->nr_gparts_in = 0;
  if (p->gparts_out == NULL) {
    p->size_gparts_out = proxy_buffinit;
Pedro Gonnet's avatar
Pedro Gonnet committed
425
426
    if ((p->gparts_out = (struct gpart *)malloc(sizeof(struct gpart) *
                                                p->size_gparts_out)) == NULL)
427
428
429
      error("Failed to allocate gparts_out buffers.");
  }
  p->nr_gparts_out = 0;
430
}