From 068ef2af50dadc3f1eb2f23d620d1f15143d2b4e Mon Sep 17 00:00:00 2001
From: Matthieu Schaller <matthieu.schaller@durham.ac.uk>
Date: Wed, 6 Jan 2016 23:56:15 +0000
Subject: [PATCH] Created an MPI type for the particles. This is useful for
particle exchanges when redistributing. Up to 2^31 particles can now be
exchanged at once.
---
src/Makefile.am | 2 +-
src/engine.c | 27 +++++++++++----------
src/engine.h | 12 ++++++++++
src/part.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++
src/part.h | 21 +++++++++++++++--
5 files changed, 109 insertions(+), 15 deletions(-)
create mode 100644 src/part.c
diff --git a/src/Makefile.am b/src/Makefile.am
index 0fdc515714..c71cbbbbd1 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -41,7 +41,7 @@ include_HEADERS = space.h runner.h queue.h task.h lock.h cell.h part.h const.h \
AM_SOURCES = space.c runner.c queue.c task.c cell.c engine.c \
serial_io.c timers.c debug.c scheduler.c proxy.c parallel_io.c \
units.c common_io.c single_io.c multipole.c version.c map.c \
- kernel.c tools.c
+ kernel.c tools.c part.c
# Include files for distribution, not installation.
noinst_HEADERS = atomic.h cycle.h error.h inline.h kernel.h vector.h \
diff --git a/src/engine.c b/src/engine.c
index 2f242f2ec3..c9a1de94cf 100644
--- a/src/engine.c
+++ b/src/engine.c
@@ -47,6 +47,7 @@
#include "cycle.h"
#include "debug.h"
#include "error.h"
+#include "part.h"
#include "timers.h"
#ifdef LEGACY_GADGET2_SPH
@@ -215,28 +216,24 @@ void engine_redistribute(struct engine *e) {
offset_send += counts[ind_send];
offset_recv += counts[ind_recv];
} else {
- if (MPI_Isend(&s->parts[offset_send],
- sizeof(struct part) * counts[ind_send], MPI_BYTE, k,
- 2 * ind_send + 0, MPI_COMM_WORLD,
+ if (MPI_Isend(&s->parts[offset_send], counts[ind_send],
+ *(e->part_mpi_type), k, 2 * ind_send + 0, MPI_COMM_WORLD,
&reqs[4 * k]) != MPI_SUCCESS)
error("Failed to isend parts to node %i.", k);
- if (MPI_Isend(&s->xparts[offset_send],
- sizeof(struct xpart) * counts[ind_send], MPI_BYTE, k,
- 2 * ind_send + 1, MPI_COMM_WORLD,
+ if (MPI_Isend(&s->xparts[offset_send], counts[ind_send],
+ *(e->xpart_mpi_type), k, 2 * ind_send + 1, MPI_COMM_WORLD,
&reqs[4 * k + 1]) != MPI_SUCCESS)
error("Failed to isend xparts to node %i.", k);
offset_send += counts[ind_send];
}
}
if (k != nodeID && counts[ind_recv] > 0) {
- if (MPI_Irecv(&parts_new[offset_recv],
- sizeof(struct part) * counts[ind_recv], MPI_BYTE, k,
- 2 * ind_recv + 0, MPI_COMM_WORLD,
+ if (MPI_Irecv(&parts_new[offset_recv], counts[ind_recv],
+ *(e->part_mpi_type), k, 2 * ind_recv + 0, MPI_COMM_WORLD,
&reqs[4 * k + 2]) != MPI_SUCCESS)
error("Failed to emit irecv of parts from node %i.", k);
- if (MPI_Irecv(&xparts_new[offset_recv],
- sizeof(struct xpart) * counts[ind_recv], MPI_BYTE, k,
- 2 * ind_recv + 1, MPI_COMM_WORLD,
+ if (MPI_Irecv(&xparts_new[offset_recv], counts[ind_recv],
+ *(e->xpart_mpi_type), k, 2 * ind_recv + 1, MPI_COMM_WORLD,
&reqs[4 * k + 3]) != MPI_SUCCESS)
error("Failed to emit irecv of parts from node %i.", k);
offset_recv += counts[ind_recv];
@@ -2202,6 +2199,12 @@ void engine_init(struct engine *e, struct space *s, float dt, int nr_threads,
#endif
}
+/* Construct types for MPI communications */
+#ifdef WITH_MPI
+ part_create_mpi_type(e->part_mpi_type);
+ xpart_create_mpi_type(e->xpart_mpi_type);
+#endif
+
/* First of all, init the barrier and lock it. */
if (pthread_mutex_init(&e->barrier_mutex, NULL) != 0)
error("Failed to initialize barrier mutex.");
diff --git a/src/engine.h b/src/engine.h
index 4da743f315..4ab86288a0 100644
--- a/src/engine.h
+++ b/src/engine.h
@@ -19,6 +19,14 @@
#ifndef SWIFT_ENGINE_H
#define SWIFT_ENGINE_H
+/* Config parameters. */
+#include "../config.h"
+
+/* MPI headers. */
+#ifdef WITH_MPI
+#include <mpi.h>
+#endif
+
/* Some standard headers. */
#include <pthread.h>
@@ -125,6 +133,10 @@ struct engine {
/* Linked list for cell-task association. */
struct link *links;
int nr_links, size_links;
+
+ /* MPI data type for the particle transfers */
+ MPI_Datatype *part_mpi_type;
+ MPI_Datatype *xpart_mpi_type;
};
/* Function prototypes. */
diff --git a/src/part.c b/src/part.c
new file mode 100644
index 0000000000..1923e3f044
--- /dev/null
+++ b/src/part.c
@@ -0,0 +1,62 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2016 Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* MPI headers. */
+#ifdef WITH_MPI
+#include <mpi.h>
+#endif
+
+/* This object's header. */
+#include "part.h"
+
+#ifdef WITH_MPI
+/**
+ * @brief Registers and returns an MPI type for the particles
+ *
+ */
+void part_create_mpi_type(MPI_Datatype* part_type) {
+
+ /* This is not the recommended way of doing this. */
+ /* One should define the structure field by field */
+ /* But as long as we don't do serialization via MPI-IO */
+ /* we don't really care. */
+ MPI_Type_contiguous(sizeof(struct part) / sizeof(unsigned char), MPI_BYTE,
+ part_type);
+ MPI_Type_commit(part_type);
+}
+
+/**
+ * @brief Registers and returns an MPI type for the xparticles
+ *
+ */
+void xpart_create_mpi_type(MPI_Datatype* xpart_type) {
+
+ /* This is not the recommended way of doing this. */
+ /* One should define the structure field by field */
+ /* But as long as we don't do serialization via MPI-IO */
+ /* we don't really care. */
+ MPI_Type_contiguous(sizeof(struct xpart) / sizeof(unsigned char), MPI_BYTE,
+ xpart_type);
+ MPI_Type_commit(xpart_type);
+}
+
+#endif
diff --git a/src/part.h b/src/part.h
index 1ad62888b3..e2940e29c1 100644
--- a/src/part.h
+++ b/src/part.h
@@ -19,6 +19,18 @@
#ifndef SWIFT_PART_H
#define SWIFT_PART_H
+/* Config parameters. */
+#include "../config.h"
+
+/* Some standard headers. */
+#include <stdlib.h>
+
+/* MPI headers. */
+#ifdef WITH_MPI
+#include <mpi.h>
+#endif
+
+/* Local headers. */
#include "const.h"
/* Some constants. */
@@ -72,7 +84,7 @@ struct gpart {
size_t id;
/* Pointer to corresponding SPH part. */
- struct part *part;
+ struct part* part;
};
} __attribute__((aligned(part_align)));
@@ -162,8 +174,13 @@ struct part {
unsigned long long id;
/* Associated gravitas. */
- struct gpart *gpart;
+ struct gpart* gpart;
} __attribute__((aligned(part_align)));
+#ifdef WITH_MPI
+void part_create_mpi_type(MPI_Datatype* part_type);
+void xpart_create_mpi_type(MPI_Datatype* xpart_type);
+#endif
+
#endif /* SWIFT_PART_H */
--
GitLab