diff --git a/src/Makefile.am b/src/Makefile.am
index 0fdc51571424e2a9ad91d1d7c714d91446702161..c71cbbbbd1b1463a61e8b138fb85525d74ebbe03 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -41,7 +41,7 @@ include_HEADERS = space.h runner.h queue.h task.h lock.h cell.h part.h const.h \
AM_SOURCES = space.c runner.c queue.c task.c cell.c engine.c \
serial_io.c timers.c debug.c scheduler.c proxy.c parallel_io.c \
units.c common_io.c single_io.c multipole.c version.c map.c \
- kernel.c tools.c
+ kernel.c tools.c part.c
# Include files for distribution, not installation.
noinst_HEADERS = atomic.h cycle.h error.h inline.h kernel.h vector.h \
diff --git a/src/engine.c b/src/engine.c
index d0a80854a7de6c5e58df131288792e4dc312ce53..cd4d6944ef43f1ac6d580384cb3a616e7644a03f 100644
--- a/src/engine.c
+++ b/src/engine.c
@@ -47,6 +47,7 @@
#include "cycle.h"
#include "debug.h"
#include "error.h"
+#include "part.h"
#include "timers.h"
#ifdef LEGACY_GADGET2_SPH
@@ -215,28 +216,24 @@ void engine_redistribute(struct engine *e) {
offset_send += counts[ind_send];
offset_recv += counts[ind_recv];
} else {
- if (MPI_Isend(&s->parts[offset_send],
- sizeof(struct part) * counts[ind_send], MPI_BYTE, k,
- 2 * ind_send + 0, MPI_COMM_WORLD,
+ if (MPI_Isend(&s->parts[offset_send], counts[ind_send],
+ e->part_mpi_type, k, 2 * ind_send + 0, MPI_COMM_WORLD,
&reqs[4 * k]) != MPI_SUCCESS)
error("Failed to isend parts to node %i.", k);
- if (MPI_Isend(&s->xparts[offset_send],
- sizeof(struct xpart) * counts[ind_send], MPI_BYTE, k,
- 2 * ind_send + 1, MPI_COMM_WORLD,
+ if (MPI_Isend(&s->xparts[offset_send], counts[ind_send],
+ e->xpart_mpi_type, k, 2 * ind_send + 1, MPI_COMM_WORLD,
&reqs[4 * k + 1]) != MPI_SUCCESS)
error("Failed to isend xparts to node %i.", k);
offset_send += counts[ind_send];
}
}
if (k != nodeID && counts[ind_recv] > 0) {
- if (MPI_Irecv(&parts_new[offset_recv],
- sizeof(struct part) * counts[ind_recv], MPI_BYTE, k,
- 2 * ind_recv + 0, MPI_COMM_WORLD,
+ if (MPI_Irecv(&parts_new[offset_recv], counts[ind_recv], e->part_mpi_type,
+ k, 2 * ind_recv + 0, MPI_COMM_WORLD,
&reqs[4 * k + 2]) != MPI_SUCCESS)
error("Failed to emit irecv of parts from node %i.", k);
- if (MPI_Irecv(&xparts_new[offset_recv],
- sizeof(struct xpart) * counts[ind_recv], MPI_BYTE, k,
- 2 * ind_recv + 1, MPI_COMM_WORLD,
+ if (MPI_Irecv(&xparts_new[offset_recv], counts[ind_recv],
+ e->xpart_mpi_type, k, 2 * ind_recv + 1, MPI_COMM_WORLD,
&reqs[4 * k + 3]) != MPI_SUCCESS)
error("Failed to emit irecv of parts from node %i.", k);
offset_recv += counts[ind_recv];
@@ -2190,7 +2187,7 @@ void engine_init(struct engine *e, struct space *s, float dt, int nr_threads,
/* Are we doing stuff in parallel? */
if (nr_nodes > 1) {
-#ifndef HAVE_MPI
+#ifndef WITH_MPI
error("SWIFT was not compiled with MPI support.");
#else
e->policy |= engine_policy_mpi;
@@ -2202,6 +2199,12 @@ void engine_init(struct engine *e, struct space *s, float dt, int nr_threads,
#endif
}
+/* Construct types for MPI communications */
+#ifdef WITH_MPI
+ part_create_mpi_type(&e->part_mpi_type);
+ xpart_create_mpi_type(&e->xpart_mpi_type);
+#endif
+
/* First of all, init the barrier and lock it. */
if (pthread_mutex_init(&e->barrier_mutex, NULL) != 0)
error("Failed to initialize barrier mutex.");
diff --git a/src/engine.h b/src/engine.h
index 4da743f3151704b2d32a6086352957e4d947bfc6..017455dc7da812f20703686e5d6b4715862546f1 100644
--- a/src/engine.h
+++ b/src/engine.h
@@ -19,6 +19,14 @@
#ifndef SWIFT_ENGINE_H
#define SWIFT_ENGINE_H
+/* Config parameters. */
+#include "../config.h"
+
+/* MPI headers. */
+#ifdef WITH_MPI
+#include <mpi.h>
+#endif
+
/* Some standard headers. */
#include <pthread.h>
@@ -125,6 +133,12 @@ struct engine {
/* Linked list for cell-task association. */
struct link *links;
int nr_links, size_links;
+
+#ifdef WITH_MPI
+ /* MPI data type for the particle transfers */
+ MPI_Datatype part_mpi_type;
+ MPI_Datatype xpart_mpi_type;
+#endif
};
/* Function prototypes. */
diff --git a/src/part.c b/src/part.c
new file mode 100644
index 0000000000000000000000000000000000000000..6d49887ad72d6afdd1f419cfdaca88090e781f3e
--- /dev/null
+++ b/src/part.c
@@ -0,0 +1,68 @@
+/*******************************************************************************
+ * This file is part of SWIFT.
+ * Copyright (c) 2016 Matthieu Schaller (matthieu.schaller@durham.ac.uk)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ ******************************************************************************/
+
+/* Config parameters. */
+#include "../config.h"
+
+/* MPI headers. */
+#ifdef WITH_MPI
+#include <mpi.h>
+#endif
+
+/* This object's header. */
+#include "part.h"
+
+#ifdef WITH_MPI
+/**
+ * @brief Registers and returns an MPI type for the particles
+ *
+ * @param part_type The type container
+ */
+void part_create_mpi_type(MPI_Datatype* part_type) {
+
+ /* This is not the recommended way of doing this.
+ One should define the structure field by field
+ But as long as we don't do serialization via MPI-IO
+ we don't really care.
+ Also we would have to modify this function everytime something
+ is added to the part structure. */
+ MPI_Type_contiguous(sizeof(struct part) / sizeof(unsigned char), MPI_BYTE,
+ part_type);
+ MPI_Type_commit(part_type);
+}
+
+/**
+ * @brief Registers and returns an MPI type for the xparticles
+ *
+ * @param xpart_type The type container
+ */
+void xpart_create_mpi_type(MPI_Datatype* xpart_type) {
+
+ /* This is not the recommended way of doing this.
+ One should define the structure field by field
+ But as long as we don't do serialization via MPI-IO
+ we don't really care.
+ Also we would have to modify this function everytime something
+ is added to the part structure. */
+ MPI_Type_contiguous(sizeof(struct xpart) / sizeof(unsigned char), MPI_BYTE,
+ xpart_type);
+ MPI_Type_commit(xpart_type);
+}
+
+#endif
diff --git a/src/part.h b/src/part.h
index 1ad62888b3accd38c6c3f5e13f49988d2c6d69d1..e2940e29c1b008b55efbb7dfa178bae80c134add 100644
--- a/src/part.h
+++ b/src/part.h
@@ -19,6 +19,18 @@
#ifndef SWIFT_PART_H
#define SWIFT_PART_H
+/* Config parameters. */
+#include "../config.h"
+
+/* Some standard headers. */
+#include <stdlib.h>
+
+/* MPI headers. */
+#ifdef WITH_MPI
+#include <mpi.h>
+#endif
+
+/* Local headers. */
#include "const.h"
/* Some constants. */
@@ -72,7 +84,7 @@ struct gpart {
size_t id;
/* Pointer to corresponding SPH part. */
- struct part *part;
+ struct part* part;
};
} __attribute__((aligned(part_align)));
@@ -162,8 +174,13 @@ struct part {
unsigned long long id;
/* Associated gravitas. */
- struct gpart *gpart;
+ struct gpart* gpart;
} __attribute__((aligned(part_align)));
+#ifdef WITH_MPI
+void part_create_mpi_type(MPI_Datatype* part_type);
+void xpart_create_mpi_type(MPI_Datatype* xpart_type);
+#endif
+
#endif /* SWIFT_PART_H */
diff --git a/src/scheduler.c b/src/scheduler.c
index ae83ddc0ff5a0fcf65f515fe61ba40cdeb26be96..7b26b866a6d9d376cb18ad9c25979bf95a296ae5 100644
--- a/src/scheduler.c
+++ b/src/scheduler.c
@@ -1048,8 +1048,8 @@ void scheduler_enqueue(struct scheduler *s, struct task *t) {
break;
case task_type_recv:
#ifdef WITH_MPI
- if ((err = MPI_Irecv(t->ci->parts, sizeof(struct part) * t->ci->count,
- MPI_BYTE, t->ci->nodeID, t->flags, MPI_COMM_WORLD,
+ if ((err = MPI_Irecv(t->ci->parts, t->ci->count, s->part_mpi_type,
+ t->ci->nodeID, t->flags, MPI_COMM_WORLD,
&t->req)) != MPI_SUCCESS) {
char buff[MPI_MAX_ERROR_STRING];
int len;
@@ -1066,9 +1066,9 @@ void scheduler_enqueue(struct scheduler *s, struct task *t) {
break;
case task_type_send:
#ifdef WITH_MPI
- if ((err = MPI_Isend(t->ci->parts, sizeof(struct part) * t->ci->count,
- MPI_BYTE, t->cj->nodeID, t->flags, MPI_COMM_WORLD,
- &t->req)) != MPI_SUCCESS) {
+ if ((err = MPI_Isend(t->ci->parts, t->ci->count, s->part_mpi_type,
+ t->cj->nodeID, t->flags, MPI_COMM_WORLD,
+ &t->req)) != MPI_SUCCESS) {
char buff[MPI_MAX_ERROR_STRING];
int len;
MPI_Error_string(err, buff, &len);
@@ -1306,4 +1306,10 @@ void scheduler_init(struct scheduler *s, struct space *space, int nr_tasks,
s->tasks = NULL;
s->tasks_ind = NULL;
scheduler_reset(s, nr_tasks);
+
+/* Construct types for MPI communications */
+#ifdef WITH_MPI
+ part_create_mpi_type(&s->part_mpi_type);
+ xpart_create_mpi_type(&s->xpart_mpi_type);
+#endif
}
diff --git a/src/scheduler.h b/src/scheduler.h
index 9bdb9b27da4089723dfcd6e93912f958f4811ab6..08aadf5454525f11a5cbb5ef8cb0f4c135aa1ad2 100644
--- a/src/scheduler.h
+++ b/src/scheduler.h
@@ -20,6 +20,14 @@
#ifndef SWIFT_SCHEDULER_H
#define SWIFT_SCHEDULER_H
+/* Config parameters. */
+#include "../config.h"
+
+/* MPI headers. */
+#ifdef WITH_MPI
+#include <mpi.h>
+#endif
+
/* Some standard headers. */
#include <pthread.h>
@@ -89,6 +97,12 @@ struct scheduler {
/* The node we are working on. */
int nodeID;
+
+#ifdef WITH_MPI
+ /* MPI data type for the particle transfers */
+ MPI_Datatype part_mpi_type;
+ MPI_Datatype xpart_mpi_type;
+#endif
};
/* Function prototypes. */