diff --git a/src/engine.c b/src/engine.c index 5a89555c988fc5d4f5456e75b6fab67f03469fe6..2e2157aaa6014917be8fbe151afc2a073962a08c 100644 --- a/src/engine.c +++ b/src/engine.c @@ -32,7 +32,7 @@ #include <stdbool.h> /* MPI headers. */ -#ifdef WITH_MPI +#if 1 //#ifdef WITH_MPI #include <mpi.h> #endif @@ -139,26 +139,34 @@ void engine_make_ghost_tasks(struct engine *e, struct cell *c, * @brief Redistribute the particles amongst the nodes according * to their cell's node IDs. * + * The strategy here is as follows: + * 1) Each node counts the number of particles it has to send to each other + * node. + * 2) The number of particles of each type is then exchanged. + * 3) Each node allocates enough space for the new particles. + * 4) (Asynchronous) communications are issued to transfer the data. + * + * To simplify everything, only gparts without counterparts are transferred. + * gparts linked to a part are re-created from the part data on the receiveing + * side. + * * @param e The #engine. */ - void engine_redistribute(struct engine *e) { -#ifdef WITH_MPI +#if 1 //#ifdef WITH_MPI - int nr_nodes = e->nr_nodes, nodeID = e->nodeID; + const int nr_nodes = e->nr_nodes; + const int nodeID = e->nodeID; struct space *s = e->s; - int my_cells = 0; - int *cdim = s->cdim; + const int *cdim = s->cdim; struct cell *cells = s->cells; - int nr_cells = s->nr_cells; + const int nr_cells = s->nr_cells; ticks tic = getticks(); /* Start by sorting the particles according to their nodes and getting the counts. The counts array is indexed as count[from * nr_nodes + to]. */ - int *counts; - size_t *dest; double ih[3], dim[3]; ih[0] = s->ih[0]; ih[1] = s->ih[1]; @@ -166,6 +174,8 @@ void engine_redistribute(struct engine *e) { dim[0] = s->dim[0]; dim[1] = s->dim[1]; dim[2] = s->dim[2]; + int *counts; + size_t *dest; if ((counts = (int *)malloc(sizeof(int) *nr_nodes *nr_nodes)) == NULL || (dest = (size_t *)malloc(sizeof(size_t) * s->nr_parts)) == NULL) error("Failed to allocate count and dest buffers."); @@ -2175,6 +2185,7 @@ void engine_init(struct engine *e, struct space *s, float dt, int nr_threads, #ifdef WITH_MPI part_create_mpi_type(&e->part_mpi_type); xpart_create_mpi_type(&e->xpart_mpi_type); + gpart_create_mpi_type(&e->gpart_mpi_type); #endif /* First of all, init the barrier and lock it. */ diff --git a/src/engine.h b/src/engine.h index 741ae1f553494e435394f529606b4cb794b0e3d2..5e1dfe4c1b3339c17756e4c17a298544f183e82d 100644 --- a/src/engine.h +++ b/src/engine.h @@ -166,6 +166,11 @@ struct engine { MPI_Datatype part_mpi_type; MPI_Datatype xpart_mpi_type; #endif + + int part_mpi_tpye; + int xpart_mpi_tpye; + int gpart_mpi_tpye; + }; /* Function prototypes. */ diff --git a/src/part.c b/src/part.c index 6a99325ef23a7062fafb387fa3f3bd6b2203d057..bf4adcf270ac54cd79887481486f946b040e8c28 100644 --- a/src/part.c +++ b/src/part.c @@ -65,4 +65,22 @@ void xpart_create_mpi_type(MPI_Datatype* xpart_type) { MPI_Type_commit(xpart_type); } +/** + * @brief Registers and returns an MPI type for the gparticles + * + * @param gpart_type The type container + */ +void gpart_create_mpi_type(MPI_Datatype* gpart_type) { + + /* This is not the recommended way of doing this. + One should define the structure field by field + But as long as we don't do serialization via MPI-IO + we don't really care. + Also we would have to modify this function everytime something + is added to the part structure. */ + MPI_Type_contiguous(sizeof(struct gpart) / sizeof(unsigned char), MPI_BYTE, + gpart_type); + MPI_Type_commit(gpart_type); +} + #endif diff --git a/src/part.h b/src/part.h index 865403e8c2c157dc5a8ff7a32bc41be676d7919b..03eb28a648070e2735dfc20f4d50730aba7aed2c 100644 --- a/src/part.h +++ b/src/part.h @@ -54,6 +54,7 @@ #ifdef WITH_MPI void part_create_mpi_type(MPI_Datatype* part_type); void xpart_create_mpi_type(MPI_Datatype* xpart_type); +void gpart_create_mpi_type(MPI_Datatype* gpart_type); #endif #endif /* SWIFT_PART_H */