diff --git a/examples/main.c b/examples/main.c index cb453a74605f7ac05f9aaee1c1c63dc14f8bf522..8429838cf5c78c70bc65f19c6b1c963a544c9166 100644 --- a/examples/main.c +++ b/examples/main.c @@ -1394,11 +1394,6 @@ int main(int argc, char *argv[]) { #endif } -#ifdef WITH_MPI - if ((res = MPI_Finalize()) != MPI_SUCCESS) - error("call to MPI_Finalize failed with error %i.", res); -#endif - /* Remove the stop file if used. Do this anyway, we could have missed the * stop file if normal exit happened first. */ if (myrank == 0) force_stop = restart_stop_now(restart_dir, 1); @@ -1420,6 +1415,11 @@ int main(int argc, char *argv[]) { engine_clean(&e, /*fof=*/0); free(params); +#ifdef WITH_MPI + if ((res = MPI_Finalize()) != MPI_SUCCESS) + error("call to MPI_Finalize failed with error %i.", res); +#endif + /* Say goodbye. */ if (myrank == 0) message("done. Bye."); diff --git a/src/collectgroup.c b/src/collectgroup.c index 8dd9e974f41c979f4c50e84fb555f9b80db19e25..af140c9150a640829ce783149f0c68bc7e732975 100644 --- a/src/collectgroup.c +++ b/src/collectgroup.c @@ -383,4 +383,9 @@ static void mpicollect_create_MPI_type(void) { /* Create the reduction operation */ MPI_Op_create(mpicollectgroup1_reduce, 1, &mpicollectgroup1_reduce_op); } + +void mpicollect_free_MPI_type(void) { + MPI_Type_free(&mpicollectgroup1_type); + MPI_Op_free(&mpicollectgroup1_reduce_op); +} #endif diff --git a/src/collectgroup.h b/src/collectgroup.h index 4e5dd0e0f65298e3873c9b9c47d021d6b6c0c415..9e973142953beec552e8c76fdbb789dc6325587a 100644 --- a/src/collectgroup.h +++ b/src/collectgroup.h @@ -77,5 +77,7 @@ void collectgroup1_init( long long total_nr_cells, long long total_nr_tasks, float tasks_per_cell, const struct star_formation_history sfh); void collectgroup1_reduce(struct collectgroup1 *grp1); - +#ifdef WITH_MPI +void mpicollect_free_MPI_type(void); +#endif #endif /* SWIFT_COLLECTGROUP_H */ diff --git a/src/engine.c b/src/engine.c index 55a1f1d6081fe8c114ca39202a9d3bef5d02fc10..8479d6cb71af43ad615b8f334bd1e4a51f7e6f12 100644 --- a/src/engine.c +++ b/src/engine.c @@ -37,6 +37,7 @@ /* MPI headers. */ #ifdef WITH_MPI + #include <mpi.h> #endif @@ -1113,8 +1114,7 @@ void engine_allocate_foreign_particles(struct engine *e) { /* Allocate space for the foreign particles we will receive */ if (count_parts_in > s->size_parts_foreign) { - if (s->parts_foreign != NULL) - swift_free("sparts_foreign", s->parts_foreign); + if (s->parts_foreign != NULL) swift_free("parts_foreign", s->parts_foreign); s->size_parts_foreign = engine_foreign_alloc_margin * count_parts_in; if (swift_memalign("parts_foreign", (void **)&s->parts_foreign, part_align, sizeof(struct part) * s->size_parts_foreign) != 0) @@ -4687,6 +4687,21 @@ void engine_clean(struct engine *e, const int fof) { scheduler_clean(&e->sched); space_clean(e->s); threadpool_clean(&e->threadpool); +#if defined(WITH_MPI) + for (int i = 0; i < e->nr_proxies; ++i) { + proxy_clean(&e->proxies[i]); + } + free(e->proxy_ind); + free(e->proxies); + + /* Free types */ + part_free_mpi_types(); + multipole_free_mpi_types(); + stats_free_mpi_type(); + proxy_free_mpi_type(); + task_free_mpi_comms(); + mpicollect_free_MPI_type(); +#endif /* Close files */ if (!fof && e->nodeID == 0) { diff --git a/src/feedback/EAGLE/feedback.c b/src/feedback/EAGLE/feedback.c index c280d6cd829d819bede91bf17df0b8861ad9fbbc..d041969e808ac95a4f7f4de06a4d2cb8ccaf41d7 100644 --- a/src/feedback/EAGLE/feedback.c +++ b/src/feedback/EAGLE/feedback.c @@ -717,6 +717,14 @@ void compute_stellar_evolution(const struct feedback_props* feedback_props, /* Properties collected in the stellar density loop. */ const float ngb_gas_mass = sp->feedback_data.to_collect.ngb_mass; + + /* Check if there are neighbours, otherwise exit */ + if (ngb_gas_mass == 0.f) { + feedback_reset_feedback(sp, feedback_props); + return; + } + + /* Update the enrichment weights */ const float enrichment_weight_inv = sp->feedback_data.to_collect.enrichment_weight_inv; diff --git a/src/feedback/EAGLE/feedback_iact.h b/src/feedback/EAGLE/feedback_iact.h index 582294a24a1574498914ba3d5aa0a50bac566a73..12f6b0bced2e1f69dff7e6571408295179d1fbaa 100644 --- a/src/feedback/EAGLE/feedback_iact.h +++ b/src/feedback/EAGLE/feedback_iact.h @@ -250,7 +250,11 @@ runner_iact_nonsym_feedback_apply(const float r2, const float *dx, const double injected_energy = si->feedback_data.to_distribute.energy * Omega_frac; - /* Apply energy conservation to recover the new thermal energy of the gas */ + /* Apply energy conservation to recover the new thermal energy of the gas + * Note: in some specific cases the new_thermal_energy could be lower + * than the current_thermal_energy, this is mainly the case if the change + * in mass is relatively small and the velocity vectors between both the + * gas particle and the star particle have a small angle. */ const double new_thermal_energy = current_kinetic_energy_gas + current_thermal_energy + injected_energy - new_kinetic_energy_gas; @@ -258,11 +262,6 @@ runner_iact_nonsym_feedback_apply(const float r2, const float *dx, /* Convert this to a specific thermal energy */ const double u_new_enrich = new_thermal_energy * new_mass_inv; -#ifdef SWIFT_DEBUG_CHECKS - if (new_thermal_energy < 0.99 * current_thermal_energy) - error("Enrichment is cooling the gas"); -#endif - /* Do the energy injection. */ hydro_set_physical_internal_energy(pj, xpj, cosmo, u_new_enrich); hydro_set_drifted_physical_internal_energy(pj, cosmo, u_new_enrich); diff --git a/src/multipole.c b/src/multipole.c index a77e6fce297802fb4118b7ac3d4c6a9bf4ecfd22..03adccd0d4386d837c2294307e51fbeb144bcfb7 100644 --- a/src/multipole.c +++ b/src/multipole.c @@ -86,4 +86,8 @@ void multipole_create_mpi_types(void) { MPI_Op_create(gravity_tensors_mpi_reduce, 1, &multipole_mpi_reduce_op); } +void multipole_free_mpi_types(void) { + MPI_Type_free(&multipole_mpi_type); + MPI_Op_free(&multipole_mpi_reduce_op); +} #endif diff --git a/src/multipole.h b/src/multipole.h index bbe8c49db544425f4f35d57884c6036d3f83c905..4cbac0e7634982b7c25574126858ac049094b5a8 100644 --- a/src/multipole.h +++ b/src/multipole.h @@ -215,6 +215,7 @@ struct gravity_tensors { extern MPI_Datatype multipole_mpi_type; extern MPI_Op multipole_mpi_reduce_op; void multipole_create_mpi_types(void); +void multipole_free_mpi_types(void); #endif /** diff --git a/src/part.c b/src/part.c index d8f57a8b3db3d5f29667dff162fb03d8b117d3a3..cb73aba1afa958e7a5e65919af819687ba9df863 100644 --- a/src/part.c +++ b/src/part.c @@ -433,4 +433,13 @@ void part_create_mpi_types(void) { error("Failed to create MPI type for bparts."); } } + +void part_free_mpi_types(void) { + + MPI_Type_free(&part_mpi_type); + MPI_Type_free(&xpart_mpi_type); + MPI_Type_free(&gpart_mpi_type); + MPI_Type_free(&spart_mpi_type); + MPI_Type_free(&bpart_mpi_type); +} #endif diff --git a/src/part.h b/src/part.h index 440e5e1eb3415e65d7d9cc8b611bb98de107a1b2..eb8489daae5f408b40a85ec07561232c0a325496 100644 --- a/src/part.h +++ b/src/part.h @@ -152,6 +152,7 @@ extern MPI_Datatype spart_mpi_type; extern MPI_Datatype bpart_mpi_type; void part_create_mpi_types(void); +void part_free_mpi_types(void); #endif #endif /* SWIFT_PART_H */ diff --git a/src/periodic.h b/src/periodic.h index 05cb2fe454feadf8fe75942294443a726fc94d89..e3656efc614b983ecfd300d30984b51bf8f01054 100644 --- a/src/periodic.h +++ b/src/periodic.h @@ -28,15 +28,15 @@ /** * @brief Limits the value of x to be between a and b * - * Only wraps once. If x > 2b, the returned value will be larger than b. - * Similarly for x < -b. + * Only wraps once. If x > a + 2(b - a), the returned value will be larger than + * b. Similarly for x < a - (b - a), the value will be smaller than a. */ -#define box_wrap(x, a, b) \ - ({ \ - const __typeof__(x) _x = (x); \ - const __typeof__(a) _a = (a); \ - const __typeof__(b) _b = (b); \ - _x < _a ? (_x + _b) : ((_x >= _b) ? (_x - _b) : _x); \ +#define box_wrap(x, a, b) \ + ({ \ + const __typeof__(x) _x = (x); \ + const __typeof__(a) _a = (a); \ + const __typeof__(b) _b = (b); \ + _x < _a ? (_x + (_b - _a)) : ((_x >= _b) ? (_x - (_b - _a)) : _x); \ }) /** diff --git a/src/proxy.c b/src/proxy.c index 4e7e979a68c311ecdde7d36f214a6d5dcded4f5e..338f1beaa8734168d8139902a2746cd00250bffd 100644 --- a/src/proxy.c +++ b/src/proxy.c @@ -983,6 +983,29 @@ void proxy_init(struct proxy *p, int mynodeID, int nodeID) { p->nr_bparts_out = 0; } +/** + * @brief Free the memory allocated by a #proxy + */ +void proxy_clean(struct proxy *p) { + + free(p->cells_in); + free(p->cells_out); + free(p->cells_in_type); + free(p->cells_out_type); + swift_free("pcells_in", p->pcells_in); + swift_free("pcells_out", p->pcells_out); + swift_free("parts_out", p->parts_out); + swift_free("xparts_out", p->xparts_out); + swift_free("gparts_out", p->gparts_out); + swift_free("sparts_out", p->sparts_out); + swift_free("bparts_out", p->bparts_out); + swift_free("parts_in", p->parts_in); + swift_free("xparts_in", p->xparts_in); + swift_free("gparts_in", p->gparts_in); + swift_free("sparts_in", p->sparts_in); + swift_free("bparts_in", p->bparts_in); +} + /** * @brief Registers the MPI types for the proxy cells. */ @@ -998,3 +1021,11 @@ void proxy_create_mpi_type(void) { error("SWIFT was not compiled with MPI support."); #endif } + +void proxy_free_mpi_type(void) { +#ifdef WITH_MPI + MPI_Type_free(&pcell_mpi_type); +#else + error("SWIFT was not compiled with MPI support."); +#endif +} diff --git a/src/proxy.h b/src/proxy.h index c59c8ff84356188ab7935ab2151f1c8075045095..efe8021c9e7c2548bf1001b8072e5218a4dd4511 100644 --- a/src/proxy.h +++ b/src/proxy.h @@ -98,6 +98,7 @@ struct proxy { /* Function prototypes. */ void proxy_init(struct proxy *p, int mynodeID, int nodeID); +void proxy_clean(struct proxy *p); void proxy_parts_load(struct proxy *p, const struct part *parts, const struct xpart *xparts, int N); void proxy_gparts_load(struct proxy *p, const struct gpart *gparts, int N); @@ -112,5 +113,6 @@ void proxy_cells_exchange(struct proxy *proxies, int num_proxies, void proxy_tags_exchange(struct proxy *proxies, int num_proxies, struct space *s); void proxy_create_mpi_type(void); +void proxy_free_mpi_type(void); #endif /* SWIFT_PROXY_H */ diff --git a/src/scheduler.c b/src/scheduler.c index a776d8e4421dbbed4e0240b34243a573854c9a26..9d8072da3d48171d1073b63045095a57f9e9cbbe 100644 --- a/src/scheduler.c +++ b/src/scheduler.c @@ -522,6 +522,10 @@ void scheduler_write_dependencies(struct scheduler *s, int verbose) { /* Be clean */ free(task_dep); +#ifdef WITH_MPI + MPI_Type_free(&data_type); + MPI_Op_free(&sum); +#endif if (verbose) message("Printing task graph took %.3f %s.", diff --git a/src/space.c b/src/space.c index d8986df02e0e67d4a988f5b06bb684424d0621fc..27fe2a30b501222eb605634fdbdfbb8bf9d2a632 100644 --- a/src/space.c +++ b/src/space.c @@ -5468,6 +5468,12 @@ void space_clean(struct space *s) { swift_free("gparts", s->gparts); swift_free("sparts", s->sparts); swift_free("bparts", s->bparts); +#ifdef WITH_MPI + swift_free("parts_foreign", s->parts_foreign); + swift_free("sparts_foreign", s->sparts_foreign); + swift_free("gparts_foreign", s->gparts_foreign); + swift_free("bparts_foreign", s->bparts_foreign); +#endif } /** diff --git a/src/statistics.c b/src/statistics.c index 8866f345a4d0ccad8d3a50f30f6b07ff7787dbbd..11b64acecb79d9466176ef91e4b9cd70e2ebb743 100644 --- a/src/statistics.c +++ b/src/statistics.c @@ -414,4 +414,9 @@ void stats_create_mpi_type(void) { /* Create the reduction operation */ MPI_Op_create(stats_add_mpi, 1, &statistics_mpi_reduce_op); } + +void stats_free_mpi_type(void) { + MPI_Type_free(&statistics_mpi_type); + MPI_Op_free(&statistics_mpi_reduce_op); +} #endif diff --git a/src/statistics.h b/src/statistics.h index b741eac3d406d767f5652234b9a16d82464cc456..63254c21c54e6d8e5790f29f3973876b338a140f 100644 --- a/src/statistics.h +++ b/src/statistics.h @@ -77,6 +77,7 @@ extern MPI_Datatype statistics_mpi_type; extern MPI_Op statistics_mpi_reduce_op; void stats_create_mpi_type(void); +void stats_free_mpi_type(void); #endif #endif /* SWIFT_STATISTICS_H */ diff --git a/src/task.c b/src/task.c index 1a091599e389283836a3e9d418baf56e2d602223..cc7f459f26cd5b6b1b85e28162477cc31ad56174 100644 --- a/src/task.c +++ b/src/task.c @@ -883,6 +883,14 @@ void task_create_mpi_comms(void) { MPI_Comm_dup(MPI_COMM_WORLD, &subtaskMPI_comms[i]); } } +/** + * @brief Create global communicators for each of the subtasks. + */ +void task_free_mpi_comms(void) { + for (int i = 0; i < task_subtype_count; i++) { + MPI_Comm_free(&subtaskMPI_comms[i]); + } +} #endif /** diff --git a/src/task.h b/src/task.h index d033984c0e80814bc686be75c23d044a42f895f9..264f73f497eb90fe0481a636c7eb47450679c4ac 100644 --- a/src/task.h +++ b/src/task.h @@ -245,5 +245,6 @@ void task_get_group_name(int type, int subtype, char *cluster); #ifdef WITH_MPI void task_create_mpi_comms(void); +void task_free_mpi_comms(void); #endif #endif /* SWIFT_TASK_H */