/******************************************************************************* * This file is part of SWIFT. * Copyright (c) 2012 Pedro Gonnet (pedro.gonnet@durham.ac.uk) * Matthieu Schaller (schaller@strw.leidenuniv.nl) * 2015 Peter W. Draper (p.w.draper@durham.ac.uk) * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published * by the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see . * ******************************************************************************/ /* Config parameters. */ #include /* MPI headers. */ #ifdef WITH_MPI #include #endif /* This object's header. */ #include "runner.h" /* Local headers. */ #include "engine.h" #include "timers.h" /** * @brief Construct the cell properties from the received #part. * * @param r The runner thread. * @param c The cell. * @param clear_sorts Should we clear the sort flag and hence trigger a sort ? * @param timer Are we timing this ? */ void runner_do_recv_part(struct runner *r, struct cell *c, int clear_sorts, int timer) { #ifdef WITH_MPI struct part *restrict parts = c->hydro.parts; const size_t nr_parts = c->hydro.count; const integertime_t ti_current = r->e->ti_current; const timebin_t max_active_bin = r->e->max_active_bin; TIMER_TIC; integertime_t ti_hydro_end_min = max_nr_timesteps; timebin_t time_bin_min = num_time_bins; timebin_t time_bin_max = 0; float h_max = 0.f; float h_max_active = 0.f; #ifdef SWIFT_DEBUG_CHECKS if (c->nodeID == engine_rank) error("Updating a local cell!"); #endif if (clear_sorts == 2) { /* This is a call for the first RT recv task. Check whether * we need to clear the sorts now. In the case of a foreign * cell where no xv comms are done, but RT is active, we * need to force a sort after the gradient recv. */ clear_sorts = !cell_get_flag(c, cell_flag_skip_rt_sort); cell_clear_flag(c, cell_flag_skip_rt_sort); } else if (clear_sorts == -1) { /* When running with RT and symmetric MPI exchanges, the first RT * recv tasks (gradients) may not necessarily run, i.e. in cases * where a local cell is inactive and needs to be updated by an * active foreign cell. Should that occur, we need to make sure * that the skip_rt_sort cell flag has been properly cleaned up * first. */ cell_clear_flag(c, cell_flag_skip_rt_sort); /* clear_sorts == -1 is abused to mark the case where this * function is called for a recv_rt_transport task. We're not * actually sorting now, so don't clear the sorts. */ clear_sorts = 0; } /* Clear this cell's sorted mask. */ if (clear_sorts) c->hydro.sorted = 0; /* If this cell is a leaf, collect the particle data. */ if (!c->split) { /* Collect everything... */ for (size_t k = 0; k < nr_parts; k++) { if (parts[k].time_bin == time_bin_inhibited) continue; time_bin_min = min(time_bin_min, parts[k].time_bin); time_bin_max = max(time_bin_max, parts[k].time_bin); h_max = max(h_max, parts[k].h); parts[k].gpart = NULL; if (parts[k].time_bin <= max_active_bin) h_max_active = max(h_max_active, parts[k].h); } /* Convert into a time */ ti_hydro_end_min = get_integer_time_end(ti_current, time_bin_min); } /* Otherwise, recurse and collect. */ else { for (int k = 0; k < 8; k++) { if (c->progeny[k] != NULL && c->progeny[k]->hydro.count > 0) { runner_do_recv_part(r, c->progeny[k], clear_sorts, 0); ti_hydro_end_min = min(ti_hydro_end_min, c->progeny[k]->hydro.ti_end_min); h_max = max(h_max, c->progeny[k]->hydro.h_max); h_max_active = max(h_max_active, c->progeny[k]->hydro.h_max_active); } } } #ifdef SWIFT_DEBUG_CHECKS if (!(r->e->policy & engine_policy_timestep_sync) && !(r->e->policy & engine_policy_timestep_limiter) && ti_hydro_end_min < ti_current) error( "Received a cell at an incorrect time c->ti_end_min=%lld, " "e->ti_current=%lld.", ti_hydro_end_min, ti_current); #endif /* ... and store. */ // c->hydro.ti_end_min = ti_hydro_end_min; c->hydro.ti_old_part = ti_current; c->hydro.h_max = h_max; c->hydro.h_max_active = h_max_active; if (timer) TIMER_TOC(timer_dorecv_part); #else error("SWIFT was not compiled with MPI support."); #endif } /** * @brief Construct the cell properties from the received #gpart. * * @param r The runner thread. * @param c The cell. * @param timer Are we timing this ? */ void runner_do_recv_gpart(struct runner *r, struct cell *c, int timer) { #ifdef WITH_MPI const struct gpart *restrict gparts = c->grav.parts; const size_t nr_gparts = c->grav.count; const integertime_t ti_current = r->e->ti_current; TIMER_TIC; integertime_t ti_gravity_end_min = max_nr_timesteps; timebin_t time_bin_min = num_time_bins; timebin_t time_bin_max = 0; #ifdef SWIFT_DEBUG_CHECKS if (c->nodeID == engine_rank) error("Updating a local cell!"); #endif /* If this cell is a leaf, collect the particle data. */ if (!c->split) { /* Collect everything... */ for (size_t k = 0; k < nr_gparts; k++) { if (gparts[k].time_bin == time_bin_inhibited) continue; time_bin_min = min(time_bin_min, gparts[k].time_bin); time_bin_max = max(time_bin_max, gparts[k].time_bin); } /* Convert into a time */ ti_gravity_end_min = get_integer_time_end(ti_current, time_bin_min); } /* Otherwise, recurse and collect. */ else { for (int k = 0; k < 8; k++) { if (c->progeny[k] != NULL && c->progeny[k]->grav.count > 0) { runner_do_recv_gpart(r, c->progeny[k], 0); ti_gravity_end_min = min(ti_gravity_end_min, c->progeny[k]->grav.ti_end_min); } } } #ifdef SWIFT_DEBUG_CHECKS if (!(r->e->policy & engine_policy_timestep_sync) && !(r->e->policy & engine_policy_timestep_limiter) && ti_gravity_end_min < ti_current) error( "Received a cell at an incorrect time c->ti_end_min=%lld, " "e->ti_current=%lld.", ti_gravity_end_min, ti_current); #endif /* ... and store. */ // c->grav.ti_end_min = ti_gravity_end_min; c->grav.ti_old_part = ti_current; if (timer) TIMER_TOC(timer_dorecv_gpart); #else error("SWIFT was not compiled with MPI support."); #endif } /** * @brief Construct the cell properties from the received #spart. * * @param r The runner thread. * @param c The cell. * @param clear_sorts Should we clear the sort flag and hence trigger a sort ? * @param timer Are we timing this ? */ void runner_do_recv_spart(struct runner *r, struct cell *c, int clear_sorts, int timer) { #ifdef WITH_MPI struct spart *restrict sparts = c->stars.parts; const size_t nr_sparts = c->stars.count; const integertime_t ti_current = r->e->ti_current; const timebin_t max_active_bin = r->e->max_active_bin; TIMER_TIC; integertime_t ti_stars_end_min = max_nr_timesteps; timebin_t time_bin_min = num_time_bins; timebin_t time_bin_max = 0; float h_max = 0.f; float h_max_active = 0.f; #ifdef SWIFT_DEBUG_CHECKS if (c->nodeID == engine_rank) error("Updating a local cell!"); #endif /* Clear this cell's sorted mask. */ if (clear_sorts) c->stars.sorted = 0; /* If this cell is a leaf, collect the particle data. */ if (!c->split) { /* Collect everything... */ for (size_t k = 0; k < nr_sparts; k++) { #ifdef DEBUG_INTERACTIONS_STARS sparts[k].num_ngb_feedback = 0; #endif if (sparts[k].time_bin == time_bin_inhibited) continue; time_bin_min = min(time_bin_min, sparts[k].time_bin); time_bin_max = max(time_bin_max, sparts[k].time_bin); h_max = max(h_max, sparts[k].h); sparts[k].gpart = NULL; if (sparts[k].time_bin <= max_active_bin) h_max_active = max(h_max_active, sparts[k].h); } /* Convert into a time */ ti_stars_end_min = get_integer_time_end(ti_current, time_bin_min); } /* Otherwise, recurse and collect. */ else { for (int k = 0; k < 8; k++) { if (c->progeny[k] != NULL && c->progeny[k]->stars.count > 0) { runner_do_recv_spart(r, c->progeny[k], clear_sorts, 0); ti_stars_end_min = min(ti_stars_end_min, c->progeny[k]->stars.ti_end_min); h_max = max(h_max, c->progeny[k]->stars.h_max); h_max_active = max(h_max_active, c->progeny[k]->stars.h_max_active); } } } #ifdef SWIFT_DEBUG_CHECKS if (ti_stars_end_min < ti_current && !(r->e->policy & engine_policy_star_formation)) error( "Received a cell at an incorrect time c->ti_end_min=%lld, " "e->ti_current=%lld.", ti_stars_end_min, ti_current); #endif /* ... and store. */ // c->grav.ti_end_min = ti_gravity_end_min; c->stars.ti_old_part = ti_current; c->stars.h_max = h_max; c->stars.h_max_active = h_max_active; if (timer) TIMER_TOC(timer_dorecv_spart); #else error("SWIFT was not compiled with MPI support."); #endif } /** * @brief Construct the cell properties from the received #bpart. * * Note that we do not need to clear the sorts since we do not sort * the black holes. * * @param r The runner thread. * @param c The cell. * @param clear_sorts Should we clear the sort flag and hence trigger a sort ? * @param timer Are we timing this ? */ void runner_do_recv_bpart(struct runner *r, struct cell *c, int clear_sorts, int timer) { #ifdef WITH_MPI struct bpart *restrict bparts = c->black_holes.parts; const size_t nr_bparts = c->black_holes.count; const integertime_t ti_current = r->e->ti_current; const timebin_t max_active_bin = r->e->max_active_bin; TIMER_TIC; integertime_t ti_black_holes_end_min = max_nr_timesteps; timebin_t time_bin_min = num_time_bins; timebin_t time_bin_max = 0; float h_max = 0.f; float h_max_active = 0.f; #ifdef SWIFT_DEBUG_CHECKS if (c->nodeID == engine_rank) error("Updating a local cell!"); #endif /* If this cell is a leaf, collect the particle data. */ if (!c->split) { /* Collect everything... */ for (size_t k = 0; k < nr_bparts; k++) { #ifdef DEBUG_INTERACTIONS_BLACK_HOLES bparts[k].num_ngb_force = 0; #endif if (bparts[k].time_bin == time_bin_inhibited) continue; time_bin_min = min(time_bin_min, bparts[k].time_bin); time_bin_max = max(time_bin_max, bparts[k].time_bin); h_max = max(h_max, bparts[k].h); bparts[k].gpart = NULL; if (bparts[k].time_bin <= max_active_bin) h_max_active = max(h_max_active, bparts[k].h); } /* Convert into a time */ ti_black_holes_end_min = get_integer_time_end(ti_current, time_bin_min); } /* Otherwise, recurse and collect. */ else { for (int k = 0; k < 8; k++) { if (c->progeny[k] != NULL && c->progeny[k]->black_holes.count > 0) { runner_do_recv_bpart(r, c->progeny[k], clear_sorts, 0); ti_black_holes_end_min = min(ti_black_holes_end_min, c->progeny[k]->black_holes.ti_end_min); h_max = max(h_max, c->progeny[k]->black_holes.h_max); h_max_active = max(h_max_active, c->progeny[k]->black_holes.h_max_active); } } } #ifdef SWIFT_DEBUG_CHECKS if (ti_black_holes_end_min < ti_current) error( "Received a cell at an incorrect time c->ti_end_min=%lld, " "e->ti_current=%lld.", ti_black_holes_end_min, ti_current); #endif /* ... and store. */ // c->grav.ti_end_min = ti_gravity_end_min; c->black_holes.ti_old_part = ti_current; c->black_holes.h_max = h_max; c->black_holes.h_max_active = h_max_active; if (timer) TIMER_TOC(timer_dorecv_bpart); #else error("SWIFT was not compiled with MPI support."); #endif }