diff --git a/src/debug.c b/src/debug.c index 8bc24c96a301d1d4ff87a07a129b98d19fca7a3a..4c521397b12c555923f8cea8a4f0cdb4c4551b97 100644 --- a/src/debug.c +++ b/src/debug.c @@ -276,11 +276,13 @@ int checkCellhdxmax(const struct cell *c, int *depth) { * only. */ static void dumpCells_map(struct cell *c, void *data) { - uintptr_t *ldata = (uintptr_t *)data; + size_t *ldata = (size_t *)data; FILE *file = (FILE *)ldata[0]; struct engine *e = (struct engine *)ldata[1]; float ntasks = c->nr_tasks; - int pactive = (int)ldata[2]; + int active = (int)ldata[2]; + int mpiactive = (int)ldata[3]; + int pactive = (int)ldata[4]; #if SWIFT_DEBUG_CHECKS /* The c->nr_tasks field does not include all the tasks. So let's check this @@ -301,31 +303,44 @@ static void dumpCells_map(struct cell *c, void *data) { } #endif - /* Only locally active cells are dumped. */ + /* Only cells with particles are dumped. */ if (c->count > 0 || c->gcount > 0 || c->scount > 0) { - /* If requested we work out how many particles are active in this cell. */ - int pactcount = 0; - if (pactive) { - const struct part *parts = c->parts; - for (int k = 0; k < c->count; k++) - if (part_is_active(&parts[k], e)) pactcount++; - struct gpart *gparts = c->gparts; - for (int k = 0; k < c->gcount; k++) - if (gpart_is_active(&gparts[k], e)) pactcount++; - struct spart *sparts = c->sparts; - for (int k = 0; k < c->scount; k++) - if (spart_is_active(&sparts[k], e)) pactcount++; - } +/* In MPI mode we may only output cells with foreign partners. + * These define the edges of the partitions. */ #if WITH_MPI - int sendto = (c->send_xv != NULL); + if (mpiactive) + mpiactive = (c->send_xv != NULL); + else + mpiactive = 1; #else - int sendto = 0; + mpiactive = 1; #endif - /* Local cells that are active and are super cells and have MPI tasks. */ - if (c->nodeID == e->nodeID && cell_is_active(c, e) && (c->super == c) && - sendto) + /* Active cells, otherwise all. */ + if (active) + active = cell_is_active(c, e); + else + active = 1; + + /* So output local super cells that are active and have MPI tasks as + * requested. */ + if (c->nodeID == e->nodeID && (c->super == c) && active && mpiactive) { + + /* If requested we work out how many particles are active in this cell. */ + int pactcount = 0; + if (pactive) { + const struct part *parts = c->parts; + for (int k = 0; k < c->count; k++) + if (part_is_active(&parts[k], e)) pactcount++; + struct gpart *gparts = c->gparts; + for (int k = 0; k < c->gcount; k++) + if (gpart_is_active(&gparts[k], e)) pactcount++; + struct spart *sparts = c->sparts; + for (int k = 0; k < c->scount; k++) + if (spart_is_active(&sparts[k], e)) pactcount++; + } + fprintf(file, " %6.3f %6.3f %6.3f %6.3f %6.3f %6.3f %6d %6d %6d %6d %6d %6d " "%6.1f %20lld %6d %6d %6d %6d %6d\n", @@ -334,6 +349,7 @@ static void dumpCells_map(struct cell *c, void *data) { c->depth, ntasks, c->ti_end_min, get_time_bin(c->ti_end_min), (c->super == c), cell_is_active(c, e), c->nodeID, c->nodeID == e->nodeID); + } } } @@ -344,19 +360,21 @@ static void dumpCells_map(struct cell *c, void *data) { * * @param prefix base output filename, result is written to * %prefix%_%rank%_%step%.dat + * @param active just output active cells. + * @param mpiactive just output MPI active cells, i.e. those with foreign cells. * @param pactive also output a count of active particles. * @param s the space holding the cells to dump. * @param rank node ID of MPI rank, or 0 if not relevant. * @param step the current engine step, or some unique integer. */ -void dumpCells(const char *prefix, int pactive, struct space *s, int rank, - int step) { +void dumpCells(const char *prefix, int active, int mpiactive, int pactive, + struct space *s, int rank, int step) { FILE *file = NULL; /* Name of output file. */ char fname[200]; - sprintf(fname, "%s_%03d.dat", prefix, step); + sprintf(fname, "%s_%03d_%03d.dat", prefix, rank, step); file = fopen(fname, "w"); /* Header. */ @@ -367,10 +385,12 @@ void dumpCells(const char *prefix, int pactive, struct space *s, int rank, "actcount", "depth", "tasks", "ti_end_min", "timebin", "issuper", "active", "rank", "local"); - uintptr_t data[3]; + size_t data[5]; data[0] = (size_t)file; data[1] = (size_t)s->e; - data[2] = (size_t)pactive; + data[2] = (size_t)active; + data[3] = (size_t)mpiactive; + data[4] = (size_t)pactive; space_map_cells_pre(s, 1, dumpCells_map, &data); fclose(file); } diff --git a/src/debug.h b/src/debug.h index 5e5fb4819e990349b57db7fa6c0bfe9dacba9d21..2cff37fa6f6f03185dc769bb770fe3a98a424ce1 100644 --- a/src/debug.h +++ b/src/debug.h @@ -36,8 +36,8 @@ void printgParticle_single(struct gpart *gp); int checkSpacehmax(struct space *s); int checkCellhdxmax(const struct cell *c, int *depth); -void dumpCells(const char *prefix, int pactive, struct space *s, int rank, - int step); +void dumpCells(const char *prefix, int active, int mpiactive, int pactive, + struct space *s, int rank, int step); #ifdef HAVE_METIS #include "metis.h" diff --git a/src/engine.c b/src/engine.c index 869432417d848e68290a44ba4e63cc9b3dc75b9c..fdcf80abd69450ad2eabbcc05d90d248ca5efa7f 100644 --- a/src/engine.c +++ b/src/engine.c @@ -3750,6 +3750,9 @@ void engine_step(struct engine *e) { /* Print the number of active tasks ? */ if (e->verbose) engine_print_task_counts(e); +/* Dump local cells and active particle counts. */ +/* dumpCells("cells", 0, 0, 1, e->s, e->nodeID, e->step); */ + #ifdef SWIFT_DEBUG_CHECKS /* Check that we have the correct total mass in the top-level multipoles */ size_t num_gpart_mpole = 0;