diff --git a/examples/main.c b/examples/main.c index dcc113ab6af6a06e7c20ac1aac7c2d3b715f7ef3..32ec730d34523c7df6a8ee15f09a027d22bde483 100644 --- a/examples/main.c +++ b/examples/main.c @@ -548,7 +548,7 @@ int main(int argc, char *argv[]) { /* Repartition the space amongst the nodes? */ #ifdef WITH_MPI - if (j % 100 == 2) e.forcerepart = reparttype; + if (j % 3 == 2) e.forcerepart = reparttype; #endif /* Reset timers */ diff --git a/src/engine.c b/src/engine.c index e989aefd5343eb7f377c6c1e8bc41f9875f2f42e..a0b86722feda8ee3b6950a3b344119f570c8c7d9 100644 --- a/src/engine.c +++ b/src/engine.c @@ -317,6 +317,22 @@ void engine_redistribute(struct engine *e) { MPI_COMM_WORLD) != MPI_SUCCESS) error("Failed to allreduce particle transfer counts."); + if (e->nodeID == 0) { + size_t total = 0; + size_t unmoved = 0; + for (int p = 0, r = 0; p < nr_nodes; p++) { + for (int s = 0; s < nr_nodes; s++) { + total += counts[r]; + if (p == s) + unmoved += counts[r]; + r++; + } + } + message("total = %ld, unmoved = %ld, fraction = %f", total, unmoved, + (double)unmoved/(double)total); + } + + /* Get all the g_counts from all the nodes. */ if (MPI_Allreduce(MPI_IN_PLACE, g_counts, nr_nodes * nr_nodes, MPI_INT, MPI_SUM, MPI_COMM_WORLD) != MPI_SUCCESS) diff --git a/src/partition.c b/src/partition.c index 89ba3f2835cb78e07ec2bc5cb04c3f8f71751563..26f1ea151e24b0297d68138ae3b6881fb67fa973 100644 --- a/src/partition.c +++ b/src/partition.c @@ -278,6 +278,18 @@ static void split_metis(struct space *s, int nregions, int *celllist) { #endif #if defined(WITH_MPI) && defined(HAVE_METIS) + +/* qsort support. */ +struct indexval { + int index; + int count; +}; +static int indexvalcmp(const void *p1, const void *p2) { + const struct indexval *iv1 = (const struct indexval *)p1; + const struct indexval *iv2 = (const struct indexval *)p2; + return iv2->count - iv1->count; +} + /** * @brief Partition the given space into a number of connected regions. * @@ -382,14 +394,86 @@ static void pick_metis(struct space *s, int nregions, int *vertexw, int *edgew, if (regionid[k] < 0 || regionid[k] >= nregions) error("Got bad nodeID %" PRIDX " for cell %i.", regionid[k], k); + /* We want a solution in which the current regions of the space are + * preserved when possible, to avoid unneccesary particle movement. + * So try mapping the current regions to the new regions and reassigning + * those with the greatest number of common cells... */ + int keymax = nregions + nregions * nregions; + struct indexval *ivs = malloc(sizeof(struct indexval) * keymax); + bzero(ivs, sizeof(struct indexval) * keymax); + for (int k = 0; k < ncells; k++) { + int index = regionid[k] + nregions * s->cells_top[k].nodeID; + ivs[index].count++; + ivs[index].index = index; + //message( "incr: %d %d %d %d %d", index, ivs[index].count, + // regionid[k], nregions, s->cells_top[k].nodeID); + } + + /* Need to keep indices..., not counts, just sort by counts. XXX */ + //message("unsorted"); + //for (int k = 0; k < keymax; k++) { + // message(" %d %d", ivs[k].index, ivs[k].count); + //} + qsort(ivs, keymax, sizeof(struct indexval), indexvalcmp); + //message("sorted"); + //for (int k = 0; k < keymax; k++) { + // message(" %d %d", ivs[k].index, ivs[k].count); + //} + + /* Go through the ivs using the largest counts first, these are the + * regions with the most cells in common, old partition to new. */ + int *oldmap = malloc(sizeof(int) * nregions); + int *newmap = malloc(sizeof(int) * nregions); + for (int k = 0; k < nregions; k++) { + oldmap[k] = -1; + newmap[k] = -1; + } + for (int k = 0; k < keymax; k++) { + + /* Stop when all regions with common cells have been considered. */ + if (ivs[k].count == 0) break; + + /* Store old and new IDs, if not already used. */ + int oldregion = ivs[k].index / nregions; + int newregion = ivs[k].index - oldregion * nregions; + if (newmap[newregion] == -1 && oldmap[oldregion] == -1) { + newmap[newregion] = oldregion; + oldmap[oldregion] = newregion; + //message("mapping: %d to %d", newregion, oldregion); + //message(" : %d %d %d", ivs[k].index, ivs[k].count, nregions); + } + } + + /* Need to handle any regions that did not get selected. */ + int spare = 0; + for (int k = 0; k < nregions; k++) { + if (newmap[k] == -1) { + //message("unmapped newmap[%d] = %d", k, newmap[k]); + for (int j = spare; j < nregions; j++) { + if (oldmap[j] == -1) { + //message("used oldmap[%d] = %d", j, j); + newmap[k] = j; + oldmap[j] = j; + spare = j; + break; + } + } + } + } + /* Set the cell list to the region index. */ for (int k = 0; k < ncells; k++) { - celllist[k] = regionid[k]; + //message("mapping: %d to %d", regionid[k], newmap[regionid[k]]); + celllist[k] = newmap[regionid[k]]; + //celllist[k] = regionid[k]; } /* Clean up. */ if (weights_v != NULL) free(weights_v); if (weights_e != NULL) free(weights_e); + free(ivs); + free(oldmap); + free(newmap); free(xadj); free(adjncy); free(regionid);