Add new repartition methods based on different weight strategies.

Move more code into partition.c and remove param.h.

examples/main.c

@@ -84,14 +84,20 @@ int main(int argc, char *argv[]) {
   int nr_nodes = 1, myrank = 0;
   FILE *file_thread;
   int with_outputs = 1;
-  struct pgrid pgrid;
-
-  /* Default parition type is grid. */
-  pgrid.type = GRID_GRID;
-  pgrid.grid[0] = 1;
-  pgrid.grid[1] = 1;
-  pgrid.grid[2] = 1;
+  struct initpart ipart;
+#if defined(WITH_MPI) && defined(HAVE_METIS)
+  enum repart_type reparttype = REPART_METIS_BOTH;
+#endif
 
+  /* Default initial partition type is grid for shared memory and when
+   * METIS is not available. */
+  ipart.type = INITPART_GRID;
+  ipart.grid[0] = 1;
+  ipart.grid[1] = 1;
+  ipart.grid[2] = 1;
+#if defined(WITH_MPI) && defined(HAVE_METIS)
+  ipart.type = INITPART_METIS_NOWEIGHT;
+#endif
 
 /* Choke on FP-exceptions. */
 // feenableexcept( FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW );
@@ -115,9 +121,9 @@ int main(int argc, char *argv[]) {
   fflush(stdout);
 
   /* Set a default grid so that grid[0]*grid[1]*grid[2] == nr_nodes. */
-  factor(nr_nodes, &pgrid.grid[0], &pgrid.grid[1]);
-  factor(nr_nodes / pgrid.grid[1], &pgrid.grid[0], &pgrid.grid[2]);
-  factor(pgrid.grid[0] * pgrid.grid[1], &pgrid.grid[1], &pgrid.grid[0]);
+  factor(nr_nodes, &ipart.grid[0], &ipart.grid[1]);
+  factor(nr_nodes / ipart.grid[1], &ipart.grid[0], &ipart.grid[2]);
+  factor(ipart.grid[0] * ipart.grid[1], &ipart.grid[1], &ipart.grid[0]);
 #endif
 
   /* Greeting message */
@@ -127,7 +133,7 @@ int main(int argc, char *argv[]) {
   bzero(&s, sizeof(struct space));
 
   /* Parse the options */
-  while ((c = getopt(argc, argv, "a:c:d:f:g:m:q:r:s:t:w:yz:")) != -1)
+  while ((c = getopt(argc, argv, "a:c:d:e:f:m:p:q:r:s:t:w:yz:")) != -1)
     switch (c) {
       case 'a':
         if (sscanf(optarg, "%lf", &scaling) != 1)
@@ -146,32 +152,48 @@ int main(int argc, char *argv[]) {
         if (myrank == 0) message("dt set to %e.", dt_max);
         fflush(stdout);
         break;
+      case 'e':
+        /* REpartition type "b", "v", "e". */
+#if defined(WITH_MPI) && defined(HAVE_METIS)
+        switch (optarg[0]) {
+          case 'b':
+            reparttype = REPART_METIS_BOTH;
+            break;
+          case 'v':
+            reparttype = REPART_METIS_VERTEX;
+            break;
+          case 'e':
+            reparttype = REPART_METIS_EDGE;
+            break;
+        }
+#endif
+        break;
       case 'f':
         if (!strcpy(ICfileName, optarg)) error("Error parsing IC file name.");
         break;
-      case 'g':
-        /* Grid is one of "g", "r", "m", "w", or "v". g can be followed by three
-         * numbers. */
+      case 'p':
+        /* Parition type is one of "g", "r", "m", "w", or "v"; "g" can be
+         * followed by three numbers defining the grid. */
         switch (optarg[0]) {
           case 'g':
-            pgrid.type = GRID_GRID;
+            ipart.type = INITPART_GRID;
             if (strlen(optarg) > 2) {
-              if (sscanf(optarg, "g %i %i %i", &pgrid.grid[0], &pgrid.grid[1],
-                         &pgrid.grid[2]) != 3) 
+              if (sscanf(optarg, "g %i %i %i", &ipart.grid[0], &ipart.grid[1],
+                         &ipart.grid[2]) != 3) 
                 error("Error parsing grid.");
             }
             break;
           case 'r':
-            pgrid.type = GRID_RANDOM;
+            ipart.type = INITPART_RANDOM;
             break;
           case 'm':
-            pgrid.type = GRID_METIS_NOWEIGHT;
+            ipart.type = INITPART_METIS_NOWEIGHT;
             break;
           case 'w':
-            pgrid.type = GRID_METIS_WEIGHT;
+            ipart.type = INITPART_METIS_WEIGHT;
             break;
           case 'v':
-            pgrid.type = GRID_VECTORIZE;
+            ipart.type = INITPART_VECTORIZE;
             break;
         }
         break;
@@ -223,7 +245,8 @@ int main(int argc, char *argv[]) {
 #if defined(WITH_MPI)
   if (myrank == 0) {
     message("Running with %i thread(s) per node.", nr_threads);
-    message("grid set to [ %i %i %i ].", pgrid.grid[0], pgrid.grid[1], pgrid.grid[2]);
+    
+    message("grid set to [ %i %i %i ].", ipart.grid[0], ipart.grid[1], ipart.grid[2]);
 
     if (nr_nodes == 1) {
       message("WARNING: you are running with one MPI rank.");
@@ -356,7 +379,7 @@ int main(int argc, char *argv[]) {
 
 #ifdef WITH_MPI
   /* Split the space. */
-  engine_split(&e, &pgrid);
+  engine_split(&e, &ipart);
   engine_redistribute(&e);
 #endif
 
@@ -410,8 +433,8 @@ int main(int argc, char *argv[]) {
 
 /* Repartition the space amongst the nodes? */
 #if defined(WITH_MPI) && defined(HAVE_METIS)
-    //if (j % 100 == 2) e.forcerepart = 1;
-    if (j % 10 == 9) e.forcerepart = 1;
+    //if (j % 100 == 2) e.forcerepart = reparttype;
+    if (j % 10 == 9) e.forcerepart = reparttype;
 #endif
 
     timers_reset(timers_mask_all);

src/engine.h

@@ -29,7 +29,7 @@
 #include "scheduler.h"
 #include "space.h"
 #include "task.h"
-#include "param.h"
+#include "partition.h"
 
 /* Some constants. */
 #define engine_policy_none 0
@@ -118,7 +118,8 @@ struct engine {
   ticks tic_step;
 
   /* Force the engine to rebuild? */
-  int forcerebuild, forcerepart;
+  int forcerebuild;
+  enum repart_type forcerepart;
 
   /* How many steps have we done with the same set of tasks? */
   int tasks_age;
@@ -136,7 +137,7 @@ void engine_launch(struct engine *e, int nr_runners, unsigned int mask);
 void engine_prepare(struct engine *e);
 void engine_step(struct engine *e);
 void engine_maketasks(struct engine *e);
-void engine_split(struct engine *e, struct pgrid *grid);
+void engine_split(struct engine *e, struct initpart *ipart);
 int engine_exchange_strays(struct engine *e, int offset, int *ind, int N);
 void engine_rebuild(struct engine *e);
 void engine_repartition(struct engine *e);

src/param.h

-/*******************************************************************************
- * This file is part of SWIFT.
- * Copyright (C) 2015 Peter W. Draper (p.w.draper@durham.ac.uk)
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Lesser General Public License as published
- * by the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- ******************************************************************************/
-#ifndef SWIFT_PARAM_H
-#define SWIFT_PARAM_H
-
-
-/* Initial partition grid struct. Defines type of partitioning to use and any
- * related parameters. */
-enum grid_types {
-  GRID_GRID = 0,
-  GRID_RANDOM,
-  GRID_VECTORIZE,
-  GRID_METIS_WEIGHT,
-  GRID_METIS_NOWEIGHT
-};
-
-struct pgrid {
-  enum grid_types type;
-  int grid[3];
-};
-
-
-#endif /* SWIFT_PARAM_H */

src/partition.c

@@ -47,15 +47,29 @@
 #include "partition.h"
 #include "const.h"
 #include "error.h"
-#include "param.h"
 #include "debug.h"
 
-
 /* Useful defines. */
 #define MAX(a, b) ((a) > (b) ? (a) : (b))
 #define MIN(a, b) ((a) > (b) ? (b) : (a))
 #define CHUNK 512
 
+/* Simple descriptions of initial partition types for reports. */
+const char *initpart_name[] = {
+  "gridded cells",
+  "random point associated cells",
+  "vectorized point associated cells",
+  "METIS particle weighted cells",
+  "METIS unweighted cells"
+};
+
+/* Simple descriptions of repartition types for reports. */
+const char *repart_name[] = {
+  "METIS edge and vertex weighted cells",
+  "METIS vertex weighted cells",
+  "METIS edge weights"
+};
+
 /*  Vectorisation support */
 /*  ===================== */
 
@@ -744,12 +758,16 @@ int main(int argc, char *argv[]) {
  *
  * @param s the space of cells to partition.
  * @param nregions the number of regions required in the partition.
-* @param weights for the cells, sizeof number of cells if used, NULL 
- *        for unit weights.
- * @param celllist on exit this contains the ids of the select region,
+ * @param vertexw weights for the cells, sizeof number of cells if used, 
+ *        NULL for unit weights.
+ * @param edgew weights for the graph edges between all cells, sizeof number
+ *        of cells * 26 if used, NULL for unit weights. Need to be packed
+ *        in CSR format, so same as adjcny array.
+ * @param celllist on exit this contains the ids of the selected regions,
  *        sizeof number of cells.
  */
-void part_pick_metis(struct space *s, int nregions, int *weights, int *celllist) {
+void part_pick_metis(struct space *s, int nregions, int *vertexw,
+                     int *edgew, int *celllist) {
 
 #if defined(HAVE_METIS)
 
@@ -770,9 +788,14 @@ void part_pick_metis(struct space *s, int nregions, int *weights, int *celllist)
   idx_t *adjncy;
   if ((adjncy = (idx_t *)malloc(sizeof(idx_t) * 26 * ncells)) == NULL)
     error("Failed to allocate adjncy array.");
-  idx_t *weights_v;
-  if ((weights_v = (idx_t *)malloc(sizeof(idx_t) * ncells)) == NULL)
-    error("Failed to allocate weights array");
+  idx_t *weights_v = NULL;
+  if (vertexw != NULL)
+    if ((weights_v = (idx_t *)malloc(sizeof(idx_t) * ncells)) == NULL)
+      error("Failed to allocate vertex weights array");
+  idx_t *weights_e = NULL;
+  if (edgew != NULL)
+    if ((weights_e = (idx_t *)malloc( 26 *sizeof(idx_t) * ncells)) == NULL)
+      error("Failed to allocate edge weights array");
   idx_t *regionid;
   if ((regionid = (idx_t *)malloc(sizeof(idx_t) * ncells)) == NULL)
     error("Failed to allocate regionid array");
@@ -823,17 +846,25 @@ void part_pick_metis(struct space *s, int nregions, int *weights, int *celllist)
   for (int k = 0; k < ncells; k++) xadj[k + 1] = xadj[k] + 26;
 
   /* Init the vertex weights array. */
-  if ( weights != NULL) {
+  if (vertexw != NULL) {
     for (int k = 0; k < ncells; k++) {
-      if ( weights[k] > 0 ) {
-        weights_v[k] = weights[k];
+      if ( vertexw[k] > 0 ) {
+        weights_v[k] = vertexw[k];
       } else {
         weights_v[k] = 1;
       }
     }
-  } else {
-    for (int k = 0; k < ncells; k++)
-      weights_v[k] = 1;
+  }
+
+  /* Init the edges weights array. */
+  if (edgew != NULL) {
+    for (int k = 0; k < 26 * ncells; k++) {
+      if (edgew[k] > 0 ) {
+        weights_e[k] = edgew[k];
+      } else {
+        weights_e[k] = 1;
+      }
+    }
   }
 
   /* Set the METIS options. */
@@ -853,10 +884,10 @@ void part_pick_metis(struct space *s, int nregions, int *weights, int *celllist)
 
   /* Dump graph in METIS format */
   dumpMETISGraph("metis_graph", idx_ncells, one, xadj, adjncy,
-                 weights_v, NULL, NULL);
+                 weights_v, weights_e, NULL);
 
   if (METIS_PartGraphKway(&idx_ncells, &one, xadj, adjncy, weights_v,
-                          NULL, NULL, &idx_nregions, NULL, NULL,
+                          weights_e, NULL, &idx_nregions, NULL, NULL,
                           options, &objval, regionid) != METIS_OK)
     error("Call to METIS_PartGraphKway failed.");
 
@@ -894,7 +925,7 @@ void part_pick_metis(struct space *s, int nregions, int *weights, int *celllist)
   for (int l = 0, k = 0; l < s->cdim[0]; l++) {
     for (int m = 0; m < s->cdim[1]; m++) {
       for (int n = 0; n < s->cdim[2]; n++) {
-        message("node %d %d %d -> %d %d", l, m, n, regionid[k], weights_v[k]);
+        message("node %d %d %d -> %d", l, m, n, regionid[k]);
         k++;
       }
     }

src/partition.h

@@ -22,13 +22,43 @@
 #include "space.h"
 #include "cell.h"
 
+/* Initial partitioning types. */
+enum initpart_type {
+  INITPART_GRID = 0,
+  INITPART_RANDOM,
+  INITPART_VECTORIZE,
+  INITPART_METIS_WEIGHT,
+  INITPART_METIS_NOWEIGHT
+};
+
+/* Simple descriptions of types for reports. */
+extern const char *initpart_name[];
+
+/* The selected initial partition type and any related metadata. */
+struct initpart {
+  enum initpart_type type;
+  int grid[3];
+};
+
+/* Repartition type to use. */
+enum repart_type {
+  REPART_NONE = 0,
+  REPART_METIS_BOTH,
+  REPART_METIS_VERTEX,
+  REPART_METIS_EDGE
+};
+
+/* Simple descriptions of types for reports. */
+extern const char *repart_name[];
+
 int part_pick_random(struct space *s, int nregions, float *samplelist);
 void part_split_random(struct space *s, int nregions, float *samplelist);
 
 void part_pick_vector(struct space *s, int nregions, int *samplecells);
 void part_split_vector(struct space *s, int nregions, int *samplecells);
 
-void part_pick_metis(struct space *s, int nregions, int *weight, int *celllist);
+void part_pick_metis(struct space *s, int nregions, int *vertexw, 
+                     int *edgew, int *celllist);
 void part_split_metis(struct space *s, int nregions, int *celllist);
 
-#endif /* SWIFT_POISSON_DISC_H */
+#endif /* SWIFT_PARTITION_H */

src/swift.h

@@ -46,7 +46,6 @@
 #include "timers.h"
 #include "units.h"
 #include "tools.h"
-#include "param.h"
 #include "partition.h"
 #include "version.h"