diff --git a/src/debug.c b/src/debug.c
index 601f63d6e11bbbf95f62eaef1ec6ec7ec06d3ad9..616d111cbc8f88e52f890bad25063b6c1666313a 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -26,6 +26,7 @@
/* Some standard headers. */
#include <float.h>
#include <stdio.h>
+#include <unistd.h>
/* This object's header. */
#include "debug.h"
@@ -450,3 +451,69 @@ void dumpCellRanks(const char *prefix, struct cell *cells_top, int nr_cells) {
}
#endif /* HAVE_MPI */
+
+/**
+ * @brief parse the process /proc/self/statm file to get the process
+ * memory use (in KB). Top field in ().
+ *
+ * @param size total virtual memory (VIRT)
+ * @param resident resident non-swapped memory (RES)
+ * @param share shared (mmap'd) memory (SHR)
+ * @param trs text (exe) resident set (CODE)
+ * @param lrs library resident set
+ * @param drs data+stack resident set (DATA)
+ * @param dt dirty pages (nDRT)
+ */
+void getProcMemUse(long *size, long *resident, long *share, long *trs,
+ long *lrs, long *drs, long *dt) {
+
+ /* Open the file. */
+ FILE *file = fopen("/proc/self/statm", "r");
+ if (file != NULL) {
+ int nscan = fscanf(file, "%ld %ld %ld %ld %ld %ld %ld", size, resident,
+ share, trs, lrs, drs, dt);
+
+ if (nscan == 7) {
+ /* Convert pages into bytes. Usually 4096, but could be 512 on some
+ * systems so take care in conversion to KB. */
+ long sz = sysconf(_SC_PAGESIZE);
+ *size *= sz;
+ *resident *= sz;
+ *share *= sz;
+ *trs *= sz;
+ *lrs *= sz;
+ *drs *= sz;
+ *dt *= sz;
+
+ *size /= 1024;
+ *resident /= 1024;
+ *share /= 1024;
+ *trs /= 1024;
+ *lrs /= 1024;
+ *drs /= 1024;
+ *dt /= 1024;
+ } else {
+ error("Failed to read sufficient fields from /proc/self/statm");
+ }
+ fclose(file);
+ } else {
+ error("Failed to open /proc/self/statm");
+ }
+}
+
+/**
+ * @brief Print the current memory use of the process. A la "top".
+ */
+void printProcMemUse() {
+ long size;
+ long resident;
+ long share;
+ long trs;
+ long lrs;
+ long drs;
+ long dt;
+ getProcMemUse(&size, &resident, &share, &trs, &lrs, &drs, &dt);
+ printf("## VIRT = %ld , RES = %ld , SHR = %ld , CODE = %ld, DATA = %ld\n",
+ size, resident, share, trs, drs);
+ fflush(stdout);
+}
diff --git a/src/debug.h b/src/debug.h
index 7422a6f7f9815490966f08415e0312876ce0123f..7dca848b6bf4e44de5f40fa8e1c0849e8ee3d0b4 100644
--- a/src/debug.h
+++ b/src/debug.h
@@ -44,4 +44,7 @@ void dumpMETISGraph(const char *prefix, idx_t nvtxs, idx_t ncon, idx_t *xadj,
void dumpCellRanks(const char *prefix, struct cell *cells_top, int nr_cells);
#endif
+void getProcMemUse(long *size, long *resident, long *share, long *trs,
+ long *lrs, long *drs, long *dt);
+void printProcMemUse();
#endif /* SWIFT_DEBUG_H */
diff --git a/src/engine.c b/src/engine.c
index b5b2a96c0b7d1fe74c900f25efa3d61d23cefcd6..4c4c315f9c00d5c1e135da5ac8a8e7c82397986b 100644
--- a/src/engine.c
+++ b/src/engine.c
@@ -57,6 +57,7 @@
#include "error.h"
#include "gravity.h"
#include "hydro.h"
+#include "map.h"
#include "minmax.h"
#include "parallel_io.h"
#include "part.h"
@@ -898,6 +899,16 @@ void engine_repartition(struct engine *e) {
partition_repartition(e->reparttype, e->nodeID, e->nr_nodes, e->s,
e->sched.tasks, e->sched.nr_tasks);
+ /* Partitioning requires copies of the particles, so we need to reduce the
+ * memory in use to the minimum, we can free the sorting indices and the
+ * tasks as these will be regenerated at the next rebuild. */
+
+ /* Sorting indices. */
+ if (e->s->cells_top != NULL) space_free_cells(e->s);
+
+ /* Task arrays. */
+ scheduler_free_tasks(&e->sched);
+
/* Now comes the tricky part: Exchange particles between all nodes.
This is done in two steps, first allreducing a matrix of
how many particles go from where to where, then re-allocating
diff --git a/src/scheduler.c b/src/scheduler.c
index 813bf9ad9e9ffe63b6fac58da5171ed54bc4362e..4081cde0489b1b439ceb46fc9b4e191541f15bef 100644
--- a/src/scheduler.c
+++ b/src/scheduler.c
@@ -966,9 +966,7 @@ void scheduler_reset(struct scheduler *s, int size) {
if (size > s->size) {
/* Free existing task lists if necessary. */
- if (s->tasks != NULL) free(s->tasks);
- if (s->tasks_ind != NULL) free(s->tasks_ind);
- if (s->tid_active != NULL) free(s->tid_active);
+ scheduler_free_tasks(s);
/* Allocate the new lists. */
if (posix_memalign((void *)&s->tasks, task_align,
@@ -1644,11 +1642,29 @@ void scheduler_print_tasks(const struct scheduler *s, const char *fileName) {
*/
void scheduler_clean(struct scheduler *s) {
- free(s->tasks);
- free(s->tasks_ind);
+ scheduler_free_tasks(s);
free(s->unlocks);
free(s->unlock_ind);
- free(s->tid_active);
for (int i = 0; i < s->nr_queues; ++i) queue_clean(&s->queues[i]);
free(s->queues);
}
+
+/**
+ * @brief Free the task arrays allocated by this #scheduler.
+ */
+void scheduler_free_tasks(struct scheduler *s) {
+
+ if (s->tasks != NULL) {
+ free(s->tasks);
+ s->tasks = NULL;
+ }
+ if (s->tasks_ind != NULL) {
+ free(s->tasks_ind);
+ s->tasks_ind = NULL;
+ }
+ if (s->tid_active != NULL) {
+ free(s->tid_active);
+ s->tid_active = NULL;
+ }
+ s->size = 0;
+}
diff --git a/src/scheduler.h b/src/scheduler.h
index f38e5fb4d849842217756b7b93713a5e1375c9c5..cdb3a28f1a3482e934480b0c0ccd18f18b66d437 100644
--- a/src/scheduler.h
+++ b/src/scheduler.h
@@ -143,5 +143,6 @@ void scheduler_set_unlocks(struct scheduler *s);
void scheduler_dump_queue(struct scheduler *s);
void scheduler_print_tasks(const struct scheduler *s, const char *fileName);
void scheduler_clean(struct scheduler *s);
+void scheduler_free_tasks(struct scheduler *s);
#endif /* SWIFT_SCHEDULER_H */
diff --git a/src/space.c b/src/space.c
index fc7732f45350b37600198b057583baf5b761c28c..45fad5c8b25ce6099c2017c45efa9c0746ebd79f 100644
--- a/src/space.c
+++ b/src/space.c
@@ -253,6 +253,15 @@ void space_rebuild_recycle_mapper(void *map_data, int num_elements,
}
}
+/**
+ * @brief Free up any allocated cells.
+ */
+void space_free_cells(struct space *s) {
+ threadpool_map(&s->e->threadpool, space_rebuild_recycle_mapper,
+ s->cells_top, s->nr_cells, sizeof(struct cell), 0, s);
+ s->maxdepth = 0;
+}
+
/**
* @brief Re-build the top-level cell grid.
*
@@ -379,13 +388,15 @@ void space_regrid(struct space *s, int verbose) {
/* Free the old cells, if they were allocated. */
if (s->cells_top != NULL) {
- threadpool_map(&s->e->threadpool, space_rebuild_recycle_mapper,
- s->cells_top, s->nr_cells, sizeof(struct cell), 0, s);
+ space_free_cells(s);
free(s->cells_top);
free(s->multipoles_top);
- s->maxdepth = 0;
}
+ /* Also free the task arrays, these will be regenerated and we can use the
+ * memory while copying the particle arrays. */
+ if (s->e != NULL) scheduler_free_tasks(&s->e->sched);
+
/* Set the new cell dimensions only if smaller. */
for (int k = 0; k < 3; k++) {
s->cdim[k] = cdim[k];
@@ -492,9 +503,7 @@ void space_regrid(struct space *s, int verbose) {
else { /* Otherwise, just clean up the cells. */
/* Free the old cells, if they were allocated. */
- threadpool_map(&s->e->threadpool, space_rebuild_recycle_mapper,
- s->cells_top, s->nr_cells, sizeof(struct cell), 0, s);
- s->maxdepth = 0;
+ space_free_cells(s);
}
if (verbose)
diff --git a/src/space.h b/src/space.h
index 7fe8f27ecaf671738e03881d547d6cb29a61503a..dbbba714c2b3c9841905b2ba54e4f2d854b820a6 100644
--- a/src/space.h
+++ b/src/space.h
@@ -226,5 +226,6 @@ void space_check_timesteps(struct space *s);
void space_replicate(struct space *s, int replicate, int verbose);
void space_reset_task_counters(struct space *s);
void space_clean(struct space *s);
+void space_free_cells(struct space *s);
#endif /* SWIFT_SPACE_H */