Commit c123d484 authored by Peter W. Draper's avatar Peter W. Draper
Browse files

Merge branch 'blueGene'

Conflicts:
	examples/test.c
parents b65bbfd3 1aa3c67a
...@@ -554,6 +554,7 @@ int main(int argc, char *argv[]) { ...@@ -554,6 +554,7 @@ int main(int argc, char *argv[]) {
int c, icount, j, k, N, periodic = 1; int c, icount, j, k, N, periodic = 1;
long long N_total = -1; long long N_total = -1;
int nr_threads = 1, nr_queues = -1, runs = INT_MAX; int nr_threads = 1, nr_queues = -1, runs = INT_MAX;
int dump_tasks = 0;
int data[2]; int data[2];
double dim[3] = {1.0, 1.0, 1.0}, shift[3] = {0.0, 0.0, 0.0}; double dim[3] = {1.0, 1.0, 1.0}, shift[3] = {0.0, 0.0, 0.0};
double h_max = -1.0, scaling = 1.0; double h_max = -1.0, scaling = 1.0;
...@@ -605,7 +606,7 @@ int main(int argc, char *argv[]) { ...@@ -605,7 +606,7 @@ int main(int argc, char *argv[]) {
bzero(&s, sizeof(struct space)); bzero(&s, sizeof(struct space));
/* Parse the options */ /* Parse the options */
while ((c = getopt(argc, argv, "a:c:d:f:g:m:oq:r:s:t:w:z:")) != -1) while ((c = getopt(argc, argv, "a:c:d:f:g:m:q:r:s:t:w:yz:")) != -1)
switch (c) { switch (c) {
case 'a': case 'a':
if (sscanf(optarg, "%lf", &scaling) != 1) if (sscanf(optarg, "%lf", &scaling) != 1)
...@@ -663,6 +664,9 @@ int main(int argc, char *argv[]) { ...@@ -663,6 +664,9 @@ int main(int argc, char *argv[]) {
error("Error parsing sub size."); error("Error parsing sub size.");
if (myrank == 0) message("sub size set to %i.", space_subsize); if (myrank == 0) message("sub size set to %i.", space_subsize);
break; break;
case 'y':
dump_tasks = 1;
break;
case 'z': case 'z':
if (sscanf(optarg, "%d", &space_splitsize) != 1) if (sscanf(optarg, "%d", &space_splitsize) != 1)
error("Error parsing split size."); error("Error parsing split size.");
...@@ -756,7 +760,7 @@ int main(int argc, char *argv[]) { ...@@ -756,7 +760,7 @@ int main(int argc, char *argv[]) {
/* Initialize the space with this data. */ /* Initialize the space with this data. */
tic = getticks(); tic = getticks();
space_init(&s, dim, parts, N, periodic, h_max); space_init(&s, dim, parts, N, periodic, h_max, myrank == 0);
if (myrank == 0) if (myrank == 0)
message("space_init took %.3f ms.", message("space_init took %.3f ms.",
((double)(getticks() - tic)) / CPU_TPS * 1000); ((double)(getticks() - tic)) / CPU_TPS * 1000);
...@@ -796,7 +800,7 @@ int main(int argc, char *argv[]) { ...@@ -796,7 +800,7 @@ int main(int argc, char *argv[]) {
/* Initialize the engine with this space. */ /* Initialize the engine with this space. */
tic = getticks(); tic = getticks();
message("nr_nodes is %i.", nr_nodes); if (myrank == 0) message("nr_nodes is %i.", nr_nodes);
engine_init(&e, &s, dt_max, nr_threads, nr_queues, nr_nodes, myrank, engine_init(&e, &s, dt_max, nr_threads, nr_queues, nr_nodes, myrank,
ENGINE_POLICY | engine_policy_steal); ENGINE_POLICY | engine_policy_steal);
if (myrank == 0) if (myrank == 0)
...@@ -825,8 +829,9 @@ int main(int argc, char *argv[]) { ...@@ -825,8 +829,9 @@ int main(int argc, char *argv[]) {
#else #else
write_output_single(&e, &us); write_output_single(&e, &us);
#endif #endif
message("writing particle properties took %.3f ms.", if (myrank == 0)
((double)(getticks() - tic)) / CPU_TPS * 1000); message("writing particle properties took %.3f ms.",
((double)(getticks() - tic)) / CPU_TPS * 1000);
fflush(stdout); fflush(stdout);
} }
...@@ -916,40 +921,43 @@ int main(int argc, char *argv[]) { ...@@ -916,40 +921,43 @@ int main(int argc, char *argv[]) {
(double)runner_hist_bins[k]); (double)runner_hist_bins[k]);
#endif #endif
/* Dump the task data. */ /* Dump the task data. */
if (dump_tasks) {
#ifdef WITH_MPI #ifdef WITH_MPI
file_thread = fopen("thread_info_MPI.dat", "w"); file_thread = fopen("thread_info_MPI.dat", "w");
for (j = 0; j < nr_nodes; j++) { for (j = 0; j < nr_nodes; j++) {
MPI_Barrier(MPI_COMM_WORLD); MPI_Barrier(MPI_COMM_WORLD);
if (j == myrank) { if (j == myrank) {
fprintf(file_thread, " %03i 0 0 0 0 %lli 0 0 0 0\n", myrank, e.tic_step); fprintf(file_thread, " %03i 0 0 0 0 %lli 0 0 0 0\n", myrank,
for (k = 0; k < e.sched.nr_tasks; k++) e.tic_step);
if (!e.sched.tasks[k].skip && !e.sched.tasks[k].implicit) for (k = 0; k < e.sched.nr_tasks; k++)
fprintf( if (!e.sched.tasks[k].skip && !e.sched.tasks[k].implicit)
file_thread, " %03i %i %i %i %i %lli %lli %i %i %i\n", myrank, fprintf(
e.sched.tasks[k].rid, e.sched.tasks[k].type, file_thread, " %03i %i %i %i %i %lli %lli %i %i %i\n", myrank,
e.sched.tasks[k].subtype, (e.sched.tasks[k].cj == NULL), e.sched.tasks[k].rid, e.sched.tasks[k].type,
e.sched.tasks[k].tic, e.sched.tasks[k].toc, e.sched.tasks[k].subtype, (e.sched.tasks[k].cj == NULL),
e.sched.tasks[k].ci->count, e.sched.tasks[k].tic, e.sched.tasks[k].toc,
(e.sched.tasks[k].cj != NULL) ? e.sched.tasks[k].cj->count : 0, e.sched.tasks[k].ci->count,
e.sched.tasks[k].flags); (e.sched.tasks[k].cj != NULL) ? e.sched.tasks[k].cj->count : 0,
fflush(stdout); e.sched.tasks[k].flags);
sleep(1); fflush(stdout);
sleep(1);
}
} }
} fclose(file_thread);
fclose(file_thread);
#else #else
file_thread = fopen("thread_info.dat", "w"); file_thread = fopen("thread_info.dat", "w");
for (k = 0; k < e.sched.nr_tasks; k++) for (k = 0; k < e.sched.nr_tasks; k++)
if (!e.sched.tasks[k].skip && !e.sched.tasks[k].implicit) if (!e.sched.tasks[k].skip && !e.sched.tasks[k].implicit)
fprintf(file_thread, " %i %i %i %i %lli %lli %i %i\n", fprintf(file_thread, " %i %i %i %i %lli %lli %i %i\n",
e.sched.tasks[k].rid, e.sched.tasks[k].type, e.sched.tasks[k].rid, e.sched.tasks[k].type,
e.sched.tasks[k].subtype, (e.sched.tasks[k].cj == NULL), e.sched.tasks[k].subtype, (e.sched.tasks[k].cj == NULL),
e.sched.tasks[k].tic, e.sched.tasks[k].toc, e.sched.tasks[k].tic, e.sched.tasks[k].toc,
e.sched.tasks[k].ci->count, e.sched.tasks[k].ci->count,
(e.sched.tasks[k].cj == NULL) ? 0 : e.sched.tasks[k].cj->count); (e.sched.tasks[k].cj == NULL) ? 0 : e.sched.tasks[k].cj->count);
fclose(file_thread); fclose(file_thread);
#endif #endif
}
if (with_outputs) { if (with_outputs) {
/* Write final output. */ /* Write final output. */
...@@ -972,7 +980,7 @@ int main(int argc, char *argv[]) { ...@@ -972,7 +980,7 @@ int main(int argc, char *argv[]) {
#endif #endif
/* Say goodbye. */ /* Say goodbye. */
message("done."); if (myrank == 0) message("done.");
/* All is calm, all is bright. */ /* All is calm, all is bright. */
return 0; return 0;
......
...@@ -1370,7 +1370,7 @@ void engine_rebuild(struct engine *e) { ...@@ -1370,7 +1370,7 @@ void engine_rebuild(struct engine *e) {
/* Re-build the space. */ /* Re-build the space. */
// tic = getticks(); // tic = getticks();
space_rebuild(e->s, 0.0); space_rebuild(e->s, 0.0, e->nodeID == 0);
// message( "space_rebuild took %.3f ms." , (double)(getticks() - // message( "space_rebuild took %.3f ms." , (double)(getticks() -
// tic)/CPU_TPS*1000 ); // tic)/CPU_TPS*1000 );
...@@ -2041,8 +2041,9 @@ void engine_split(struct engine *e, int *grid) { ...@@ -2041,8 +2041,9 @@ void engine_split(struct engine *e, int *grid) {
engine_makeproxies(e); engine_makeproxies(e);
/* Re-allocate the local parts. */ /* Re-allocate the local parts. */
message("Re-allocating parts array from %i to %i.", s->size_parts, if (e->nodeID == 0)
(int)(s->nr_parts * 1.2)); message("Re-allocating parts array from %i to %i.", s->size_parts,
(int)(s->nr_parts * 1.2));
s->size_parts = s->nr_parts * 1.2; s->size_parts = s->nr_parts * 1.2;
struct part *parts_new; struct part *parts_new;
struct xpart *xparts_new; struct xpart *xparts_new;
...@@ -2094,13 +2095,16 @@ void engine_init(struct engine *e, struct space *s, float dt, int nr_threads, ...@@ -2094,13 +2095,16 @@ void engine_init(struct engine *e, struct space *s, float dt, int nr_threads,
for (i = 1; i < maxint; i *= 2) for (i = 1; i < maxint; i *= 2)
for (j = maxint / i / 2; j < maxint; j += maxint / i) for (j = maxint / i / 2; j < maxint; j += maxint / i)
if (j < nr_cores && j != 0) cpuid[k++] = j; if (j < nr_cores && j != 0) cpuid[k++] = j;
#ifdef WITHMPI
printf("engine_init: cpu map is [ "); if (nodeID == 0) {
#ifdef WITH_MPI
message("engine_init: cpu map is [ ");
#else #else
printf("[%03i] engine_init: cpu map is [ ", nodeID); printf("[%03i] engine_init: cpu map is [ ", nodeID);
#endif #endif
for (i = 0; i < nr_cores; i++) printf("%i ", cpuid[i]); for (i = 0; i < nr_cores; i++) printf("%i ", cpuid[i]);
printf("].\n"); printf("].\n");
}
} }
#endif #endif
......
...@@ -149,9 +149,10 @@ void space_rebuild_recycle(struct space *s, struct cell *c) { ...@@ -149,9 +149,10 @@ void space_rebuild_recycle(struct space *s, struct cell *c) {
* *
* @param s The #space. * @param s The #space.
* @param cell_max Maximum cell edge length. * @param cell_max Maximum cell edge length.
* @param verbose Print messages to stdout or not.
*/ */
void space_regrid(struct space *s, double cell_max) { void space_regrid(struct space *s, double cell_max, int verbose) {
float h_max = s->cell_min / kernel_gamma / space_stretch, dmin; float h_max = s->cell_min / kernel_gamma / space_stretch, dmin;
int i, j, k, cdim[3], nr_parts = s->nr_parts; int i, j, k, cdim[3], nr_parts = s->nr_parts;
...@@ -182,7 +183,7 @@ void space_regrid(struct space *s, double cell_max) { ...@@ -182,7 +183,7 @@ void space_regrid(struct space *s, double cell_max) {
h_max = buff; h_max = buff;
} }
#endif #endif
message("h_max is %.3e (cell_max=%.3e).", h_max, cell_max); if (verbose) message("h_max is %.3e (cell_max=%.3e).", h_max, cell_max);
/* Get the new putative cell dimensions. */ /* Get the new putative cell dimensions. */
for (k = 0; k < 3; k++) for (k = 0; k < 3; k++)
...@@ -253,7 +254,9 @@ void space_regrid(struct space *s, double cell_max) { ...@@ -253,7 +254,9 @@ void space_regrid(struct space *s, double cell_max) {
} }
/* Be verbose about the change. */ /* Be verbose about the change. */
message("set cell dimensions to [ %i %i %i ].", cdim[0], cdim[1], cdim[2]); if (verbose)
message("set cell dimensions to [ %i %i %i ].", cdim[0], cdim[1],
cdim[2]);
fflush(stdout); fflush(stdout);
} /* re-build upper-level cells? */ } /* re-build upper-level cells? */
...@@ -289,10 +292,11 @@ void space_regrid(struct space *s, double cell_max) { ...@@ -289,10 +292,11 @@ void space_regrid(struct space *s, double cell_max) {
* *
* @param s The #space in which to update the cells. * @param s The #space in which to update the cells.
* @param cell_max Maximal cell size. * @param cell_max Maximal cell size.
* @param verbose Print messages to stdout or not
* *
*/ */
void space_rebuild(struct space *s, double cell_max) { void space_rebuild(struct space *s, double cell_max, int verbose) {
int j, k, cdim[3], nr_parts = s->nr_parts, nr_gparts = s->nr_gparts; int j, k, cdim[3], nr_parts = s->nr_parts, nr_gparts = s->nr_gparts;
struct cell *restrict c, *restrict cells; struct cell *restrict c, *restrict cells;
...@@ -305,7 +309,7 @@ void space_rebuild(struct space *s, double cell_max) { ...@@ -305,7 +309,7 @@ void space_rebuild(struct space *s, double cell_max) {
// message( "re)building space..." ); fflush(stdout); // message( "re)building space..." ); fflush(stdout);
/* Re-grid if necessary, or just re-set the cell data. */ /* Re-grid if necessary, or just re-set the cell data. */
space_regrid(s, cell_max); space_regrid(s, cell_max, verbose);
cells = s->cells; cells = s->cells;
/* Run through the particles and get their cell index. */ /* Run through the particles and get their cell index. */
...@@ -1073,6 +1077,7 @@ struct cell *space_getcell(struct space *s) { ...@@ -1073,6 +1077,7 @@ struct cell *space_getcell(struct space *s) {
* @param N The number of parts in the space. * @param N The number of parts in the space.
* @param periodic flag whether the domain is periodic or not. * @param periodic flag whether the domain is periodic or not.
* @param h_max The maximal interaction radius. * @param h_max The maximal interaction radius.
* @param verbose Print messages to stdout or not
* *
* Makes a grid of edge length > r_max and fills the particles * Makes a grid of edge length > r_max and fills the particles
* into the respective cells. Cells containing more than #space_splitsize * into the respective cells. Cells containing more than #space_splitsize
...@@ -1081,7 +1086,7 @@ struct cell *space_getcell(struct space *s) { ...@@ -1081,7 +1086,7 @@ struct cell *space_getcell(struct space *s) {
*/ */
void space_init(struct space *s, double dim[3], struct part *parts, int N, void space_init(struct space *s, double dim[3], struct part *parts, int N,
int periodic, double h_max) { int periodic, double h_max, int verbose) {
/* Store eveything in the space. */ /* Store eveything in the space. */
s->dim[0] = dim[0]; s->dim[0] = dim[0];
...@@ -1151,5 +1156,5 @@ void space_init(struct space *s, double dim[3], struct part *parts, int N, ...@@ -1151,5 +1156,5 @@ void space_init(struct space *s, double dim[3], struct part *parts, int N,
if (lock_init(&s->lock) != 0) error("Failed to create space spin-lock."); if (lock_init(&s->lock) != 0) error("Failed to create space spin-lock.");
/* Build the cells and the tasks. */ /* Build the cells and the tasks. */
space_regrid(s, h_max); space_regrid(s, h_max, verbose);
} }
...@@ -119,7 +119,7 @@ struct cell *space_getcell(struct space *s); ...@@ -119,7 +119,7 @@ struct cell *space_getcell(struct space *s);
int space_getsid(struct space *s, struct cell **ci, struct cell **cj, int space_getsid(struct space *s, struct cell **ci, struct cell **cj,
double *shift); double *shift);
void space_init(struct space *s, double dim[3], struct part *parts, int N, void space_init(struct space *s, double dim[3], struct part *parts, int N,
int periodic, double h_max); int periodic, double h_max, int verbose);
void space_map_cells_pre(struct space *s, int full, void space_map_cells_pre(struct space *s, int full,
void (*fun)(struct cell *c, void *data), void *data); void (*fun)(struct cell *c, void *data), void *data);
void space_map_parts(struct space *s, void space_map_parts(struct space *s,
...@@ -127,7 +127,7 @@ void space_map_parts(struct space *s, ...@@ -127,7 +127,7 @@ void space_map_parts(struct space *s,
void *data); void *data);
void space_map_cells_post(struct space *s, int full, void space_map_cells_post(struct space *s, int full,
void (*fun)(struct cell *c, void *data), void *data); void (*fun)(struct cell *c, void *data), void *data);
void space_rebuild(struct space *s, double h_max); void space_rebuild(struct space *s, double h_max, int verbose);
void space_recycle(struct space *s, struct cell *c); void space_recycle(struct space *s, struct cell *c);
void space_split(struct space *s, struct cell *c); void space_split(struct space *s, struct cell *c);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment