Skip to content
Snippets Groups Projects
Commit ade8f509 authored by Pedro Gonnet's avatar Pedro Gonnet
Browse files

allow for 64-bit indices in METIS. 

Former-commit-id: 8dace811a1d3e0fbb6f28fbd6783afbfa2e7014e
parent 21e577f1
Branches
Tags
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 50 additions and 56 deletions
src/engine.c
+ 50
− 56
View file @ ade8f509
......@@ -229,12 +229,12 @@ void engine_redistribute ( struct engine *e ) {
int res;
if ( ( res = MPI_Waitall( 4*nr_nodes , reqs , stats ) ) != MPI_SUCCESS ) {
for ( k = 0 ; k < 4*nr_nodes ; k++ ) {
char buff[ MPI_MAX_ERROR_STRING ];
int res;
MPI_Error_string( stats[k].MPI_ERROR , buff , &res );
message( "request %i has error '%s'." , k , buff );
}
message( "counts is [ %i %i %i %i ]." , counts[0] , counts[1] , counts[2] , counts[3] );
char buff[ MPI_MAX_ERROR_STRING ];
int res;
MPI_Error_string( stats[k].MPI_ERROR , buff , &res );
message( "request %i has error '%s'." , k , buff );
}
message( "counts is [ %i %i %i %i ]." , counts[0] , counts[1] , counts[2] , counts[3] );
error( "Failed during waitall for part data." );
}
......@@ -282,7 +282,7 @@ void engine_repartition ( struct engine *e ) {
#if defined(WITH_MPI) && defined(HAVE_METIS)
int i, j, k, l, cid, cjd, ii, jj, kk, res, w;
idx_t *inds;
idx_t *inds, *nodeIDs;
idx_t *weights_v, *weights_e;
struct space *s = e->s;
int nr_cells = s->nr_cells, my_cells = 0;
......@@ -290,7 +290,7 @@ void engine_repartition ( struct engine *e ) {
int ind[3], *cdim = s->cdim;
struct task *t, *tasks = e->sched.tasks;
struct cell *ci, *cj;
int nr_nodes = e->nr_nodes, nodeID = e->nodeID, *nodeIDs;
int nr_nodes = e->nr_nodes, nodeID = e->nodeID;
float wscale = 0.0001, vscale = 0.001;
/* Clear the repartition flag. */
......@@ -424,13 +424,21 @@ void engine_repartition ( struct engine *e ) {
}
/* Merge the weights arrays accross all nodes. */
#if IDXTYPEWIDTH==32
if ( ( res = MPI_Reduce( ( nodeID == 0 ) ? MPI_IN_PLACE : weights_v , weights_v , nr_cells , MPI_INT , MPI_SUM , 0 , MPI_COMM_WORLD ) ) != MPI_SUCCESS ) {
#else
if ( ( res = MPI_Reduce( ( nodeID == 0 ) ? MPI_IN_PLACE : weights_v , weights_v , nr_cells , MPI_LONG_LONG_INT , MPI_SUM , 0 , MPI_COMM_WORLD ) ) != MPI_SUCCESS ) {
#endif
char buff[ MPI_MAX_ERROR_STRING ];
MPI_Error_string( res , buff , &i );
error( "Failed to allreduce vertex weights (%s)." , buff );
}
#if IDXTYPEWIDTH==32
if ( MPI_Reduce( ( nodeID == 0 ) ? MPI_IN_PLACE : weights_e , weights_e , 26*nr_cells , MPI_INT , MPI_SUM , 0 , MPI_COMM_WORLD ) != MPI_SUCCESS )
error( "Failed to allreduce edge weights." );
#else
if ( MPI_Reduce( ( nodeID == 0 ) ? MPI_IN_PLACE : weights_e , weights_e , 26*nr_cells , MPI_LONG_LONG_INT , MPI_SUM , 0 , MPI_COMM_WORLD ) != MPI_SUCCESS )
#endif
error( "Failed to allreduce edge weights." );
/* As of here, only one node needs to compute the partition. */
if ( nodeID == 0 ) {
......@@ -466,9 +474,9 @@ void engine_repartition ( struct engine *e ) {
for ( k = 0 ; k < 26*nr_cells ; k++ )
if ( weights_e[k] == 0 )
weights_e[k] = 1;
for ( k = 0 ; k < nr_cells ; k++ )
for ( k = 0 ; k < nr_cells ; k++ )
if ( ( weights_v[k] *= vscale ) == 0 )
weights_v[k] = 1;
weights_v[k] = 1;
/* Allocate and fill the connection array. */
idx_t *offsets;
......@@ -484,27 +492,25 @@ void engine_repartition ( struct engine *e ) {
options[ METIS_OPTION_OBJTYPE ] = METIS_OBJTYPE_CUT;
options[ METIS_OPTION_NUMBERING ] = 0;
options[ METIS_OPTION_CONTIG ] = 1;
options[ METIS_OPTION_NCUTS ] = 10;
options[ METIS_OPTION_NITER ] = 20;
// options[ METIS_OPTION_UFACTOR ] = 1;
options[ METIS_OPTION_NCUTS ] = 10;
options[ METIS_OPTION_NITER ] = 20;
// options[ METIS_OPTION_UFACTOR ] = 1;
/* Set the initial partition, although this is probably ignored. */
for ( k = 0 ; k < nr_cells ; k++ )
nodeIDs[k] = cells[k].nodeID;
/* Call METIS. */
int one = 1;
idx_t one = 1, idx_nr_cells = nr_cells, idx_nr_nodes = nr_nodes;
idx_t objval;
if ( METIS_PartGraphRecursive( &nr_cells , &one , offsets , inds , weights_v , NULL , weights_e , &nr_nodes , NULL , NULL , options , &objval , nodeIDs ) != METIS_OK )
if ( METIS_PartGraphRecursive( &idx_nr_cells , &one , offsets , inds , weights_v , NULL , weights_e , &idx_nr_nodes , NULL , NULL , options , &objval , nodeIDs ) != METIS_OK )
error( "Call to METIS_PartGraphKway failed." );
/* Dump the 3d array of cell IDs. */
printf( "engine_repartition: nodeIDs = [" );
for ( i = 0 ; i < cdim[0] ; i++ )
for ( j = 0 ; j < cdim[1] ; j++ )
for ( k = 0 ; k < cdim[2] ; k++ )
printf( "%i " , nodeIDs[ cell_getid( cdim , i , j , k ) ] );
printf("]; nodeIDs = reshape(nodeIDs,%i,%i,%i);\n",cdim[0],cdim[1],cdim[2]);
/* Dump the 3d array of cell IDs. */
printf( "engine_repartition: nodeIDs = reshape( [" );
for ( i = 0 ; i < cdim[0]*cdim[1]*cdim[2] ; i++ )
printf( "%i " , nodeIDs[ i ] );
printf("] ,%i,%i,%i);\n",cdim[0],cdim[1],cdim[2]);
}
......@@ -832,13 +838,13 @@ int engine_exchange_strays ( struct engine *e , struct part *parts , struct xpar
for ( k = 0 ; k < 2*(nr_in + nr_out) ; k++ ) {
int err;
if ( ( err = MPI_Waitany( 2*e->nr_proxies , reqs_in , &pid , &status ) ) != MPI_SUCCESS ) {
char buff[ MPI_MAX_ERROR_STRING ];
int res;
MPI_Error_string( err , buff , &res );
char buff[ MPI_MAX_ERROR_STRING ];
int res;
MPI_Error_string( err , buff , &res );
error( "MPI_Waitany failed (%s)." , buff );
}
if ( pid == MPI_UNDEFINED )
break;
}
if ( pid == MPI_UNDEFINED )
break;
if ( pid == MPI_UNDEFINED )
break;
// message( "request from proxy %i has arrived." , pid );
......@@ -958,7 +964,7 @@ void engine_maketasks ( struct engine *e ) {
t->ci->density = engine_addlink( e , t->ci->density , t );
atomic_inc( &t->ci->nr_density );
if ( t->ci->nr_density > 27*8 )
error( "Density overflow." );
error( "Density overflow." );
}
}
else if ( t->type == task_type_pair ) {
......@@ -969,8 +975,8 @@ void engine_maketasks ( struct engine *e ) {
atomic_inc( &t->ci->nr_density );
t->cj->density = engine_addlink( e , t->cj->density , t );
atomic_inc( &t->cj->nr_density );
if ( t->ci->nr_density > 8*27 || t->cj->nr_density > 8*27 )
error( "Density overflow." );
if ( t->ci->nr_density > 8*27 || t->cj->nr_density > 8*27 )
error( "Density overflow." );
}
}
else if ( t->type == task_type_sub ) {
......@@ -983,9 +989,9 @@ void engine_maketasks ( struct engine *e ) {
if ( t->cj != NULL ) {
t->cj->density = engine_addlink( e , t->cj->density , t );
atomic_inc( &t->cj->nr_density );
if ( t->cj->nr_density > 8*27 )
error( "Density overflow." );
}
if ( t->cj->nr_density > 8*27 )
error( "Density overflow." );
}
}
}
}
......@@ -1313,7 +1319,7 @@ void engine_prepare ( struct engine *e ) {
if ( MPI_Allreduce( &rebuild , &buff , 1 , MPI_INT , MPI_MAX , MPI_COMM_WORLD ) != MPI_SUCCESS )
error( "Failed to aggreggate the rebuild flag accross nodes." );
rebuild = buff;
// message( "rebuild allreduce took %.3f ms." , (double)(getticks() - tic)/CPU_TPS*1000 );
// message( "rebuild allreduce took %.3f ms." , (double)(getticks() - tic)/CPU_TPS*1000 );
#endif
e->tic_step = getticks();
......@@ -1321,15 +1327,15 @@ void engine_prepare ( struct engine *e ) {
if ( rebuild ) {
// tic = getticks();
engine_rebuild( e );
// message( "engine_rebuild took %.3f ms." , (double)(getticks() - tic)/CPU_TPS*1000 );
}
// message( "engine_rebuild took %.3f ms." , (double)(getticks() - tic)/CPU_TPS*1000 );
}
/* Re-rank the tasks every now and then. */
if ( e->tasks_age % engine_tasksreweight == 1 ) {
// tic = getticks();
// tic = getticks();
scheduler_reweight( &e->sched );
// message( "scheduler_reweight took %.3f ms." , (double)(getticks() - tic)/CPU_TPS*1000 );
}
// message( "scheduler_reweight took %.3f ms." , (double)(getticks() - tic)/CPU_TPS*1000 );
}
e->tasks_age += 1;
TIMER_TOC( timer_prepare );
......@@ -1444,9 +1450,9 @@ void engine_collect_kick2 ( struct cell *c ) {
// ih = 1.0f / p.h;
// p.rho = 0.0f; p.rho_dh = 0.0f;
// p.density.wcount = 0.0f; p.density.wcount_dh = 0.0f;
// p.density.div_v = 0.0;
// for ( k=0 ; k < 3 ; k++)
// p.density.curl_v[k] = 0.0;
// p.density.div_v = 0.0;
// for ( k=0 ; k < 3 ; k++)
// p.density.curl_v[k] = 0.0;
//
// /* Loop over all particle pairs (force). */
// for ( k = 0 ; k < N ; k++ ) {
......@@ -1877,7 +1883,6 @@ void engine_split ( struct engine *e , int *grid ) {
int ind[3];
struct space *s = e->s;
struct cell *c;
struct part *p;
/* If we've got the wrong number of nodes, fail. */
if ( e->nr_nodes != grid[0]*grid[1]*grid[2] )
......@@ -1898,17 +1903,6 @@ void engine_split ( struct engine *e , int *grid ) {
/* Make the proxies. */
engine_makeproxies( e );
/* For now, just kill any particle outside of our grid. */
for ( k = 0 ; k < s->nr_parts ; k++ ) {
p = &s->parts[k];
if ( s->cells[ cell_getid( s->cdim , p->x[0]*s->ih[0] , p->x[1]*s->ih[1] , p->x[2]*s->ih[2] ) ].nodeID != e->nodeID ) {
s->nr_parts -= 1;
s->parts[k] = s->parts[ s->nr_parts ];
s->xparts[k] = s->xparts[ s->nr_parts ];
k -= 1;
}
}
/* Re-allocate the local parts. */
s->size_parts = s->nr_parts * 1.2;
struct part *parts_new;
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment