diff --git a/src/cell.h b/src/cell.h index 548da065c1ce6f2c0d1e09c8ca4caea5241b0faa..4e05aebf0342adfe1db998371f311461728f421c 100644 --- a/src/cell.h +++ b/src/cell.h @@ -133,7 +133,7 @@ struct cell { int nodeID; /* Bit mask of the proxies this cell is registered with. */ - unsigned int sendto; + unsigned long long int sendto; /* Pointer to this cell's packed representation. */ struct pcell *pcell; diff --git a/src/engine.c b/src/engine.c index 362c1ea278483c0a2c84d8d44cfa068f54e84545..1f6a7acf5bfba809e9a9c39f79769e463a61381a 100644 --- a/src/engine.c +++ b/src/engine.c @@ -91,7 +91,7 @@ void engine_repartition ( struct engine *e ) { idx_t *inds; idx_t *weights_v, *weights_e; struct space *s = e->s; - int nr_cells = s->nr_cells; + int nr_cells = s->nr_cells, my_cells = 0; struct cell *cells = s->cells; int ind[3], *cdim = s->cdim; struct task *t, *tasks = e->sched.tasks; @@ -254,8 +254,11 @@ void engine_repartition ( struct engine *e ) { error( "Failed to bcast the node IDs." ); /* Set the cell nodeIDs and clear any non-local parts. */ - for ( k = 0 ; k < nr_cells ; k++ ) + for ( k = 0 ; k < nr_cells ; k++ ) { cells[k].nodeID = nodeIDs[k]; + if ( nodeIDs[k] == nodeID ) + my_cells += 1; + } /* Clean up. */ free( inds ); @@ -349,7 +352,7 @@ void engine_repartition ( struct engine *e ) { s->size_parts = 2*nr_parts; /* Be verbose about what just happened. */ - message( "node %i now has %i parts." , nodeID , nr_parts ); + message( "node %i now has %i parts in %i cells." , nodeID , nr_parts , my_cells ); /* Clean up other stuff. */ free( reqs ); @@ -553,7 +556,7 @@ void engine_exchange_cells ( struct engine *e ) { } /* Wait for all the sends to have finnished too. */ - if ( MPI_Waitall( nr_proxies , reqs_out , &status ) != MPI_SUCCESS ) + if ( MPI_Waitall( nr_proxies , reqs_out , MPI_STATUSES_IGNORE ) != MPI_SUCCESS ) error( "MPI_Waitall on sends failed." ); /* Count the number of particles we need to import and re-allocate @@ -677,7 +680,7 @@ int engine_exchange_strays ( struct engine *e , struct part *parts , struct xpar /* Wait for all the sends to have finnished too. */ if ( nr_out > 0 ) - if ( MPI_Waitall( e->nr_proxies , reqs_out , &status ) != MPI_SUCCESS ) + if ( MPI_Waitall( e->nr_proxies , reqs_out , MPI_STATUSES_IGNORE ) != MPI_SUCCESS ) error( "MPI_Waitall on sends failed." ); /* Return the number of harvested parts. */ @@ -1597,6 +1600,8 @@ void engine_makeproxies ( struct engine *e ) { if ( cells[cid].nodeID == e->nodeID && cells[cjd].nodeID != e->nodeID ) { pid = e->proxy_ind[ cells[cjd].nodeID ]; if ( pid < 0 ) { + if ( e->nr_proxies == engine_maxproxies ) + error( "Maximum number of proxies exceeded." ); proxy_init( &proxies[ e->nr_proxies ] , e->nodeID , cells[cjd].nodeID ); e->proxy_ind[ cells[cjd].nodeID ] = e->nr_proxies; pid = e->nr_proxies; @@ -1604,12 +1609,14 @@ void engine_makeproxies ( struct engine *e ) { } proxy_addcell_in( &proxies[pid] , &cells[cjd] ); proxy_addcell_out( &proxies[pid] , &cells[cid] ); - cells[cid].sendto |= ( 1 << pid ); + cells[cid].sendto |= ( 1ULL << pid ); } if ( cells[cjd].nodeID == e->nodeID && cells[cid].nodeID != e->nodeID ) { pid = e->proxy_ind[ cells[cid].nodeID ]; if ( pid < 0 ) { + if ( e->nr_proxies == engine_maxproxies ) + error( "Maximum number of proxies exceeded." ); proxy_init( &proxies[ e->nr_proxies ] , e->nodeID , cells[cid].nodeID ); e->proxy_ind[ cells[cid].nodeID ] = e->nr_proxies; pid = e->nr_proxies; @@ -1617,7 +1624,7 @@ void engine_makeproxies ( struct engine *e ) { } proxy_addcell_in( &proxies[pid] , &cells[cid] ); proxy_addcell_out( &proxies[pid] , &cells[cjd] ); - cells[cjd].sendto |= ( 1 << pid ); + cells[cjd].sendto |= ( 1ULL << pid ); } } } diff --git a/src/engine.h b/src/engine.h index c16d47a184ce0830a2c9ec7179915fae2109589e..a9f6e6d254c47d3f027df738dceeb7054cf4bca8 100644 --- a/src/engine.h +++ b/src/engine.h @@ -33,7 +33,7 @@ #define engine_queue_scale 1.2 #define engine_maxtaskspercell 32 -#define engine_maxproxies 36 +#define engine_maxproxies 64 /* The rank of the engine as a global variable (for messages). */