Commit 040aa5f0 authored by Pedro Gonnet's avatar Pedro Gonnet
Browse files

don't just assume that we have enough space to accommodate for strays,...

don't just assume that we have enough space to accommodate for strays, re-allocate the parts array if needed.


Former-commit-id: 3dc666ba37ab16f0fab536586e1d50498d0bc73e
parent 88cab578
...@@ -813,15 +813,14 @@ void engine_exchange_cells ( struct engine *e ) { ...@@ -813,15 +813,14 @@ void engine_exchange_cells ( struct engine *e ) {
* @brief Exchange straying parts with other nodes. * @brief Exchange straying parts with other nodes.
* *
* @param e The #engine. * @param e The #engine.
* @param parts An array of straying parts. * @param offset The index in the parts array as of which the foreign parts reside.
* @param xparts The corresponding xparts.
* @param ind The ID of the foreign #cell. * @param ind The ID of the foreign #cell.
* @param N The number of stray parts. * @param N The number of stray parts.
* *
* @return The number of arrived parts copied to parts and xparts. * @return The number of arrived parts copied to parts and xparts.
*/ */
int engine_exchange_strays ( struct engine *e , struct part *parts , struct xpart *xparts , int *ind , int N ) { int engine_exchange_strays ( struct engine *e , int offset , int *ind , int N ) {
#ifdef WITH_MPI #ifdef WITH_MPI
...@@ -830,6 +829,7 @@ int engine_exchange_strays ( struct engine *e , struct part *parts , struct xpar ...@@ -830,6 +829,7 @@ int engine_exchange_strays ( struct engine *e , struct part *parts , struct xpar
MPI_Request reqs_out[ 2*engine_maxproxies ]; MPI_Request reqs_out[ 2*engine_maxproxies ];
MPI_Status status; MPI_Status status;
struct proxy *p; struct proxy *p;
struct space *s = e->s;
/* Re-set the proxies. */ /* Re-set the proxies. */
for ( k = 0 ; k < e->nr_proxies ; k++ ) for ( k = 0 ; k < e->nr_proxies ; k++ )
...@@ -840,7 +840,7 @@ int engine_exchange_strays ( struct engine *e , struct part *parts , struct xpar ...@@ -840,7 +840,7 @@ int engine_exchange_strays ( struct engine *e , struct part *parts , struct xpar
pid = e->proxy_ind[ e->s->cells[ ind[k] ].nodeID ]; pid = e->proxy_ind[ e->s->cells[ ind[k] ].nodeID ];
if ( pid < 0 ) if ( pid < 0 )
error( "Do not have a proxy for the requested nodeID." ); error( "Do not have a proxy for the requested nodeID." );
proxy_parts_load( &e->proxies[pid] , &parts[k] , &xparts[k] , 1 ); proxy_parts_load( &e->proxies[pid] , &s->parts[offset + k] , &s->xparts[offset + k] , 1 );
} }
/* Launch the proxies. */ /* Launch the proxies. */
...@@ -863,8 +863,28 @@ int engine_exchange_strays ( struct engine *e , struct part *parts , struct xpar ...@@ -863,8 +863,28 @@ int engine_exchange_strays ( struct engine *e , struct part *parts , struct xpar
if ( MPI_Waitall( e->nr_proxies , reqs_out , MPI_STATUSES_IGNORE ) != MPI_SUCCESS ) if ( MPI_Waitall( e->nr_proxies , reqs_out , MPI_STATUSES_IGNORE ) != MPI_SUCCESS )
error( "MPI_Waitall on sends failed." ); error( "MPI_Waitall on sends failed." );
/* Return the number of harvested parts. */ /* Count the total number of incomming particles and make sure we have
/* Set the requests for the particle data. */ enough space to accommodate them. */
int count_in = 0;
for ( k = 0 ; k < e->nr_proxies ; k++ )
count_in += e->proxies[k].nr_parts_in;
message("sent out %i particles, got %i back.", N, count_in);
if ( offset + count_in > s->size_parts ) {
s->size_parts = (offset + count_in) * 1.05;
struct part *parts_new;
struct xpart *xparts_new;
if ( posix_memalign( (void **)&parts_new , part_align , sizeof(struct part) * s->size_parts ) != 0 ||
posix_memalign( (void **)&xparts_new , part_align , sizeof(struct xpart) * s->size_parts ) != 0 )
error( "Failed to allocate new part data." );
memcpy( parts_new , s->parts , sizeof(struct part) * offset );
memcpy( xparts_new , s->xparts , sizeof(struct xpart) * offset );
free( s->parts );
free( s->xparts );
s->parts = parts_new;
s->xparts = xparts_new;
}
/* Collect the requests for the particle data from the proxies. */
for ( k = 0 ; k < e->nr_proxies ; k++ ) { for ( k = 0 ; k < e->nr_proxies ; k++ ) {
if ( e->proxies[k].nr_parts_in > 0 ) { if ( e->proxies[k].nr_parts_in > 0 ) {
reqs_in[2*k] = e->proxies[k].req_parts_in; reqs_in[2*k] = e->proxies[k].req_parts_in;
...@@ -887,21 +907,19 @@ int engine_exchange_strays ( struct engine *e , struct part *parts , struct xpar ...@@ -887,21 +907,19 @@ int engine_exchange_strays ( struct engine *e , struct part *parts , struct xpar
for ( k = 0 ; k < 2*(nr_in + nr_out) ; k++ ) { for ( k = 0 ; k < 2*(nr_in + nr_out) ; k++ ) {
int err; int err;
if ( ( err = MPI_Waitany( 2*e->nr_proxies , reqs_in , &pid , &status ) ) != MPI_SUCCESS ) { if ( ( err = MPI_Waitany( 2*e->nr_proxies , reqs_in , &pid , &status ) ) != MPI_SUCCESS ) {
char buff[ MPI_MAX_ERROR_STRING ]; char buff[ MPI_MAX_ERROR_STRING ];
int res; int res;
MPI_Error_string( err , buff , &res ); MPI_Error_string( err , buff , &res );
error( "MPI_Waitany failed (%s)." , buff ); error( "MPI_Waitany failed (%s)." , buff );
} }
if ( pid == MPI_UNDEFINED )
break;
if ( pid == MPI_UNDEFINED ) if ( pid == MPI_UNDEFINED )
break; break;
// message( "request from proxy %i has arrived." , pid ); // message( "request from proxy %i has arrived." , pid );
if ( reqs_in[pid & ~1] == MPI_REQUEST_NULL && if ( reqs_in[pid & ~1] == MPI_REQUEST_NULL &&
reqs_in[pid | 1 ] == MPI_REQUEST_NULL ) { reqs_in[pid | 1 ] == MPI_REQUEST_NULL ) {
p = &e->proxies[pid/2]; p = &e->proxies[pid/2];
memcpy( &parts[count] , p->parts_in , sizeof(struct part) * p->nr_parts_in ); memcpy( &s->parts[offset + count] , p->parts_in , sizeof(struct part) * p->nr_parts_in );
memcpy( &xparts[count] , p->xparts_in , sizeof(struct xpart) * p->nr_parts_in ); memcpy( &s->xparts[offset + count] , p->xparts_in , sizeof(struct xpart) * p->nr_parts_in );
count += p->nr_parts_in; count += p->nr_parts_in;
/* for ( int k = 0 ; k < p->nr_parts_in ; k++ ) /* for ( int k = 0 ; k < p->nr_parts_in ; k++ )
message( "received particle %lli, x=[%.3e %.3e %.3e], h=%.3e, from node %i." , message( "received particle %lli, x=[%.3e %.3e %.3e], h=%.3e, from node %i." ,
...@@ -1993,6 +2011,7 @@ void engine_split ( struct engine *e , int *grid ) { ...@@ -1993,6 +2011,7 @@ void engine_split ( struct engine *e , int *grid ) {
engine_makeproxies( e ); engine_makeproxies( e );
/* Re-allocate the local parts. */ /* Re-allocate the local parts. */
message("Re-allocating parts array from %i to %i.", s->size_parts, (int)(s->nr_parts * 1.2));
s->size_parts = s->nr_parts * 1.2; s->size_parts = s->nr_parts * 1.2;
struct part *parts_new; struct part *parts_new;
struct xpart *xparts_new; struct xpart *xparts_new;
......
...@@ -127,7 +127,7 @@ void engine_prepare ( struct engine *e ); ...@@ -127,7 +127,7 @@ void engine_prepare ( struct engine *e );
void engine_step ( struct engine *e ); void engine_step ( struct engine *e );
void engine_maketasks ( struct engine *e ); void engine_maketasks ( struct engine *e );
void engine_split ( struct engine *e , int *grid ); void engine_split ( struct engine *e , int *grid );
int engine_exchange_strays ( struct engine *e , struct part *parts , struct xpart *xparts , int *ind , int N ); int engine_exchange_strays ( struct engine *e , int offset , int *ind , int N );
void engine_rebuild ( struct engine *e ); void engine_rebuild ( struct engine *e );
void engine_repartition ( struct engine *e ); void engine_repartition ( struct engine *e );
void engine_makeproxies ( struct engine *e ); void engine_makeproxies ( struct engine *e );
......
...@@ -357,7 +357,7 @@ void space_rebuild ( struct space *s , double cell_max ) { ...@@ -357,7 +357,7 @@ void space_rebuild ( struct space *s , double cell_max ) {
ind[k] = ind[ nr_parts ]; ind[k] = ind[ nr_parts ];
ind[ nr_parts ] = t; ind[ nr_parts ] = t;
} }
s->nr_parts = nr_parts + engine_exchange_strays( s->e , &parts[nr_parts] , &xparts[nr_parts] , &ind[nr_parts] , s->nr_parts - nr_parts ); s->nr_parts = nr_parts + engine_exchange_strays( s->e , nr_parts , &ind[nr_parts] , s->nr_parts - nr_parts );
for ( k = nr_parts ; k < s->nr_parts ; k++ ) { for ( k = nr_parts ; k < s->nr_parts ; k++ ) {
p = &parts[k]; p = &parts[k];
ind[k] = cell_getid( cdim , p->x[0]*ih[0] , p->x[1]*ih[1] , p->x[2]*ih[2] ); ind[k] = cell_getid( cdim , p->x[0]*ih[0] , p->x[1]*ih[1] , p->x[2]*ih[2] );
...@@ -1175,7 +1175,7 @@ void space_init ( struct space *s , double dim[3] , struct part *parts , int N , ...@@ -1175,7 +1175,7 @@ void space_init ( struct space *s , double dim[3] , struct part *parts , int N ,
} }
/* Allocate the xtra parts array. */ /* Allocate the xtra parts array. */
if ( posix_memalign( (void *)&s->xparts , 32 , N * sizeof(struct xpart) ) != 0 ) if ( posix_memalign( (void *)&s->xparts , part_align , N * sizeof(struct xpart) ) != 0 )
error( "Failed to allocate xparts." ); error( "Failed to allocate xparts." );
bzero( s->xparts , N * sizeof(struct xpart) ); bzero( s->xparts , N * sizeof(struct xpart) );
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment