diff --git a/src/engine.c b/src/engine.c index 2c46aa768f5186719658159b90ccca329dc926c8..362c1ea278483c0a2c84d8d44cfa068f54e84545 100644 --- a/src/engine.c +++ b/src/engine.c @@ -609,7 +609,7 @@ int engine_exchange_strays ( struct engine *e , struct part *parts , struct xpar #ifdef WITH_MPI - int k, pid, count = 0; + int k, pid, count = 0, nr_in = 0, nr_out = 0; MPI_Request reqs_in[ engine_maxproxies ]; MPI_Request reqs_out[ engine_maxproxies ]; MPI_Status status; @@ -644,13 +644,23 @@ int engine_exchange_strays ( struct engine *e , struct part *parts , struct xpar /* Set the requests for the particle data. */ for ( k = 0 ; k < e->nr_proxies ; k++ ) { - reqs_in[k] = e->proxies[k].req_xparts_in; - reqs_out[k] = e->proxies[k].req_xparts_out; + if ( e->proxies[k].nr_parts_in > 0 ) { + reqs_in[k] = e->proxies[k].req_xparts_in; + nr_in += 1; + } + else + reqs_in[k] = MPI_REQUEST_NULL; + if ( e->proxies[k].nr_parts_out > 0 ) { + reqs_out[k] = e->proxies[k].req_xparts_out; + nr_out += 1; + } + else + reqs_out[k] = MPI_REQUEST_NULL; } /* Wait for each part array to come in and collect the new parts from the proxies. */ - for ( k = 0 ; k < e->nr_proxies ; k++ ) { + for ( k = 0 ; k < nr_in ; k++ ) { if ( MPI_Waitany( e->nr_proxies , reqs_in , &pid , &status ) != MPI_SUCCESS || pid == MPI_UNDEFINED ) error( "MPI_Waitany failed." ); @@ -666,8 +676,9 @@ int engine_exchange_strays ( struct engine *e , struct part *parts , struct xpar } /* Wait for all the sends to have finnished too. */ - if ( MPI_Waitall( e->nr_proxies , reqs_out , &status ) != MPI_SUCCESS ) - error( "MPI_Waitall on sends failed." ); + if ( nr_out > 0 ) + if ( MPI_Waitall( e->nr_proxies , reqs_out , &status ) != MPI_SUCCESS ) + error( "MPI_Waitall on sends failed." ); /* Return the number of harvested parts. */ return count; diff --git a/src/proxy.c b/src/proxy.c index 8191b3d0b7c060e5d2a950fb208726957fedbfb8..acc1499789f9c6ea9c1a26550a99afdec6e1d320 100644 --- a/src/proxy.c +++ b/src/proxy.c @@ -211,15 +211,17 @@ void proxy_parts_exch1 ( struct proxy *p ) { // message( "isent particle count (%i) from node %i to node %i." , p->nr_parts_out , p->mynodeID , p->nodeID ); fflush(stdout); /* Send the particle buffers. */ - if ( MPI_Isend( p->parts_out , sizeof(struct part)*p->nr_parts_out , MPI_BYTE , p->nodeID , p->mynodeID*proxy_tag_shift + proxy_tag_parts , MPI_COMM_WORLD , &p->req_parts_out ) != MPI_SUCCESS || - MPI_Isend( p->xparts_out , sizeof(struct xpart)*p->nr_parts_out , MPI_BYTE , p->nodeID , p->mynodeID*proxy_tag_shift + proxy_tag_xparts , MPI_COMM_WORLD , &p->req_xparts_out ) != MPI_SUCCESS ) - error( "Failed to isend part data." ); - MPI_Request_free( &p->req_parts_out ); - // message( "isent particle data (%i) to node %i." , p->nr_parts_out , p->nodeID ); fflush(stdout); - /* for ( int k = 0 ; k < p->nr_parts_out ; k++ ) - message( "sending particle %lli, x=[%.3e %.3e %.3e], h=%.3e, to node %i." , - p->parts_out[k].id , p->parts_out[k].x[0] , p->parts_out[k].x[1] , p->parts_out[k].x[2] , - p->parts_out[k].h , p->nodeID ); */ + if ( p->nr_parts_out > 0 ) { + if ( MPI_Isend( p->parts_out , sizeof(struct part)*p->nr_parts_out , MPI_BYTE , p->nodeID , p->mynodeID*proxy_tag_shift + proxy_tag_parts , MPI_COMM_WORLD , &p->req_parts_out ) != MPI_SUCCESS || + MPI_Isend( p->xparts_out , sizeof(struct xpart)*p->nr_parts_out , MPI_BYTE , p->nodeID , p->mynodeID*proxy_tag_shift + proxy_tag_xparts , MPI_COMM_WORLD , &p->req_xparts_out ) != MPI_SUCCESS ) + error( "Failed to isend part data." ); + MPI_Request_free( &p->req_parts_out ); + // message( "isent particle data (%i) to node %i." , p->nr_parts_out , p->nodeID ); fflush(stdout); + /* for ( int k = 0 ; k < p->nr_parts_out ; k++ ) + message( "sending particle %lli, x=[%.3e %.3e %.3e], h=%.3e, to node %i." , + p->parts_out[k].id , p->parts_out[k].x[0] , p->parts_out[k].x[1] , p->parts_out[k].x[2] , + p->parts_out[k].h , p->nodeID ); */ + } /* Receive the number of particles. */ if ( MPI_Irecv( &p->nr_parts_in , 1 , MPI_INT , p->nodeID , p->nodeID*proxy_tag_shift + proxy_tag_count , MPI_COMM_WORLD , &p->req_parts_count_in ) != MPI_SUCCESS ) @@ -249,10 +251,12 @@ void proxy_parts_exch2 ( struct proxy *p ) { } /* Receive the particle buffers. */ - if ( MPI_Irecv( p->parts_in , sizeof(struct part)*p->nr_parts_in , MPI_BYTE , p->nodeID , p->nodeID*proxy_tag_shift + proxy_tag_parts , MPI_COMM_WORLD , &p->req_parts_in ) != MPI_SUCCESS || - MPI_Irecv( p->xparts_in , sizeof(struct xpart)*p->nr_parts_in , MPI_BYTE , p->nodeID , p->nodeID*proxy_tag_shift + proxy_tag_xparts , MPI_COMM_WORLD , &p->req_xparts_in ) != MPI_SUCCESS ) - error( "Failed to irecv part data." ); - // message( "irecv particle data (%i) from node %i." , p->nr_parts_in , p->nodeID ); fflush(stdout); + if ( p->nr_parts_in > 0 ) { + if ( MPI_Irecv( p->parts_in , sizeof(struct part)*p->nr_parts_in , MPI_BYTE , p->nodeID , p->nodeID*proxy_tag_shift + proxy_tag_parts , MPI_COMM_WORLD , &p->req_parts_in ) != MPI_SUCCESS || + MPI_Irecv( p->xparts_in , sizeof(struct xpart)*p->nr_parts_in , MPI_BYTE , p->nodeID , p->nodeID*proxy_tag_shift + proxy_tag_xparts , MPI_COMM_WORLD , &p->req_xparts_in ) != MPI_SUCCESS ) + error( "Failed to irecv part data." ); + // message( "irecv particle data (%i) from node %i." , p->nr_parts_in , p->nodeID ); fflush(stdout); + } #else error( "SWIFT was not compiled with MPI support." );