force-split pairs that are too large to allow them to be done in parallel....

force-split pairs that are too large to allow them to be done in parallel. might wreak havoc in a few places so use with caution.


Former-commit-id: ab6b983a69769225314b6d46f04a88f98ab4ada1

src/engine.c

@@ -744,7 +744,7 @@ int engine_exchange_strays ( struct engine *e , struct part *parts , struct xpar
     /* Wait for each part array to come in and collect the new
        parts from the proxies. */
     for ( k = 0 ; k < 2*(nr_in + nr_out) ; k++ ) {
-        if ( MPI_Waitany( 2*e->nr_proxies , reqs_in , &pid , &status ) != MPI_SUCCESS ) {
+        if ( MPI_Waitany( 2*e->nr_proxies , reqs_in , &pid , &status ) != MPI_SUCCESS )
             error( "MPI_Waitany failed." );
         if ( pid == MPI_UNDEFINED )
             break;

src/scheduler.c

@@ -425,6 +425,25 @@ void scheduler_splittasks ( struct scheduler *s ) {
 
                 } /* split this task? */
                 
+            /* Otherwise, break it up if it is too large? */
+            else if ( scheduler_doforcesplit && ci->split && cj->split &&
+                      ( ci->count > space_maxsize && cj->count > space_maxsize ) ) {
+                      
+                message( "force splitting pair with %i and %i parts." , ci->count , cj->count );
+                      
+                /* Replace the current task. */
+                t->type = task_type_none;
+                
+                for ( j = 0 ; j < 8 ; j++ )
+                    if ( ci->progeny[j] != NULL )
+                        for ( k = 0 ; k < 8 ; k++ )
+                            if ( cj->progeny[k] != NULL ) {
+                                t = scheduler_addtask( s , task_type_pair , t->subtype , 0 , 0 , ci->progeny[j] , cj->progeny[k] , 0 );
+                                t->flags = space_getsid( s->space , &t->ci , &t->cj , shift );
+                                }
+                      
+                }
+                
             /* Otherwise, if not spilt, stitch-up the sorting. */
             else {
             

src/scheduler.h

@@ -24,6 +24,7 @@
 #define scheduler_dosub                      1
 #define scheduler_maxsteal                   10
 #define scheduler_maxtries                   2
+#define scheduler_doforcesplit               1
 
 /* Flags . */
 #define scheduler_flag_none                  0

src/space.c

@@ -53,6 +53,7 @@
 /* Split size. */
 int space_splitsize = space_splitsize_default;
 int space_subsize = space_subsize_default;
+int space_maxsize = space_maxsize_default;
 
 /* Map shift vector to sortlist. */
 const int sortlistID[27] = {

src/space.h

@@ -25,6 +25,7 @@
 #define space_cellallocchunk            1000
 #define space_splitratio                0.875f
 #define space_splitsize_default         400
+#define space_maxsize_default           3000
 #define space_subsize_default           5000
 #define space_stretch                   1.10f
 #define space_maxreldx                  0.25f
@@ -36,6 +37,7 @@
 
 /* Split size. */
 extern int space_splitsize;
+extern int space_maxsize;
 extern int space_subsize;
 
 /* Map shift vector to sortlist. */

src/task.h

@@ -20,7 +20,7 @@
 
 /* Some constants. */
 #define task_maxwait                    3
-#define task_maxunlock                  46
+#define task_maxunlock                  64
 
 
 /* The different task types. */