From 4a263c1ca869d81f7a17d91e228f3860c4bd8490 Mon Sep 17 00:00:00 2001
From: d74ksy <aidan.chalk@durham.ac.uk>
Date: Wed, 4 Nov 2015 12:02:06 +0000
Subject: [PATCH] Some fixes and additions. Now synchronises to get better
 timings

---
 examples/test_qr_mpi.c | 39 +++++++++++++++++++++++++++++++++------
 src/qsched.c           |  1 +
 2 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/examples/test_qr_mpi.c b/examples/test_qr_mpi.c
index 90af771..0a75ef7 100644
--- a/examples/test_qr_mpi.c
+++ b/examples/test_qr_mpi.c
@@ -21,6 +21,7 @@
 
 /* Standard includes. */
 #include <stdio.h>
+#include <errno.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
@@ -40,7 +41,7 @@
 #include "quicksched.h"
 #include "res.h"
 
-#define NO_TASK_TIMERS
+#define TASK_TIMERS
 
 #ifdef WITH_CBLAS
 /**
@@ -610,7 +611,7 @@ task_tids = (int*)calloc(sizeof(int), m*n*K*K);
 task_types = (int*)calloc(sizeof(int), m*n*K*K);
 #endif
 
-if(s.rank == 0){
+
   /* Allocate and fill the original matrix. */
   if ((A = (double*)malloc(sizeof(double)* m* n* K* K)) == NULL ||
       (tau = (double*)malloc(sizeof(double)* m* n* K)) == NULL ||
@@ -624,7 +625,7 @@ if(s.rank == 0){
   }
   memcpy(A, A_orig, sizeof(double) * m * n * K * K);
   bzero(tau, sizeof(double) * m * n * K);
-
+if(s.rank == 0){
   /* Initialize the scheduler. */
     /* Allocate and init the task ID and resource ID matrix. */
 //  tic = getticks();
@@ -778,8 +779,6 @@ qsched_run_MPI(&s, nr_threads, runner);
     printf("Hello world from processor %s, rank = %i, count_ranks = %i\n",
            processor_name, s.rank, s.count_ranks);
 
- //TODO Clean up the resource data.
-    qsched_free(&s);
 
 #ifdef TASK_TIMERS
 FILE *file = NULL;
@@ -795,7 +794,7 @@ FILE *file = NULL;
     for(i = 0; i < m*n*K*K; i++)
     {
         if(task_types[i] > 0)
-            fprintf(file, "%i %i %lli %lli\n", task_types[i], task_tids[i],task_start[i], task_finish[i]  );
+            fprintf(file, "%i %i %lli %lli\n", task_types[i], s.rank*nr_threads+task_tids[i],task_start[i], task_finish[i]  );
     }
 
 
@@ -805,6 +804,34 @@ FILE *file = NULL;
     free(task_start);
     free(task_finish);
 #endif
+
+    //TODO Copy all the resources back into A, synchronize them across the nodes and check correctness.
+    if(s.count_ranks > 1 && s.rank == 0)
+    {
+        errno = 0;
+        FILE *file2 = fopen("resource_allocation.out", "w");
+        if(file2 == NULL)
+            error("Failed to open file2 %d ", errno);
+        for(i = 0; i < n; i++)
+        {
+            for(j = 0; j < m; j++)
+            {
+                /*printf("j*m+i= %i\n", j*m+i);
+                printf("rid[j*m+i] = %lli\n", rid[j*m+i]);
+                printf("s.res = %p\n", &s.res[rid[j*m+i]]);
+                printf("s.res[rid[j*m+i]].node = %i\n", s.res[rid[j*m+i]].node);*/
+                fprintf(file2, "%i, ", s.res[rid[j*m+i]].node);
+            }
+            fprintf(file2, "\n");
+        }
+        fclose(file2);
+     
+    }
+ //TODO Clean up the resource data.
+    qsched_free(&s);
+    free(A);
+    free(A_orig);   
+    free(tau);
     // Finalize the MPI environment.
     MPI_Finalize();
 }
diff --git a/src/qsched.c b/src/qsched.c
index a30ae00..4d97515 100644
--- a/src/qsched.c
+++ b/src/qsched.c
@@ -3420,6 +3420,7 @@ void qsched_run_MPI ( struct qsched *s, int nr_threads, qsched_funtype fun ) {
 printf("Rank[%i]: qsched_prepare_mpi took %lli (= %e) ticks\n", s->rank,
          toc - tic, (float)(toc - tic));
 
+    MPI_Barrier(s->comm);
     message("Beginning execution of quicksched");
     tic = getticks();
     #if defined( HAVE_OPENMP )
-- 
GitLab